// SPDX-License-Identifier: GPL-2.0
/*
 * NVMe over Fabrics Persist Reservation.
 * Copyright (c) 2024 Guixin Liu, Alibaba Group.
 * All rights reserved.
 */
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/unaligned.h>
#include "nvmet.h"

#define NVMET_PR_NOTIFI_MASK_ALL \
        (1 << NVME_PR_NOTIFY_BIT_REG_PREEMPTED | \
         1 << NVME_PR_NOTIFY_BIT_RESV_RELEASED | \
         1 << NVME_PR_NOTIFY_BIT_RESV_PREEMPTED)

static inline bool nvmet_pr_parse_ignore_key(u32 cdw10)
{
        /* Ignore existing key, bit 03. */
        return (cdw10 >> 3) & 1;
}

static inline struct nvmet_ns *nvmet_pr_to_ns(struct nvmet_pr *pr)
{
        return container_of(pr, struct nvmet_ns, pr);
}

static struct nvmet_pr_registrant *
nvmet_pr_find_registrant(struct nvmet_pr *pr, uuid_t *hostid)
{
        struct nvmet_pr_registrant *reg;

        list_for_each_entry_rcu(reg, &pr->registrant_list, entry) {
                if (uuid_equal(&reg->hostid, hostid))
                        return reg;
        }
        return NULL;
}

u16 nvmet_set_feat_resv_notif_mask(struct nvmet_req *req, u32 mask)
{
        u32 nsid = le32_to_cpu(req->cmd->common.nsid);
        struct nvmet_ctrl *ctrl = req->sq->ctrl;
        struct nvmet_ns *ns;
        unsigned long idx;
        u16 status;

        if (mask & ~(NVMET_PR_NOTIFI_MASK_ALL)) {
                req->error_loc = offsetof(struct nvme_common_command, cdw11);
                return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
        }

        if (nsid != U32_MAX) {
                status = nvmet_req_find_ns(req);
                if (status)
                        return status;
                if (!req->ns->pr.enable)
                        return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;

                WRITE_ONCE(req->ns->pr.notify_mask, mask);
                goto success;
        }

        nvmet_for_each_enabled_ns(&ctrl->subsys->namespaces, idx, ns) {
                if (ns->pr.enable)
                        WRITE_ONCE(ns->pr.notify_mask, mask);
        }

success:
        nvmet_set_result(req, mask);
        return NVME_SC_SUCCESS;
}

u16 nvmet_get_feat_resv_notif_mask(struct nvmet_req *req)
{
        u16 status;

        status = nvmet_req_find_ns(req);
        if (status)
                return status;

        if (!req->ns->pr.enable)
                return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;

        nvmet_set_result(req, READ_ONCE(req->ns->pr.notify_mask));
        return status;
}

void nvmet_execute_get_log_page_resv(struct nvmet_req *req)
{
        struct nvmet_pr_log_mgr *log_mgr = &req->sq->ctrl->pr_log_mgr;
        struct nvme_pr_log next_log = {0};
        struct nvme_pr_log log = {0};
        u16 status = NVME_SC_SUCCESS;
        u64 lost_count;
        u64 cur_count;
        u64 next_count;

        mutex_lock(&log_mgr->lock);
        if (!kfifo_get(&log_mgr->log_queue, &log))
                goto out;

        /*
         * We can't get the last in kfifo.
         * Utilize the current count and the count from the next log to
         * calculate the number of lost logs, while also addressing cases
         * of overflow. If there is no subsequent log, the number of lost
         * logs is equal to the lost_count within the nvmet_pr_log_mgr.
         */
        cur_count = le64_to_cpu(log.count);
        if (kfifo_peek(&log_mgr->log_queue, &next_log)) {
                next_count = le64_to_cpu(next_log.count);
                if (next_count > cur_count)
                        lost_count = next_count - cur_count - 1;
                else
                        lost_count = U64_MAX - cur_count + next_count - 1;
        } else {
                lost_count = log_mgr->lost_count;
        }

        log.count = cpu_to_le64((cur_count + lost_count) == 0 ?
                                1 : (cur_count + lost_count));
        log_mgr->lost_count -= lost_count;

        log.nr_pages = kfifo_len(&log_mgr->log_queue);

out:
        status = nvmet_copy_to_sgl(req, 0, &log, sizeof(log));
        mutex_unlock(&log_mgr->lock);
        nvmet_req_complete(req, status);
}

static void nvmet_pr_add_resv_log(struct nvmet_ctrl *ctrl, u8 log_type,
                                  u32 nsid)
{
        struct nvmet_pr_log_mgr *log_mgr = &ctrl->pr_log_mgr;
        struct nvme_pr_log log = {0};

        mutex_lock(&log_mgr->lock);
        log_mgr->counter++;
        if (log_mgr->counter == 0)
                log_mgr->counter = 1;

        log.count = cpu_to_le64(log_mgr->counter);
        log.type = log_type;
        log.nsid = cpu_to_le32(nsid);

        if (!kfifo_put(&log_mgr->log_queue, log)) {
                pr_info("a reservation log lost, cntlid:%d, log_type:%d, nsid:%d\n",
                        ctrl->cntlid, log_type, nsid);
                log_mgr->lost_count++;
        }

        mutex_unlock(&log_mgr->lock);
}

static void nvmet_pr_resv_released(struct nvmet_pr *pr, uuid_t *hostid)
{
        struct nvmet_ns *ns = nvmet_pr_to_ns(pr);
        struct nvmet_subsys *subsys = ns->subsys;
        struct nvmet_ctrl *ctrl;

        if (test_bit(NVME_PR_NOTIFY_BIT_RESV_RELEASED, &pr->notify_mask))
                return;

        mutex_lock(&subsys->lock);
        list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
                if (!uuid_equal(&ctrl->hostid, hostid) &&
                    nvmet_pr_find_registrant(pr, &ctrl->hostid)) {
                        nvmet_pr_add_resv_log(ctrl,
                                NVME_PR_LOG_RESERVATION_RELEASED, ns->nsid);
                        nvmet_add_async_event(ctrl, NVME_AER_CSS,
                                NVME_AEN_RESV_LOG_PAGE_AVALIABLE,
                                NVME_LOG_RESERVATION);
                }
        }
        mutex_unlock(&subsys->lock);
}

static void nvmet_pr_send_event_to_host(struct nvmet_pr *pr, uuid_t *hostid,
                                          u8 log_type)
{
        struct nvmet_ns *ns = nvmet_pr_to_ns(pr);
        struct nvmet_subsys *subsys = ns->subsys;
        struct nvmet_ctrl *ctrl;

        mutex_lock(&subsys->lock);
        list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
                if (uuid_equal(hostid, &ctrl->hostid)) {
                        nvmet_pr_add_resv_log(ctrl, log_type, ns->nsid);
                        nvmet_add_async_event(ctrl, NVME_AER_CSS,
                                NVME_AEN_RESV_LOG_PAGE_AVALIABLE,
                                NVME_LOG_RESERVATION);
                }
        }
        mutex_unlock(&subsys->lock);
}

static void nvmet_pr_resv_preempted(struct nvmet_pr *pr, uuid_t *hostid)
{
        if (test_bit(NVME_PR_NOTIFY_BIT_RESV_PREEMPTED, &pr->notify_mask))
                return;

        nvmet_pr_send_event_to_host(pr, hostid,
                NVME_PR_LOG_RESERVATOIN_PREEMPTED);
}

static void nvmet_pr_registration_preempted(struct nvmet_pr *pr,
                                            uuid_t *hostid)
{
        if (test_bit(NVME_PR_NOTIFY_BIT_REG_PREEMPTED, &pr->notify_mask))
                return;

        nvmet_pr_send_event_to_host(pr, hostid,
                NVME_PR_LOG_REGISTRATION_PREEMPTED);
}

static inline void nvmet_pr_set_new_holder(struct nvmet_pr *pr, u8 new_rtype,
                                           struct nvmet_pr_registrant *reg)
{
        reg->rtype = new_rtype;
        rcu_assign_pointer(pr->holder, reg);
}

static u16 nvmet_pr_register(struct nvmet_req *req,
                             struct nvmet_pr_register_data *d)
{
        struct nvmet_ctrl *ctrl = req->sq->ctrl;
        struct nvmet_pr_registrant *new, *reg;
        struct nvmet_pr *pr = &req->ns->pr;
        u16 status = NVME_SC_SUCCESS;
        u64 nrkey = le64_to_cpu(d->nrkey);

        new = kmalloc_obj(*new);
        if (!new)
                return NVME_SC_INTERNAL;

        down(&pr->pr_sem);
        reg = nvmet_pr_find_registrant(pr, &ctrl->hostid);
        if (reg) {
                if (reg->rkey != nrkey)
                        status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
                kfree(new);
                goto out;
        }

        memset(new, 0, sizeof(*new));
        INIT_LIST_HEAD(&new->entry);
        new->rkey = nrkey;
        uuid_copy(&new->hostid, &ctrl->hostid);
        list_add_tail_rcu(&new->entry, &pr->registrant_list);

out:
        up(&pr->pr_sem);
        return status;
}

static void nvmet_pr_unregister_one(struct nvmet_pr *pr,
                                    struct nvmet_pr_registrant *reg)
{
        struct nvmet_pr_registrant *first_reg;
        struct nvmet_pr_registrant *holder;
        u8 original_rtype;

        list_del_rcu(&reg->entry);

        holder = rcu_dereference_protected(pr->holder, 1);
        if (reg != holder)
                goto out;

        original_rtype = holder->rtype;
        if (original_rtype == NVME_PR_WRITE_EXCLUSIVE_ALL_REGS ||
            original_rtype == NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS) {
                first_reg = list_first_or_null_rcu(&pr->registrant_list,
                                struct nvmet_pr_registrant, entry);
                if (first_reg)
                        first_reg->rtype = original_rtype;
                rcu_assign_pointer(pr->holder, first_reg);
        } else {
                rcu_assign_pointer(pr->holder, NULL);

                if (original_rtype == NVME_PR_WRITE_EXCLUSIVE_REG_ONLY ||
                    original_rtype == NVME_PR_EXCLUSIVE_ACCESS_REG_ONLY)
                        nvmet_pr_resv_released(pr, &reg->hostid);
        }
out:
        kfree_rcu(reg, rcu);
}

static u16 nvmet_pr_unregister(struct nvmet_req *req,
                               struct nvmet_pr_register_data *d,
                               bool ignore_key)
{
        u16 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
        struct nvmet_ctrl *ctrl = req->sq->ctrl;
        struct nvmet_pr *pr = &req->ns->pr;
        struct nvmet_pr_registrant *reg;

        down(&pr->pr_sem);
        list_for_each_entry_rcu(reg, &pr->registrant_list, entry) {
                if (uuid_equal(&reg->hostid, &ctrl->hostid)) {
                        if (ignore_key || reg->rkey == le64_to_cpu(d->crkey)) {
                                status = NVME_SC_SUCCESS;
                                nvmet_pr_unregister_one(pr, reg);
                        }
                        break;
                }
        }
        up(&pr->pr_sem);

        return status;
}

static void nvmet_pr_update_reg_rkey(struct nvmet_pr_registrant *reg,
                                     void *attr)
{
        reg->rkey = *(u64 *)attr;
}

static u16 nvmet_pr_update_reg_attr(struct nvmet_pr *pr,
                        struct nvmet_pr_registrant *reg,
                        void (*change_attr)(struct nvmet_pr_registrant *reg,
                        void *attr),
                        void *attr)
{
        struct nvmet_pr_registrant *holder;
        struct nvmet_pr_registrant *new;

        holder = rcu_dereference_protected(pr->holder, 1);
        if (reg != holder) {
                change_attr(reg, attr);
                return NVME_SC_SUCCESS;
        }

        new = kmalloc_obj(*new, GFP_ATOMIC);
        if (!new)
                return NVME_SC_INTERNAL;

        new->rkey = holder->rkey;
        new->rtype = holder->rtype;
        uuid_copy(&new->hostid, &holder->hostid);
        INIT_LIST_HEAD(&new->entry);

        change_attr(new, attr);
        list_replace_rcu(&holder->entry, &new->entry);
        rcu_assign_pointer(pr->holder, new);
        kfree_rcu(holder, rcu);

        return NVME_SC_SUCCESS;
}

static u16 nvmet_pr_replace(struct nvmet_req *req,
                            struct nvmet_pr_register_data *d,
                            bool ignore_key)
{
        u16 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
        struct nvmet_ctrl *ctrl = req->sq->ctrl;
        struct nvmet_pr *pr = &req->ns->pr;
        struct nvmet_pr_registrant *reg;
        u64 nrkey = le64_to_cpu(d->nrkey);

        down(&pr->pr_sem);
        list_for_each_entry_rcu(reg, &pr->registrant_list, entry) {
                if (uuid_equal(&reg->hostid, &ctrl->hostid)) {
                        if (ignore_key || reg->rkey == le64_to_cpu(d->crkey))
                                status = nvmet_pr_update_reg_attr(pr, reg,
                                                nvmet_pr_update_reg_rkey,
                                                &nrkey);
                        break;
                }
        }
        up(&pr->pr_sem);
        return status;
}

static void nvmet_execute_pr_register(struct nvmet_req *req)
{
        u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10);
        bool ignore_key = nvmet_pr_parse_ignore_key(cdw10);
        struct nvmet_pr_register_data *d;
        u8 reg_act = cdw10 & 0x07; /* Reservation Register Action, bit 02:00 */
        u16 status;

        d = kmalloc_obj(*d);
        if (!d) {
                status = NVME_SC_INTERNAL;
                goto out;
        }

        status = nvmet_copy_from_sgl(req, 0, d, sizeof(*d));
        if (status)
                goto free_data;

        switch (reg_act) {
        case NVME_PR_REGISTER_ACT_REG:
                status = nvmet_pr_register(req, d);
                break;
        case NVME_PR_REGISTER_ACT_UNREG:
                status = nvmet_pr_unregister(req, d, ignore_key);
                break;
        case NVME_PR_REGISTER_ACT_REPLACE:
                status = nvmet_pr_replace(req, d, ignore_key);
                break;
        default:
                req->error_loc = offsetof(struct nvme_common_command, cdw10);
                status = NVME_SC_INVALID_OPCODE | NVME_STATUS_DNR;
                break;
        }
free_data:
        kfree(d);
out:
        if (!status)
                atomic_inc(&req->ns->pr.generation);
        nvmet_req_complete(req, status);
}

static u16 nvmet_pr_acquire(struct nvmet_req *req,
                            struct nvmet_pr_registrant *reg,
                            u8 rtype)
{
        struct nvmet_pr *pr = &req->ns->pr;
        struct nvmet_pr_registrant *holder;

        holder = rcu_dereference_protected(pr->holder, 1);
        if (holder && reg != holder)
                return  NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
        if (holder && reg == holder) {
                if (holder->rtype == rtype)
                        return NVME_SC_SUCCESS;
                return NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
        }

        nvmet_pr_set_new_holder(pr, rtype, reg);
        return NVME_SC_SUCCESS;
}

static void nvmet_pr_confirm_ns_pc_ref(struct percpu_ref *ref)
{
        struct nvmet_pr_per_ctrl_ref *pc_ref =
                container_of(ref, struct nvmet_pr_per_ctrl_ref, ref);

        complete(&pc_ref->confirm_done);
}

static void nvmet_pr_set_ctrl_to_abort(struct nvmet_req *req, uuid_t *hostid)
{
        struct nvmet_pr_per_ctrl_ref *pc_ref;
        struct nvmet_ns *ns = req->ns;
        unsigned long idx;

        xa_for_each(&ns->pr_per_ctrl_refs, idx, pc_ref) {
                if (uuid_equal(&pc_ref->hostid, hostid)) {
                        percpu_ref_kill_and_confirm(&pc_ref->ref,
                                                nvmet_pr_confirm_ns_pc_ref);
                        wait_for_completion(&pc_ref->confirm_done);
                }
        }
}

static u16 nvmet_pr_unreg_all_host_by_prkey(struct nvmet_req *req, u64 prkey,
                                            uuid_t *send_hostid,
                                            bool abort)
{
        u16 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
        struct nvmet_pr_registrant *reg, *tmp;
        struct nvmet_pr *pr = &req->ns->pr;
        uuid_t hostid;

        list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) {
                if (reg->rkey == prkey) {
                        status = NVME_SC_SUCCESS;
                        uuid_copy(&hostid, &reg->hostid);
                        if (abort)
                                nvmet_pr_set_ctrl_to_abort(req, &hostid);
                        nvmet_pr_unregister_one(pr, reg);
                        if (!uuid_equal(&hostid, send_hostid))
                                nvmet_pr_registration_preempted(pr, &hostid);
                }
        }
        return status;
}

static void nvmet_pr_unreg_all_others_by_prkey(struct nvmet_req *req,
                                               u64 prkey,
                                               uuid_t *send_hostid,
                                               bool abort)
{
        struct nvmet_pr_registrant *reg, *tmp;
        struct nvmet_pr *pr = &req->ns->pr;
        uuid_t hostid;

        list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) {
                if (reg->rkey == prkey &&
                    !uuid_equal(&reg->hostid, send_hostid)) {
                        uuid_copy(&hostid, &reg->hostid);
                        if (abort)
                                nvmet_pr_set_ctrl_to_abort(req, &hostid);
                        nvmet_pr_unregister_one(pr, reg);
                        nvmet_pr_registration_preempted(pr, &hostid);
                }
        }
}

static void nvmet_pr_unreg_all_others(struct nvmet_req *req,
                                      uuid_t *send_hostid,
                                      bool abort)
{
        struct nvmet_pr_registrant *reg, *tmp;
        struct nvmet_pr *pr = &req->ns->pr;
        uuid_t hostid;

        list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) {
                if (!uuid_equal(&reg->hostid, send_hostid)) {
                        uuid_copy(&hostid, &reg->hostid);
                        if (abort)
                                nvmet_pr_set_ctrl_to_abort(req, &hostid);
                        nvmet_pr_unregister_one(pr, reg);
                        nvmet_pr_registration_preempted(pr, &hostid);
                }
        }
}

static void nvmet_pr_update_holder_rtype(struct nvmet_pr_registrant *reg,
                                         void *attr)
{
        u8 new_rtype = *(u8 *)attr;

        reg->rtype = new_rtype;
}

static u16 nvmet_pr_preempt(struct nvmet_req *req,
                            struct nvmet_pr_registrant *reg,
                            u8 rtype,
                            struct nvmet_pr_acquire_data *d,
                            bool abort)
{
        struct nvmet_ctrl *ctrl = req->sq->ctrl;
        struct nvmet_pr *pr = &req->ns->pr;
        struct nvmet_pr_registrant *holder;
        enum nvme_pr_type original_rtype;
        u64 prkey = le64_to_cpu(d->prkey);
        u16 status;

        holder = rcu_dereference_protected(pr->holder, 1);
        if (!holder)
                return nvmet_pr_unreg_all_host_by_prkey(req, prkey,
                                        &ctrl->hostid, abort);

        original_rtype = holder->rtype;
        if (original_rtype == NVME_PR_WRITE_EXCLUSIVE_ALL_REGS ||
            original_rtype == NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS) {
                if (!prkey) {
                        /*
                         * To prevent possible access from other hosts, and
                         * avoid terminate the holder, set the new holder
                         * first before unregistering.
                         */
                        nvmet_pr_set_new_holder(pr, rtype, reg);
                        nvmet_pr_unreg_all_others(req, &ctrl->hostid, abort);
                        return NVME_SC_SUCCESS;
                }
                return nvmet_pr_unreg_all_host_by_prkey(req, prkey,
                                &ctrl->hostid, abort);
        }

        if (holder == reg) {
                status = nvmet_pr_update_reg_attr(pr, holder,
                                nvmet_pr_update_holder_rtype, &rtype);
                if (!status && original_rtype != rtype)
                        nvmet_pr_resv_released(pr, &reg->hostid);
                return status;
        }

        if (prkey == holder->rkey) {
                /*
                 * Same as before, set the new holder first.
                 */
                nvmet_pr_set_new_holder(pr, rtype, reg);
                nvmet_pr_unreg_all_others_by_prkey(req, prkey, &ctrl->hostid,
                                                abort);
                if (original_rtype != rtype)
                        nvmet_pr_resv_released(pr, &reg->hostid);
                return NVME_SC_SUCCESS;
        }

        if (prkey)
                return nvmet_pr_unreg_all_host_by_prkey(req, prkey,
                                        &ctrl->hostid, abort);
        return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
}

static void nvmet_pr_do_abort(struct work_struct *w)
{
        struct nvmet_req *req = container_of(w, struct nvmet_req, r.abort_work);
        struct nvmet_pr_per_ctrl_ref *pc_ref;
        struct nvmet_ns *ns = req->ns;
        unsigned long idx;

        /*
         * The target does not support abort, just wait per-controller ref to 0.
         */
        xa_for_each(&ns->pr_per_ctrl_refs, idx, pc_ref) {
                if (percpu_ref_is_dying(&pc_ref->ref)) {
                        wait_for_completion(&pc_ref->free_done);
                        reinit_completion(&pc_ref->confirm_done);
                        reinit_completion(&pc_ref->free_done);
                        percpu_ref_resurrect(&pc_ref->ref);
                }
        }

        up(&ns->pr.pr_sem);
        nvmet_req_complete(req, NVME_SC_SUCCESS);
}

static u16 __nvmet_execute_pr_acquire(struct nvmet_req *req,
                                      struct nvmet_pr_registrant *reg,
                                      u8 acquire_act,
                                      u8 rtype,
                                      struct nvmet_pr_acquire_data *d)
{
        u16 status;

        switch (acquire_act) {
        case NVME_PR_ACQUIRE_ACT_ACQUIRE:
                status = nvmet_pr_acquire(req, reg, rtype);
                goto out;
        case NVME_PR_ACQUIRE_ACT_PREEMPT:
                status = nvmet_pr_preempt(req, reg, rtype, d, false);
                goto inc_gen;
        case NVME_PR_ACQUIRE_ACT_PREEMPT_AND_ABORT:
                status = nvmet_pr_preempt(req, reg, rtype, d, true);
                goto inc_gen;
        default:
                req->error_loc = offsetof(struct nvme_common_command, cdw10);
                status = NVME_SC_INVALID_OPCODE | NVME_STATUS_DNR;
                goto out;
        }
inc_gen:
        if (!status)
                atomic_inc(&req->ns->pr.generation);
out:
        return status;
}

static void nvmet_execute_pr_acquire(struct nvmet_req *req)
{
        u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10);
        bool ignore_key = nvmet_pr_parse_ignore_key(cdw10);
        /* Reservation type, bit 15:08 */
        u8 rtype = (u8)((cdw10 >> 8) & 0xff);
        /* Reservation acquire action, bit 02:00 */
        u8 acquire_act = cdw10 & 0x07;
        struct nvmet_ctrl *ctrl = req->sq->ctrl;
        struct nvmet_pr_acquire_data *d = NULL;
        struct nvmet_pr *pr = &req->ns->pr;
        struct nvmet_pr_registrant *reg;
        u16 status = NVME_SC_SUCCESS;

        if (ignore_key ||
            rtype < NVME_PR_WRITE_EXCLUSIVE ||
            rtype > NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS) {
                status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
                goto out;
        }

        d = kmalloc_obj(*d);
        if (!d) {
                status = NVME_SC_INTERNAL;
                goto out;
        }

        status = nvmet_copy_from_sgl(req, 0, d, sizeof(*d));
        if (status)
                goto free_data;

        status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
        down(&pr->pr_sem);
        list_for_each_entry_rcu(reg, &pr->registrant_list, entry) {
                if (uuid_equal(&reg->hostid, &ctrl->hostid) &&
                    reg->rkey == le64_to_cpu(d->crkey)) {
                        status = __nvmet_execute_pr_acquire(req, reg,
                                        acquire_act, rtype, d);
                        break;
                }
        }

        if (!status && acquire_act == NVME_PR_ACQUIRE_ACT_PREEMPT_AND_ABORT) {
                kfree(d);
                INIT_WORK(&req->r.abort_work, nvmet_pr_do_abort);
                queue_work(nvmet_wq, &req->r.abort_work);
                return;
        }

        up(&pr->pr_sem);

free_data:
        kfree(d);
out:
        nvmet_req_complete(req, status);
}

static u16 nvmet_pr_release(struct nvmet_req *req,
                            struct nvmet_pr_registrant *reg,
                            u8 rtype)
{
        struct nvmet_pr *pr = &req->ns->pr;
        struct nvmet_pr_registrant *holder;
        u8 original_rtype;

        holder = rcu_dereference_protected(pr->holder, 1);
        if (!holder || reg != holder)
                return NVME_SC_SUCCESS;

        original_rtype = holder->rtype;
        if (original_rtype != rtype)
                return NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;

        rcu_assign_pointer(pr->holder, NULL);

        if (original_rtype != NVME_PR_WRITE_EXCLUSIVE &&
            original_rtype != NVME_PR_EXCLUSIVE_ACCESS)
                nvmet_pr_resv_released(pr, &reg->hostid);

        return NVME_SC_SUCCESS;
}

static void nvmet_pr_clear(struct nvmet_req *req)
{
        struct nvmet_pr_registrant *reg, *tmp;
        struct nvmet_pr *pr = &req->ns->pr;

        rcu_assign_pointer(pr->holder, NULL);

        list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) {
                list_del_rcu(&reg->entry);
                if (!uuid_equal(&req->sq->ctrl->hostid, &reg->hostid))
                        nvmet_pr_resv_preempted(pr, &reg->hostid);
                kfree_rcu(reg, rcu);
        }

        atomic_inc(&pr->generation);
}

static u16 __nvmet_execute_pr_release(struct nvmet_req *req,
                                      struct nvmet_pr_registrant *reg,
                                      u8 release_act, u8 rtype)
{
        switch (release_act) {
        case NVME_PR_RELEASE_ACT_RELEASE:
                return nvmet_pr_release(req, reg, rtype);
        case NVME_PR_RELEASE_ACT_CLEAR:
                nvmet_pr_clear(req);
                return NVME_SC_SUCCESS;
        default:
                req->error_loc = offsetof(struct nvme_common_command, cdw10);
                return NVME_SC_INVALID_OPCODE | NVME_STATUS_DNR;
        }
}

static void nvmet_execute_pr_release(struct nvmet_req *req)
{
        u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10);
        bool ignore_key = nvmet_pr_parse_ignore_key(cdw10);
        u8 rtype = (u8)((cdw10 >> 8) & 0xff); /* Reservation type, bit 15:08 */
        u8 release_act = cdw10 & 0x07; /* Reservation release action, bit 02:00 */
        struct nvmet_ctrl *ctrl = req->sq->ctrl;
        struct nvmet_pr *pr = &req->ns->pr;
        struct nvmet_pr_release_data *d;
        struct nvmet_pr_registrant *reg;
        u16 status;

        if (ignore_key) {
                status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
                goto out;
        }

        d = kmalloc_obj(*d);
        if (!d) {
                status = NVME_SC_INTERNAL;
                goto out;
        }

        status = nvmet_copy_from_sgl(req, 0, d, sizeof(*d));
        if (status)
                goto free_data;

        status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
        down(&pr->pr_sem);
        list_for_each_entry_rcu(reg, &pr->registrant_list, entry) {
                if (uuid_equal(&reg->hostid, &ctrl->hostid) &&
                    reg->rkey == le64_to_cpu(d->crkey)) {
                        status = __nvmet_execute_pr_release(req, reg,
                                        release_act, rtype);
                        break;
                }
        }
        up(&pr->pr_sem);
free_data:
        kfree(d);
out:
        nvmet_req_complete(req, status);
}

static void nvmet_execute_pr_report(struct nvmet_req *req)
{
        u32 cdw11 = le32_to_cpu(req->cmd->common.cdw11);
        u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10);
        u32 num_bytes = 4 * (cdw10 + 1); /* cdw10 is number of dwords */
        u8 eds = cdw11 & 1; /* Extended data structure, bit 00 */
        struct nvme_registered_ctrl_ext *ctrl_eds;
        struct nvme_reservation_status_ext *data;
        struct nvmet_pr *pr = &req->ns->pr;
        struct nvmet_pr_registrant *holder;
        struct nvmet_pr_registrant *reg;
        u16 num_ctrls = 0;
        u16 status;
        u8 rtype;

        /* nvmet hostid(uuid_t) is 128 bit. */
        if (!eds) {
                req->error_loc = offsetof(struct nvme_common_command, cdw11);
                status = NVME_SC_HOST_ID_INCONSIST | NVME_STATUS_DNR;
                goto out;
        }

        if (num_bytes < sizeof(struct nvme_reservation_status_ext)) {
                req->error_loc = offsetof(struct nvme_common_command, cdw10);
                status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
                goto out;
        }

        data = kzalloc(num_bytes, GFP_KERNEL);
        if (!data) {
                status = NVME_SC_INTERNAL;
                goto out;
        }
        data->gen = cpu_to_le32(atomic_read(&pr->generation));
        data->ptpls = 0;
        ctrl_eds = data->regctl_eds;

        rcu_read_lock();
        holder = rcu_dereference(pr->holder);
        rtype = holder ? holder->rtype : 0;
        data->rtype = rtype;

        list_for_each_entry_rcu(reg, &pr->registrant_list, entry) {
                num_ctrls++;
                /*
                 * continue to get the number of all registrans.
                 */
                if (((void *)ctrl_eds + sizeof(*ctrl_eds)) >
                    ((void *)data + num_bytes))
                        continue;
                /*
                 * Dynamic controller, set cntlid to 0xffff.
                 */
                ctrl_eds->cntlid = cpu_to_le16(NVME_CNTLID_DYNAMIC);
                if (rtype == NVME_PR_WRITE_EXCLUSIVE_ALL_REGS ||
                    rtype == NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS)
                        ctrl_eds->rcsts = 1;
                if (reg == holder)
                        ctrl_eds->rcsts = 1;
                uuid_copy((uuid_t *)&ctrl_eds->hostid, &reg->hostid);
                ctrl_eds->rkey = cpu_to_le64(reg->rkey);
                ctrl_eds++;
        }
        rcu_read_unlock();

        put_unaligned_le16(num_ctrls, data->regctl);
        status = nvmet_copy_to_sgl(req, 0, data, num_bytes);
        kfree(data);
out:
        nvmet_req_complete(req, status);
}

u16 nvmet_parse_pr_cmd(struct nvmet_req *req)
{
        struct nvme_command *cmd = req->cmd;

        switch (cmd->common.opcode) {
        case nvme_cmd_resv_register:
                req->execute = nvmet_execute_pr_register;
                break;
        case nvme_cmd_resv_acquire:
                req->execute = nvmet_execute_pr_acquire;
                break;
        case nvme_cmd_resv_release:
                req->execute = nvmet_execute_pr_release;
                break;
        case nvme_cmd_resv_report:
                req->execute = nvmet_execute_pr_report;
                break;
        default:
                return 1;
        }
        return NVME_SC_SUCCESS;
}

static bool nvmet_is_req_write_cmd_group(struct nvmet_req *req)
{
        u8 opcode = req->cmd->common.opcode;

        if (req->sq->qid) {
                switch (opcode) {
                case nvme_cmd_flush:
                case nvme_cmd_write:
                case nvme_cmd_write_zeroes:
                case nvme_cmd_dsm:
                case nvme_cmd_zone_append:
                case nvme_cmd_zone_mgmt_send:
                        return true;
                default:
                        return false;
                }
        }
        return false;
}

static bool nvmet_is_req_read_cmd_group(struct nvmet_req *req)
{
        u8 opcode = req->cmd->common.opcode;

        if (req->sq->qid) {
                switch (opcode) {
                case nvme_cmd_read:
                case nvme_cmd_zone_mgmt_recv:
                        return true;
                default:
                        return false;
                }
        }
        return false;
}

u16 nvmet_pr_check_cmd_access(struct nvmet_req *req)
{
        struct nvmet_ctrl *ctrl = req->sq->ctrl;
        struct nvmet_pr_registrant *holder;
        struct nvmet_ns *ns = req->ns;
        struct nvmet_pr *pr = &ns->pr;
        u16 status = NVME_SC_SUCCESS;

        rcu_read_lock();
        holder = rcu_dereference(pr->holder);
        if (!holder)
                goto unlock;
        if (uuid_equal(&ctrl->hostid, &holder->hostid))
                goto unlock;

        /*
         * The Reservation command group is checked in executing,
         * allow it here.
         */
        switch (holder->rtype) {
        case NVME_PR_WRITE_EXCLUSIVE:
                if (nvmet_is_req_write_cmd_group(req))
                        status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
                break;
        case NVME_PR_EXCLUSIVE_ACCESS:
                if (nvmet_is_req_read_cmd_group(req) ||
                    nvmet_is_req_write_cmd_group(req))
                        status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
                break;
        case NVME_PR_WRITE_EXCLUSIVE_REG_ONLY:
        case NVME_PR_WRITE_EXCLUSIVE_ALL_REGS:
                if ((nvmet_is_req_write_cmd_group(req)) &&
                    !nvmet_pr_find_registrant(pr, &ctrl->hostid))
                        status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
                break;
        case NVME_PR_EXCLUSIVE_ACCESS_REG_ONLY:
        case NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS:
                if ((nvmet_is_req_read_cmd_group(req) ||
                    nvmet_is_req_write_cmd_group(req)) &&
                    !nvmet_pr_find_registrant(pr, &ctrl->hostid))
                        status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
                break;
        default:
                pr_warn("the reservation type is set wrong, type:%d\n",
                        holder->rtype);
                break;
        }

unlock:
        rcu_read_unlock();
        if (status)
                req->error_loc = offsetof(struct nvme_common_command, opcode);
        return status;
}

u16 nvmet_pr_get_ns_pc_ref(struct nvmet_req *req)
{
        struct nvmet_pr_per_ctrl_ref *pc_ref;

        pc_ref = xa_load(&req->ns->pr_per_ctrl_refs,
                        req->sq->ctrl->cntlid);
        if (unlikely(!percpu_ref_tryget_live(&pc_ref->ref)))
                return NVME_SC_INTERNAL;
        req->pc_ref = pc_ref;
        return NVME_SC_SUCCESS;
}

static void nvmet_pr_ctrl_ns_all_cmds_done(struct percpu_ref *ref)
{
        struct nvmet_pr_per_ctrl_ref *pc_ref =
                container_of(ref, struct nvmet_pr_per_ctrl_ref, ref);

        complete(&pc_ref->free_done);
}

static int nvmet_pr_alloc_and_insert_pc_ref(struct nvmet_ns *ns,
                                            unsigned long idx,
                                            uuid_t *hostid)
{
        struct nvmet_pr_per_ctrl_ref *pc_ref;
        int ret;

        pc_ref = kmalloc_obj(*pc_ref, GFP_ATOMIC);
        if (!pc_ref)
                return  -ENOMEM;

        ret = percpu_ref_init(&pc_ref->ref, nvmet_pr_ctrl_ns_all_cmds_done,
                        PERCPU_REF_ALLOW_REINIT, GFP_KERNEL);
        if (ret)
                goto free;

        init_completion(&pc_ref->free_done);
        init_completion(&pc_ref->confirm_done);
        uuid_copy(&pc_ref->hostid, hostid);

        ret = xa_insert(&ns->pr_per_ctrl_refs, idx, pc_ref, GFP_KERNEL);
        if (ret)
                goto exit;
        return ret;
exit:
        percpu_ref_exit(&pc_ref->ref);
free:
        kfree(pc_ref);
        return ret;
}

int nvmet_ctrl_init_pr(struct nvmet_ctrl *ctrl)
{
        struct nvmet_subsys *subsys = ctrl->subsys;
        struct nvmet_pr_per_ctrl_ref *pc_ref;
        struct nvmet_ns *ns = NULL;
        unsigned long idx;
        int ret;

        ctrl->pr_log_mgr.counter = 0;
        ctrl->pr_log_mgr.lost_count = 0;
        mutex_init(&ctrl->pr_log_mgr.lock);
        INIT_KFIFO(ctrl->pr_log_mgr.log_queue);

        /*
         * Here we are under subsys lock, if an ns not in subsys->namespaces,
         * we can make sure that ns is not enabled, and not call
         * nvmet_pr_init_ns(), see more details in nvmet_ns_enable().
         * So just check ns->pr.enable.
         */
        nvmet_for_each_enabled_ns(&subsys->namespaces, idx, ns) {
                if (ns->pr.enable) {
                        ret = nvmet_pr_alloc_and_insert_pc_ref(ns, ctrl->cntlid,
                                                        &ctrl->hostid);
                        if (ret)
                                goto free_per_ctrl_refs;
                }
        }
        return 0;

free_per_ctrl_refs:
        nvmet_for_each_enabled_ns(&subsys->namespaces, idx, ns) {
                if (ns->pr.enable) {
                        pc_ref = xa_erase(&ns->pr_per_ctrl_refs, ctrl->cntlid);
                        if (pc_ref)
                                percpu_ref_exit(&pc_ref->ref);
                        kfree(pc_ref);
                }
        }
        return ret;
}

void nvmet_ctrl_destroy_pr(struct nvmet_ctrl *ctrl)
{
        struct nvmet_pr_per_ctrl_ref *pc_ref;
        struct nvmet_ns *ns;
        unsigned long idx;

        kfifo_free(&ctrl->pr_log_mgr.log_queue);
        mutex_destroy(&ctrl->pr_log_mgr.lock);

        nvmet_for_each_enabled_ns(&ctrl->subsys->namespaces, idx, ns) {
                if (ns->pr.enable) {
                        pc_ref = xa_erase(&ns->pr_per_ctrl_refs, ctrl->cntlid);
                        if (pc_ref)
                                percpu_ref_exit(&pc_ref->ref);
                        kfree(pc_ref);
                }
        }
}

int nvmet_pr_init_ns(struct nvmet_ns *ns)
{
        struct nvmet_subsys *subsys = ns->subsys;
        struct nvmet_pr_per_ctrl_ref *pc_ref;
        struct nvmet_ctrl *ctrl = NULL;
        unsigned long idx;
        int ret;

        ns->pr.holder = NULL;
        atomic_set(&ns->pr.generation, 0);
        sema_init(&ns->pr.pr_sem, 1);
        INIT_LIST_HEAD(&ns->pr.registrant_list);
        ns->pr.notify_mask = 0;

        xa_init(&ns->pr_per_ctrl_refs);

        list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
                ret = nvmet_pr_alloc_and_insert_pc_ref(ns, ctrl->cntlid,
                                                &ctrl->hostid);
                if (ret)
                        goto free_per_ctrl_refs;
        }
        return 0;

free_per_ctrl_refs:
        xa_for_each(&ns->pr_per_ctrl_refs, idx, pc_ref) {
                xa_erase(&ns->pr_per_ctrl_refs, idx);
                percpu_ref_exit(&pc_ref->ref);
                kfree(pc_ref);
        }
        return ret;
}

void nvmet_pr_exit_ns(struct nvmet_ns *ns)
{
        struct nvmet_pr_registrant *reg, *tmp;
        struct nvmet_pr_per_ctrl_ref *pc_ref;
        struct nvmet_pr *pr = &ns->pr;
        unsigned long idx;

        list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) {
                list_del(&reg->entry);
                kfree(reg);
        }

        xa_for_each(&ns->pr_per_ctrl_refs, idx, pc_ref) {
                /*
                 * No command on ns here, we can safely free pc_ref.
                 */
                pc_ref = xa_erase(&ns->pr_per_ctrl_refs, idx);
                percpu_ref_exit(&pc_ref->ref);
                kfree(pc_ref);
        }

        xa_destroy(&ns->pr_per_ctrl_refs);
}