// SPDX-License-Identifier: GPL-2.0
/*
 * NVMe over Fabrics DH-HMAC-CHAP authentication command handling.
 * Copyright (c) 2020 Hannes Reinecke, SUSE Software Solutions.
 * All rights reserved.
 */
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/blkdev.h>
#include <linux/random.h>
#include <linux/nvme-auth.h>
#include <crypto/hash.h>
#include <crypto/kpp.h>
#include "nvmet.h"

static void nvmet_auth_expired_work(struct work_struct *work)
{
        struct nvmet_sq *sq = container_of(to_delayed_work(work),
                        struct nvmet_sq, auth_expired_work);

        pr_debug("%s: ctrl %d qid %d transaction %u expired, resetting\n",
                 __func__, sq->ctrl->cntlid, sq->qid, sq->dhchap_tid);
        sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE;
        sq->dhchap_tid = -1;
}

void nvmet_auth_sq_init(struct nvmet_sq *sq)
{
        /* Initialize in-band authentication */
        INIT_DELAYED_WORK(&sq->auth_expired_work, nvmet_auth_expired_work);
        sq->authenticated = false;
        sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE;
}

static u8 nvmet_auth_negotiate(struct nvmet_req *req, void *d)
{
        struct nvmet_ctrl *ctrl = req->sq->ctrl;
        struct nvmf_auth_dhchap_negotiate_data *data = d;
        int i, hash_id = 0, fallback_hash_id = 0, dhgid, fallback_dhgid;

        pr_debug("%s: ctrl %d qid %d: data sc_d %d napd %d authid %d halen %d dhlen %d\n",
                 __func__, ctrl->cntlid, req->sq->qid,
                 data->sc_c, data->napd, data->auth_protocol[0].dhchap.authid,
                 data->auth_protocol[0].dhchap.halen,
                 data->auth_protocol[0].dhchap.dhlen);
        req->sq->dhchap_tid = le16_to_cpu(data->t_id);
        req->sq->sc_c = data->sc_c;
        if (data->sc_c != NVME_AUTH_SECP_NOSC) {
                if (!IS_ENABLED(CONFIG_NVME_TARGET_TCP_TLS))
                        return NVME_AUTH_DHCHAP_FAILURE_CONCAT_MISMATCH;
                /* Secure concatenation can only be enabled on the admin queue */
                if (req->sq->qid)
                        return NVME_AUTH_DHCHAP_FAILURE_CONCAT_MISMATCH;
                switch (data->sc_c) {
                case NVME_AUTH_SECP_NEWTLSPSK:
                        if (nvmet_queue_tls_keyid(req->sq))
                                return NVME_AUTH_DHCHAP_FAILURE_CONCAT_MISMATCH;
                        break;
                case NVME_AUTH_SECP_REPLACETLSPSK:
                        if (!nvmet_queue_tls_keyid(req->sq))
                                return NVME_AUTH_DHCHAP_FAILURE_CONCAT_MISMATCH;
                        break;
                default:
                        return NVME_AUTH_DHCHAP_FAILURE_CONCAT_MISMATCH;
                }
                ctrl->concat = true;
        }

        if (data->napd != 1)
                return NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;

        if (data->auth_protocol[0].dhchap.authid !=
            NVME_AUTH_DHCHAP_AUTH_ID)
                return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;

        for (i = 0; i < data->auth_protocol[0].dhchap.halen; i++) {
                u8 host_hmac_id = data->auth_protocol[0].dhchap.idlist[i];

                if (!fallback_hash_id &&
                    crypto_has_shash(nvme_auth_hmac_name(host_hmac_id), 0, 0))
                        fallback_hash_id = host_hmac_id;
                if (ctrl->shash_id != host_hmac_id)
                        continue;
                hash_id = ctrl->shash_id;
                break;
        }
        if (hash_id == 0) {
                if (fallback_hash_id == 0) {
                        pr_debug("%s: ctrl %d qid %d: no usable hash found\n",
                                 __func__, ctrl->cntlid, req->sq->qid);
                        return NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
                }
                pr_debug("%s: ctrl %d qid %d: no usable hash found, falling back to %s\n",
                         __func__, ctrl->cntlid, req->sq->qid,
                         nvme_auth_hmac_name(fallback_hash_id));
                ctrl->shash_id = fallback_hash_id;
        }

        dhgid = -1;
        fallback_dhgid = -1;
        for (i = 0; i < data->auth_protocol[0].dhchap.dhlen; i++) {
                int tmp_dhgid = data->auth_protocol[0].dhchap.idlist[i + 30];

                if (tmp_dhgid != ctrl->dh_gid) {
                        dhgid = tmp_dhgid;
                        break;
                }
                if (fallback_dhgid < 0) {
                        const char *kpp = nvme_auth_dhgroup_kpp(tmp_dhgid);

                        if (crypto_has_kpp(kpp, 0, 0))
                                fallback_dhgid = tmp_dhgid;
                }
        }
        if (dhgid < 0) {
                if (fallback_dhgid < 0) {
                        pr_debug("%s: ctrl %d qid %d: no usable DH group found\n",
                                 __func__, ctrl->cntlid, req->sq->qid);
                        return NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
                }
                pr_debug("%s: ctrl %d qid %d: configured DH group %s not found\n",
                         __func__, ctrl->cntlid, req->sq->qid,
                         nvme_auth_dhgroup_name(fallback_dhgid));
                ctrl->dh_gid = fallback_dhgid;
        }
        if (ctrl->dh_gid == NVME_AUTH_DHGROUP_NULL && ctrl->concat) {
                pr_debug("%s: ctrl %d qid %d: NULL DH group invalid "
                         "for secure channel concatenation\n", __func__,
                         ctrl->cntlid, req->sq->qid);
                return NVME_AUTH_DHCHAP_FAILURE_CONCAT_MISMATCH;
        }
        pr_debug("%s: ctrl %d qid %d: selected DH group %s (%d)\n",
                 __func__, ctrl->cntlid, req->sq->qid,
                 nvme_auth_dhgroup_name(ctrl->dh_gid), ctrl->dh_gid);
        return 0;
}

static u8 nvmet_auth_reply(struct nvmet_req *req, void *d)
{
        struct nvmet_ctrl *ctrl = req->sq->ctrl;
        struct nvmf_auth_dhchap_reply_data *data = d;
        u16 dhvlen = le16_to_cpu(data->dhvlen);
        u8 *response;

        pr_debug("%s: ctrl %d qid %d: data hl %d cvalid %d dhvlen %u\n",
                 __func__, ctrl->cntlid, req->sq->qid,
                 data->hl, data->cvalid, dhvlen);

        if (dhvlen) {
                if (!ctrl->dh_tfm)
                        return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
                if (nvmet_auth_ctrl_sesskey(req, data->rval + 2 * data->hl,
                                            dhvlen) < 0)
                        return NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
        }

        response = kmalloc(data->hl, GFP_KERNEL);
        if (!response)
                return NVME_AUTH_DHCHAP_FAILURE_FAILED;

        if (!ctrl->host_key) {
                pr_warn("ctrl %d qid %d no host key\n",
                        ctrl->cntlid, req->sq->qid);
                kfree(response);
                return NVME_AUTH_DHCHAP_FAILURE_FAILED;
        }
        if (nvmet_auth_host_hash(req, response, data->hl) < 0) {
                pr_debug("ctrl %d qid %d host hash failed\n",
                         ctrl->cntlid, req->sq->qid);
                kfree(response);
                return NVME_AUTH_DHCHAP_FAILURE_FAILED;
        }

        if (memcmp(data->rval, response, data->hl)) {
                pr_info("ctrl %d qid %d host response mismatch\n",
                        ctrl->cntlid, req->sq->qid);
                pr_debug("ctrl %d qid %d rval %*ph\n",
                         ctrl->cntlid, req->sq->qid, data->hl, data->rval);
                pr_debug("ctrl %d qid %d response %*ph\n",
                         ctrl->cntlid, req->sq->qid, data->hl, response);
                kfree(response);
                return NVME_AUTH_DHCHAP_FAILURE_FAILED;
        }
        kfree(response);
        pr_debug("%s: ctrl %d qid %d host authenticated\n",
                 __func__, ctrl->cntlid, req->sq->qid);
        if (!data->cvalid && ctrl->concat) {
                pr_debug("%s: ctrl %d qid %d invalid challenge\n",
                         __func__, ctrl->cntlid, req->sq->qid);
                return NVME_AUTH_DHCHAP_FAILURE_FAILED;
        }
        req->sq->dhchap_s2 = le32_to_cpu(data->seqnum);
        if (data->cvalid) {
                req->sq->dhchap_c2 = kmemdup(data->rval + data->hl, data->hl,
                                             GFP_KERNEL);
                if (!req->sq->dhchap_c2)
                        return NVME_AUTH_DHCHAP_FAILURE_FAILED;

                pr_debug("%s: ctrl %d qid %d challenge %*ph\n",
                         __func__, ctrl->cntlid, req->sq->qid, data->hl,
                         req->sq->dhchap_c2);
        }
        /*
         * NVMe Base Spec 2.2 section 8.3.4.5.4: DH-HMAC-CHAP_Reply message
         * Sequence Number (SEQNUM): [ .. ]
         * The value 0h is used to indicate that bidirectional authentication
         * is not performed, but a challenge value C2 is carried in order to
         * generate a pre-shared key (PSK) for subsequent establishment of a
         * secure channel.
         */
        if (req->sq->dhchap_s2 == 0) {
                if (ctrl->concat)
                        nvmet_auth_insert_psk(req->sq);
                req->sq->authenticated = true;
                kfree(req->sq->dhchap_c2);
                req->sq->dhchap_c2 = NULL;
        } else if (!data->cvalid)
                req->sq->authenticated = true;

        return 0;
}

static u8 nvmet_auth_failure2(void *d)
{
        struct nvmf_auth_dhchap_failure_data *data = d;

        return data->rescode_exp;
}

u32 nvmet_auth_send_data_len(struct nvmet_req *req)
{
        return le32_to_cpu(req->cmd->auth_send.tl);
}

void nvmet_execute_auth_send(struct nvmet_req *req)
{
        struct nvmet_ctrl *ctrl = req->sq->ctrl;
        struct nvmf_auth_dhchap_success2_data *data;
        void *d;
        u32 tl;
        u16 status = 0;
        u8 dhchap_status;

        if (req->cmd->auth_send.secp != NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER) {
                status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
                req->error_loc =
                        offsetof(struct nvmf_auth_send_command, secp);
                goto done;
        }
        if (req->cmd->auth_send.spsp0 != 0x01) {
                status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
                req->error_loc =
                        offsetof(struct nvmf_auth_send_command, spsp0);
                goto done;
        }
        if (req->cmd->auth_send.spsp1 != 0x01) {
                status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
                req->error_loc =
                        offsetof(struct nvmf_auth_send_command, spsp1);
                goto done;
        }
        tl = nvmet_auth_send_data_len(req);
        if (!tl) {
                status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
                req->error_loc =
                        offsetof(struct nvmf_auth_send_command, tl);
                goto done;
        }
        if (!nvmet_check_transfer_len(req, tl)) {
                pr_debug("%s: transfer length mismatch (%u)\n", __func__, tl);
                return;
        }

        d = kmalloc(tl, GFP_KERNEL);
        if (!d) {
                status = NVME_SC_INTERNAL;
                goto done;
        }

        status = nvmet_copy_from_sgl(req, 0, d, tl);
        if (status)
                goto done_kfree;

        data = d;
        pr_debug("%s: ctrl %d qid %d type %d id %d step %x\n", __func__,
                 ctrl->cntlid, req->sq->qid, data->auth_type, data->auth_id,
                 req->sq->dhchap_step);
        if (data->auth_type != NVME_AUTH_COMMON_MESSAGES &&
            data->auth_type != NVME_AUTH_DHCHAP_MESSAGES)
                goto done_failure1;
        if (data->auth_type == NVME_AUTH_COMMON_MESSAGES) {
                if (data->auth_id == NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE) {
                        /* Restart negotiation */
                        pr_debug("%s: ctrl %d qid %d reset negotiation\n",
                                 __func__, ctrl->cntlid, req->sq->qid);
                        if (!req->sq->qid) {
                                dhchap_status = nvmet_setup_auth(ctrl, req->sq);
                                if (dhchap_status) {
                                        pr_err("ctrl %d qid 0 failed to setup re-authentication\n",
                                               ctrl->cntlid);
                                        req->sq->dhchap_status = dhchap_status;
                                        req->sq->dhchap_step =
                                                NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
                                        goto done_kfree;
                                }
                        }
                        req->sq->dhchap_step =
                                NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE;
                } else if (data->auth_id != req->sq->dhchap_step)
                        goto done_failure1;
                /* Validate negotiation parameters */
                dhchap_status = nvmet_auth_negotiate(req, d);
                if (dhchap_status == 0)
                        req->sq->dhchap_step =
                                NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE;
                else {
                        req->sq->dhchap_step =
                                NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
                        req->sq->dhchap_status = dhchap_status;
                }
                goto done_kfree;
        }
        if (data->auth_id != req->sq->dhchap_step) {
                pr_debug("%s: ctrl %d qid %d step mismatch (%d != %d)\n",
                         __func__, ctrl->cntlid, req->sq->qid,
                         data->auth_id, req->sq->dhchap_step);
                goto done_failure1;
        }
        if (le16_to_cpu(data->t_id) != req->sq->dhchap_tid) {
                pr_debug("%s: ctrl %d qid %d invalid transaction %d (expected %d)\n",
                         __func__, ctrl->cntlid, req->sq->qid,
                         le16_to_cpu(data->t_id),
                         req->sq->dhchap_tid);
                req->sq->dhchap_step =
                        NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
                req->sq->dhchap_status =
                        NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
                goto done_kfree;
        }

        switch (data->auth_id) {
        case NVME_AUTH_DHCHAP_MESSAGE_REPLY:
                dhchap_status = nvmet_auth_reply(req, d);
                if (dhchap_status == 0)
                        req->sq->dhchap_step =
                                NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1;
                else {
                        req->sq->dhchap_step =
                                NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
                        req->sq->dhchap_status = dhchap_status;
                }
                goto done_kfree;
        case NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2:
                if (ctrl->concat)
                        nvmet_auth_insert_psk(req->sq);
                req->sq->authenticated = true;
                pr_debug("%s: ctrl %d qid %d ctrl authenticated\n",
                         __func__, ctrl->cntlid, req->sq->qid);
                goto done_kfree;
        case NVME_AUTH_DHCHAP_MESSAGE_FAILURE2:
                dhchap_status = nvmet_auth_failure2(d);
                if (dhchap_status) {
                        pr_warn("ctrl %d qid %d: authentication failed (%d)\n",
                                ctrl->cntlid, req->sq->qid, dhchap_status);
                        req->sq->dhchap_status = dhchap_status;
                        req->sq->authenticated = false;
                }
                goto done_kfree;
        default:
                req->sq->dhchap_status =
                        NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE;
                req->sq->dhchap_step =
                        NVME_AUTH_DHCHAP_MESSAGE_FAILURE2;
                req->sq->authenticated = false;
                goto done_kfree;
        }
done_failure1:
        req->sq->dhchap_status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE;
        req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_FAILURE2;

done_kfree:
        kfree(d);
done:
        pr_debug("%s: ctrl %d qid %d dhchap status %x step %x\n", __func__,
                 ctrl->cntlid, req->sq->qid,
                 req->sq->dhchap_status, req->sq->dhchap_step);
        if (status)
                pr_debug("%s: ctrl %d qid %d nvme status %x error loc %d\n",
                         __func__, ctrl->cntlid, req->sq->qid,
                         status, req->error_loc);
        if (req->sq->dhchap_step != NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2 &&
            req->sq->dhchap_step != NVME_AUTH_DHCHAP_MESSAGE_FAILURE2) {
                unsigned long auth_expire_secs = ctrl->kato ? ctrl->kato : 120;

                mod_delayed_work(system_wq, &req->sq->auth_expired_work,
                                 auth_expire_secs * HZ);
                goto complete;
        }
        /* Final states, clear up variables */
        nvmet_auth_sq_free(req->sq);
        if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_FAILURE2)
                nvmet_ctrl_fatal_error(ctrl);

complete:
        nvmet_req_complete(req, status);
}

static int nvmet_auth_challenge(struct nvmet_req *req, void *d, int al)
{
        struct nvmf_auth_dhchap_challenge_data *data = d;
        struct nvmet_ctrl *ctrl = req->sq->ctrl;
        int ret = 0;
        int hash_len = nvme_auth_hmac_hash_len(ctrl->shash_id);
        int data_size = sizeof(*d) + hash_len;

        if (ctrl->dh_tfm)
                data_size += ctrl->dh_keysize;
        if (al < data_size) {
                pr_debug("%s: buffer too small (al %d need %d)\n", __func__,
                         al, data_size);
                return -EINVAL;
        }
        memset(data, 0, data_size);
        req->sq->dhchap_s1 = nvme_auth_get_seqnum();
        data->auth_type = NVME_AUTH_DHCHAP_MESSAGES;
        data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE;
        data->t_id = cpu_to_le16(req->sq->dhchap_tid);
        data->hashid = ctrl->shash_id;
        data->hl = hash_len;
        data->seqnum = cpu_to_le32(req->sq->dhchap_s1);
        req->sq->dhchap_c1 = kmalloc(data->hl, GFP_KERNEL);
        if (!req->sq->dhchap_c1)
                return -ENOMEM;
        get_random_bytes(req->sq->dhchap_c1, data->hl);
        memcpy(data->cval, req->sq->dhchap_c1, data->hl);
        if (ctrl->dh_tfm) {
                data->dhgid = ctrl->dh_gid;
                data->dhvlen = cpu_to_le16(ctrl->dh_keysize);
                ret = nvmet_auth_ctrl_exponential(req, data->cval + data->hl,
                                                  ctrl->dh_keysize);
        }
        pr_debug("%s: ctrl %d qid %d seq %d transaction %d hl %d dhvlen %zu\n",
                 __func__, ctrl->cntlid, req->sq->qid, req->sq->dhchap_s1,
                 req->sq->dhchap_tid, data->hl, ctrl->dh_keysize);
        return ret;
}

static int nvmet_auth_success1(struct nvmet_req *req, void *d, int al)
{
        struct nvmf_auth_dhchap_success1_data *data = d;
        struct nvmet_ctrl *ctrl = req->sq->ctrl;
        int hash_len = nvme_auth_hmac_hash_len(ctrl->shash_id);

        WARN_ON(al < sizeof(*data));
        memset(data, 0, sizeof(*data));
        data->auth_type = NVME_AUTH_DHCHAP_MESSAGES;
        data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1;
        data->t_id = cpu_to_le16(req->sq->dhchap_tid);
        data->hl = hash_len;
        if (req->sq->dhchap_c2) {
                if (!ctrl->ctrl_key) {
                        pr_warn("ctrl %d qid %d no ctrl key\n",
                                ctrl->cntlid, req->sq->qid);
                        return NVME_AUTH_DHCHAP_FAILURE_FAILED;
                }
                if (nvmet_auth_ctrl_hash(req, data->rval, data->hl))
                        return NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
                data->rvalid = 1;
                pr_debug("ctrl %d qid %d response %*ph\n",
                         ctrl->cntlid, req->sq->qid, data->hl, data->rval);
        }
        return 0;
}

static void nvmet_auth_failure1(struct nvmet_req *req, void *d, int al)
{
        struct nvmf_auth_dhchap_failure_data *data = d;

        WARN_ON(al < sizeof(*data));
        data->auth_type = NVME_AUTH_COMMON_MESSAGES;
        data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
        data->t_id = cpu_to_le16(req->sq->dhchap_tid);
        data->rescode = NVME_AUTH_DHCHAP_FAILURE_REASON_FAILED;
        data->rescode_exp = req->sq->dhchap_status;
}

u32 nvmet_auth_receive_data_len(struct nvmet_req *req)
{
        return le32_to_cpu(req->cmd->auth_receive.al);
}

void nvmet_execute_auth_receive(struct nvmet_req *req)
{
        struct nvmet_ctrl *ctrl = req->sq->ctrl;
        void *d;
        u32 al;
        u16 status = 0;

        if (req->cmd->auth_receive.secp != NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER) {
                status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
                req->error_loc =
                        offsetof(struct nvmf_auth_receive_command, secp);
                goto done;
        }
        if (req->cmd->auth_receive.spsp0 != 0x01) {
                status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
                req->error_loc =
                        offsetof(struct nvmf_auth_receive_command, spsp0);
                goto done;
        }
        if (req->cmd->auth_receive.spsp1 != 0x01) {
                status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
                req->error_loc =
                        offsetof(struct nvmf_auth_receive_command, spsp1);
                goto done;
        }
        al = nvmet_auth_receive_data_len(req);
        if (!al) {
                status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
                req->error_loc =
                        offsetof(struct nvmf_auth_receive_command, al);
                goto done;
        }
        if (!nvmet_check_transfer_len(req, al)) {
                pr_debug("%s: transfer length mismatch (%u)\n", __func__, al);
                return;
        }

        d = kmalloc(al, GFP_KERNEL);
        if (!d) {
                status = NVME_SC_INTERNAL;
                goto done;
        }
        pr_debug("%s: ctrl %d qid %d step %x\n", __func__,
                 ctrl->cntlid, req->sq->qid, req->sq->dhchap_step);
        switch (req->sq->dhchap_step) {
        case NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE:
                if (nvmet_auth_challenge(req, d, al) < 0) {
                        pr_warn("ctrl %d qid %d: challenge error (%d)\n",
                                ctrl->cntlid, req->sq->qid, status);
                        status = NVME_SC_INTERNAL;
                        break;
                }
                req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_REPLY;
                break;
        case NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1:
                status = nvmet_auth_success1(req, d, al);
                if (status) {
                        req->sq->dhchap_status = status;
                        req->sq->authenticated = false;
                        nvmet_auth_failure1(req, d, al);
                        pr_warn("ctrl %d qid %d: success1 status (%x)\n",
                                ctrl->cntlid, req->sq->qid,
                                req->sq->dhchap_status);
                        break;
                }
                req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2;
                break;
        case NVME_AUTH_DHCHAP_MESSAGE_FAILURE1:
                req->sq->authenticated = false;
                nvmet_auth_failure1(req, d, al);
                pr_warn("ctrl %d qid %d failure1 (%x)\n",
                        ctrl->cntlid, req->sq->qid, req->sq->dhchap_status);
                break;
        default:
                pr_warn("ctrl %d qid %d unhandled step (%d)\n",
                        ctrl->cntlid, req->sq->qid, req->sq->dhchap_step);
                req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
                req->sq->dhchap_status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
                nvmet_auth_failure1(req, d, al);
                status = 0;
                break;
        }

        status = nvmet_copy_to_sgl(req, 0, d, al);
        kfree(d);
done:
        if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2)
                nvmet_auth_sq_free(req->sq);
        else if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_FAILURE1) {
                nvmet_auth_sq_free(req->sq);
                nvmet_ctrl_fatal_error(ctrl);
        }
        nvmet_req_complete(req, status);
}