// SPDX-License-Identifier: GPL-2.0+
/*
 * Copyright IBM Corp. 2025
 *
 * s390 specific HMAC support for protected keys.
 */

#define pr_fmt(fmt) "phmac_s390: " fmt

#include <asm/cpacf.h>
#include <asm/pkey.h>
#include <crypto/engine.h>
#include <crypto/hash.h>
#include <crypto/internal/hash.h>
#include <crypto/sha2.h>
#include <linux/atomic.h>
#include <linux/cpufeature.h>
#include <linux/delay.h>
#include <linux/miscdevice.h>
#include <linux/module.h>
#include <linux/spinlock.h>

static struct crypto_engine *phmac_crypto_engine;
#define MAX_QLEN 10

static bool pkey_clrkey_allowed;
module_param_named(clrkey, pkey_clrkey_allowed, bool, 0444);
MODULE_PARM_DESC(clrkey, "Allow clear key material (default N)");

/*
 * A simple hash walk helper
 */

struct hash_walk_helper {
        struct crypto_hash_walk walk;
        const u8 *walkaddr;
        int walkbytes;
};

/*
 * Prepare hash walk helper.
 * Set up the base hash walk, fill walkaddr and walkbytes.
 * Returns 0 on success or negative value on error.
 */
static inline int hwh_prepare(struct ahash_request *req,
                              struct hash_walk_helper *hwh)
{
        hwh->walkbytes = crypto_hash_walk_first(req, &hwh->walk);
        if (hwh->walkbytes < 0)
                return hwh->walkbytes;
        hwh->walkaddr = hwh->walk.data;
        return 0;
}

/*
 * Advance hash walk helper by n bytes.
 * Progress the walkbytes and walkaddr fields by n bytes.
 * If walkbytes is then 0, pull next hunk from hash walk
 * and update walkbytes and walkaddr.
 * If n is negative, unmap hash walk and return error.
 * Returns 0 on success or negative value on error.
 */
static inline int hwh_advance(struct hash_walk_helper *hwh, int n)
{
        if (n < 0)
                return crypto_hash_walk_done(&hwh->walk, n);

        hwh->walkbytes -= n;
        hwh->walkaddr += n;
        if (hwh->walkbytes > 0)
                return 0;

        hwh->walkbytes = crypto_hash_walk_done(&hwh->walk, 0);
        if (hwh->walkbytes < 0)
                return hwh->walkbytes;

        hwh->walkaddr = hwh->walk.data;
        return 0;
}

/*
 * KMAC param block layout for sha2 function codes:
 * The layout of the param block for the KMAC instruction depends on the
 * blocksize of the used hashing sha2-algorithm function codes. The param block
 * contains the hash chaining value (cv), the input message bit-length (imbl)
 * and the hmac-secret (key). To prevent code duplication, the sizes of all
 * these are calculated based on the blocksize.
 *
 * param-block:
 * +-------+
 * | cv    |
 * +-------+
 * | imbl  |
 * +-------+
 * | key   |
 * +-------+
 *
 * sizes:
 * part | sh2-alg | calculation | size | type
 * -----+---------+-------------+------+--------
 * cv   | 224/256 | blocksize/2 |   32 |  u64[8]
 *      | 384/512 |             |   64 | u128[8]
 * imbl | 224/256 | blocksize/8 |    8 |     u64
 *      | 384/512 |             |   16 |    u128
 * key  | 224/256 | blocksize   |   96 |  u8[96]
 *      | 384/512 |             |  160 | u8[160]
 */

#define MAX_DIGEST_SIZE         SHA512_DIGEST_SIZE
#define MAX_IMBL_SIZE           sizeof(u128)
#define MAX_BLOCK_SIZE          SHA512_BLOCK_SIZE

#define SHA2_CV_SIZE(bs)        ((bs) >> 1)
#define SHA2_IMBL_SIZE(bs)      ((bs) >> 3)

#define SHA2_IMBL_OFFSET(bs)    (SHA2_CV_SIZE(bs))
#define SHA2_KEY_OFFSET(bs)     (SHA2_CV_SIZE(bs) + SHA2_IMBL_SIZE(bs))

#define PHMAC_MAX_KEYSIZE       256
#define PHMAC_SHA256_PK_SIZE    (SHA256_BLOCK_SIZE + 32)
#define PHMAC_SHA512_PK_SIZE    (SHA512_BLOCK_SIZE + 32)
#define PHMAC_MAX_PK_SIZE       PHMAC_SHA512_PK_SIZE

/* phmac protected key struct */
struct phmac_protkey {
        u32 type;
        u32 len;
        u8 protkey[PHMAC_MAX_PK_SIZE];
};

#define PK_STATE_NO_KEY              0
#define PK_STATE_CONVERT_IN_PROGRESS 1
#define PK_STATE_VALID               2

/* phmac tfm context */
struct phmac_tfm_ctx {
        /* source key material used to derive a protected key from */
        u8 keybuf[PHMAC_MAX_KEYSIZE];
        unsigned int keylen;

        /* cpacf function code to use with this protected key type */
        long fc;

        /* nr of requests enqueued via crypto engine which use this tfm ctx */
        atomic_t via_engine_ctr;

        /* spinlock to atomic read/update all the following fields */
        spinlock_t pk_lock;

        /* see PK_STATE* defines above, < 0 holds convert failure rc  */
        int pk_state;
        /* if state is valid, pk holds the protected key */
        struct phmac_protkey pk;
};

union kmac_gr0 {
        unsigned long reg;
        struct {
                unsigned long           : 48;
                unsigned long ikp       :  1;
                unsigned long iimp      :  1;
                unsigned long ccup      :  1;
                unsigned long           :  6;
                unsigned long fc        :  7;
        };
};

struct kmac_sha2_ctx {
        u8 param[MAX_DIGEST_SIZE + MAX_IMBL_SIZE + PHMAC_MAX_PK_SIZE];
        union kmac_gr0 gr0;
        u8 buf[MAX_BLOCK_SIZE];
        u64 buflen[2];
};

enum async_op {
        OP_NOP = 0,
        OP_UPDATE,
        OP_FINAL,
        OP_FINUP,
};

/* phmac request context */
struct phmac_req_ctx {
        struct hash_walk_helper hwh;
        struct kmac_sha2_ctx kmac_ctx;
        enum async_op async_op;
};

/*
 * Pkey 'token' struct used to derive a protected key value from a clear key.
 */
struct hmac_clrkey_token {
        u8  type;
        u8  res0[3];
        u8  version;
        u8  res1[3];
        u32 keytype;
        u32 len;
        u8 key[];
} __packed;

static int hash_key(const u8 *in, unsigned int inlen,
                    u8 *digest, unsigned int digestsize)
{
        unsigned long func;
        union {
                struct sha256_paramblock {
                        u32 h[8];
                        u64 mbl;
                } sha256;
                struct sha512_paramblock {
                        u64 h[8];
                        u128 mbl;
                } sha512;
        } __packed param;

#define PARAM_INIT(x, y, z)                \
        param.sha##x.h[0] = SHA##y ## _H0; \
        param.sha##x.h[1] = SHA##y ## _H1; \
        param.sha##x.h[2] = SHA##y ## _H2; \
        param.sha##x.h[3] = SHA##y ## _H3; \
        param.sha##x.h[4] = SHA##y ## _H4; \
        param.sha##x.h[5] = SHA##y ## _H5; \
        param.sha##x.h[6] = SHA##y ## _H6; \
        param.sha##x.h[7] = SHA##y ## _H7; \
        param.sha##x.mbl = (z)

        switch (digestsize) {
        case SHA224_DIGEST_SIZE:
                func = CPACF_KLMD_SHA_256;
                PARAM_INIT(256, 224, inlen * 8);
                break;
        case SHA256_DIGEST_SIZE:
                func = CPACF_KLMD_SHA_256;
                PARAM_INIT(256, 256, inlen * 8);
                break;
        case SHA384_DIGEST_SIZE:
                func = CPACF_KLMD_SHA_512;
                PARAM_INIT(512, 384, inlen * 8);
                break;
        case SHA512_DIGEST_SIZE:
                func = CPACF_KLMD_SHA_512;
                PARAM_INIT(512, 512, inlen * 8);
                break;
        default:
                return -EINVAL;
        }

#undef PARAM_INIT

        cpacf_klmd(func, &param, in, inlen);

        memcpy(digest, &param, digestsize);

        return 0;
}

/*
 * make_clrkey_token() - wrap the clear key into a pkey clearkey token.
 */
static inline int make_clrkey_token(const u8 *clrkey, size_t clrkeylen,
                                    unsigned int digestsize, u8 *dest)
{
        struct hmac_clrkey_token *token = (struct hmac_clrkey_token *)dest;
        unsigned int blocksize;
        int rc;

        token->type = 0x00;
        token->version = 0x02;
        switch (digestsize) {
        case SHA224_DIGEST_SIZE:
        case SHA256_DIGEST_SIZE:
                token->keytype = PKEY_KEYTYPE_HMAC_512;
                blocksize = 64;
                break;
        case SHA384_DIGEST_SIZE:
        case SHA512_DIGEST_SIZE:
                token->keytype = PKEY_KEYTYPE_HMAC_1024;
                blocksize = 128;
                break;
        default:
                return -EINVAL;
        }
        token->len = blocksize;

        if (clrkeylen > blocksize) {
                rc = hash_key(clrkey, clrkeylen, token->key, digestsize);
                if (rc)
                        return rc;
        } else {
                memcpy(token->key, clrkey, clrkeylen);
        }

        return 0;
}

/*
 * phmac_tfm_ctx_setkey() - Set key value into tfm context, maybe construct
 * a clear key token digestible by pkey from a clear key value.
 */
static inline int phmac_tfm_ctx_setkey(struct phmac_tfm_ctx *tfm_ctx,
                                       const u8 *key, unsigned int keylen)
{
        if (keylen > sizeof(tfm_ctx->keybuf))
                return -EINVAL;

        memcpy(tfm_ctx->keybuf, key, keylen);
        tfm_ctx->keylen = keylen;

        return 0;
}

/*
 * Convert the raw key material into a protected key via PKEY api.
 * This function may sleep - don't call in non-sleeping context.
 */
static inline int convert_key(const u8 *key, unsigned int keylen,
                              struct phmac_protkey *pk, bool tested)
{
        u32 xflags = PKEY_XFLAG_NOMEMALLOC;
        int rc, i;

        if (tested && !pkey_clrkey_allowed)
                xflags |= PKEY_XFLAG_NOCLEARKEY;

        pk->len = sizeof(pk->protkey);

        /*
         * In case of a busy card retry with increasing delay
         * of 200, 400, 800 and 1600 ms - in total 3 s.
         */
        for (rc = -EIO, i = 0; rc && i < 5; i++) {
                if (rc == -EBUSY && msleep_interruptible((1 << i) * 100)) {
                        rc = -EINTR;
                        goto out;
                }
                rc = pkey_key2protkey(key, keylen,
                                      pk->protkey, &pk->len, &pk->type,
                                      xflags);
        }

out:
        pr_debug("rc=%d\n", rc);
        return rc;
}

/*
 * (Re-)Convert the raw key material from the tfm ctx into a protected
 * key via convert_key() function. Update the pk_state, pk_type, pk_len
 * and the protected key in the tfm context.
 * Please note this function may be invoked concurrently with the very
 * same tfm context. The pk_lock spinlock in the context ensures an
 * atomic update of the pk and the pk state but does not guarantee any
 * order of update. So a fresh converted valid protected key may get
 * updated with an 'old' expired key value. As the cpacf instructions
 * detect this, refuse to operate with an invalid key and the calling
 * code triggers a (re-)conversion this does no harm. This may lead to
 * unnecessary additional conversion but never to invalid data on the
 * hash operation.
 */
static int phmac_convert_key(struct phmac_tfm_ctx *tfm_ctx, bool tested)
{
        struct phmac_protkey pk;
        int rc;

        spin_lock_bh(&tfm_ctx->pk_lock);
        tfm_ctx->pk_state = PK_STATE_CONVERT_IN_PROGRESS;
        spin_unlock_bh(&tfm_ctx->pk_lock);

        rc = convert_key(tfm_ctx->keybuf, tfm_ctx->keylen, &pk, tested);

        /* update context */
        spin_lock_bh(&tfm_ctx->pk_lock);
        if (rc) {
                tfm_ctx->pk_state = rc;
        } else {
                tfm_ctx->pk_state = PK_STATE_VALID;
                tfm_ctx->pk = pk;
        }
        spin_unlock_bh(&tfm_ctx->pk_lock);

        memzero_explicit(&pk, sizeof(pk));
        pr_debug("rc=%d\n", rc);
        return rc;
}

/*
 * kmac_sha2_set_imbl - sets the input message bit-length based on the blocksize
 */
static inline void kmac_sha2_set_imbl(u8 *param, u64 buflen_lo,
                                      u64 buflen_hi, unsigned int blocksize)
{
        u8 *imbl = param + SHA2_IMBL_OFFSET(blocksize);

        switch (blocksize) {
        case SHA256_BLOCK_SIZE:
                *(u64 *)imbl = buflen_lo * BITS_PER_BYTE;
                break;
        case SHA512_BLOCK_SIZE:
                *(u128 *)imbl = (((u128)buflen_hi << 64) + buflen_lo) << 3;
                break;
        default:
                break;
        }
}

static int phmac_kmac_update(struct ahash_request *req, bool maysleep)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
        struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
        struct kmac_sha2_ctx *ctx = &req_ctx->kmac_ctx;
        struct hash_walk_helper *hwh = &req_ctx->hwh;
        unsigned int bs = crypto_ahash_blocksize(tfm);
        bool tested = crypto_ahash_tested(tfm);
        unsigned int offset, k, n;
        int rc = 0;

        /*
         * The walk is always mapped when this function is called.
         * Note that in case of partial processing or failure the walk
         * is NOT unmapped here. So a follow up task may reuse the walk
         * or in case of unrecoverable failure needs to unmap it.
         */

        while (hwh->walkbytes > 0) {
                /* check sha2 context buffer */
                offset = ctx->buflen[0] % bs;
                if (offset + hwh->walkbytes < bs)
                        goto store;

                if (offset) {
                        /* fill ctx buffer up to blocksize and process this block */
                        n = bs - offset;
                        memcpy(ctx->buf + offset, hwh->walkaddr, n);
                        ctx->gr0.iimp = 1;
                        for (;;) {
                                k = _cpacf_kmac(&ctx->gr0.reg, ctx->param, ctx->buf, bs);
                                if (likely(k == bs))
                                        break;
                                if (unlikely(k > 0)) {
                                        /*
                                         * Can't deal with hunks smaller than blocksize.
                                         * And kmac should always return the nr of
                                         * processed bytes as 0 or a multiple of the
                                         * blocksize.
                                         */
                                        rc = -EIO;
                                        goto out;
                                }
                                /* protected key is invalid and needs re-conversion */
                                if (!maysleep) {
                                        rc = -EKEYEXPIRED;
                                        goto out;
                                }
                                rc = phmac_convert_key(tfm_ctx, tested);
                                if (rc)
                                        goto out;
                                spin_lock_bh(&tfm_ctx->pk_lock);
                                memcpy(ctx->param + SHA2_KEY_OFFSET(bs),
                                       tfm_ctx->pk.protkey, tfm_ctx->pk.len);
                                spin_unlock_bh(&tfm_ctx->pk_lock);
                        }
                        ctx->buflen[0] += n;
                        if (ctx->buflen[0] < n)
                                ctx->buflen[1]++;
                        rc = hwh_advance(hwh, n);
                        if (unlikely(rc))
                                goto out;
                        offset = 0;
                }

                /* process as many blocks as possible from the walk */
                while (hwh->walkbytes >= bs) {
                        n = (hwh->walkbytes / bs) * bs;
                        ctx->gr0.iimp = 1;
                        k = _cpacf_kmac(&ctx->gr0.reg, ctx->param, hwh->walkaddr, n);
                        if (likely(k > 0)) {
                                ctx->buflen[0] += k;
                                if (ctx->buflen[0] < k)
                                        ctx->buflen[1]++;
                                rc = hwh_advance(hwh, k);
                                if (unlikely(rc))
                                        goto out;
                        }
                        if (unlikely(k < n)) {
                                /* protected key is invalid and needs re-conversion */
                                if (!maysleep) {
                                        rc = -EKEYEXPIRED;
                                        goto out;
                                }
                                rc = phmac_convert_key(tfm_ctx, tested);
                                if (rc)
                                        goto out;
                                spin_lock_bh(&tfm_ctx->pk_lock);
                                memcpy(ctx->param + SHA2_KEY_OFFSET(bs),
                                       tfm_ctx->pk.protkey, tfm_ctx->pk.len);
                                spin_unlock_bh(&tfm_ctx->pk_lock);
                        }
                }

store:
                /* store incomplete block in context buffer */
                if (hwh->walkbytes) {
                        memcpy(ctx->buf + offset, hwh->walkaddr, hwh->walkbytes);
                        ctx->buflen[0] += hwh->walkbytes;
                        if (ctx->buflen[0] < hwh->walkbytes)
                                ctx->buflen[1]++;
                        rc = hwh_advance(hwh, hwh->walkbytes);
                        if (unlikely(rc))
                                goto out;
                }

        } /* end of while (hwh->walkbytes > 0) */

out:
        pr_debug("rc=%d\n", rc);
        return rc;
}

static int phmac_kmac_final(struct ahash_request *req, bool maysleep)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
        struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
        struct kmac_sha2_ctx *ctx = &req_ctx->kmac_ctx;
        unsigned int ds = crypto_ahash_digestsize(tfm);
        unsigned int bs = crypto_ahash_blocksize(tfm);
        bool tested = crypto_ahash_tested(tfm);
        unsigned int k, n;
        int rc = 0;

        n = ctx->buflen[0] % bs;
        ctx->gr0.iimp = 0;
        kmac_sha2_set_imbl(ctx->param, ctx->buflen[0], ctx->buflen[1], bs);
        for (;;) {
                k = _cpacf_kmac(&ctx->gr0.reg, ctx->param, ctx->buf, n);
                if (likely(k == n))
                        break;
                if (unlikely(k > 0)) {
                        /* Can't deal with hunks smaller than blocksize. */
                        rc = -EIO;
                        goto out;
                }
                /* protected key is invalid and needs re-conversion */
                if (!maysleep) {
                        rc = -EKEYEXPIRED;
                        goto out;
                }
                rc = phmac_convert_key(tfm_ctx, tested);
                if (rc)
                        goto out;
                spin_lock_bh(&tfm_ctx->pk_lock);
                memcpy(ctx->param + SHA2_KEY_OFFSET(bs),
                       tfm_ctx->pk.protkey, tfm_ctx->pk.len);
                spin_unlock_bh(&tfm_ctx->pk_lock);
        }

        memcpy(req->result, ctx->param, ds);

out:
        pr_debug("rc=%d\n", rc);
        return rc;
}

static int phmac_init(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
        struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
        struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx;
        unsigned int bs = crypto_ahash_blocksize(tfm);
        int rc = 0;

        /* zero request context (includes the kmac sha2 context) */
        memset(req_ctx, 0, sizeof(*req_ctx));

        /*
         * setkey() should have set a valid fc into the tfm context.
         * Copy this function code into the gr0 field of the kmac context.
         */
        if (!tfm_ctx->fc) {
                rc = -ENOKEY;
                goto out;
        }
        kmac_ctx->gr0.fc = tfm_ctx->fc;

        /*
         * Copy the pk from tfm ctx into kmac ctx. The protected key
         * may be outdated but update() and final() will handle this.
         */
        spin_lock_bh(&tfm_ctx->pk_lock);
        memcpy(kmac_ctx->param + SHA2_KEY_OFFSET(bs),
               tfm_ctx->pk.protkey, tfm_ctx->pk.len);
        spin_unlock_bh(&tfm_ctx->pk_lock);

out:
        pr_debug("rc=%d\n", rc);
        return rc;
}

static int phmac_update(struct ahash_request *req)
{
        struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
        struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx;
        struct hash_walk_helper *hwh = &req_ctx->hwh;
        int rc;

        /* prep the walk in the request context */
        rc = hwh_prepare(req, hwh);
        if (rc)
                goto out;

        /* Try synchronous operation if no active engine usage */
        if (!atomic_read(&tfm_ctx->via_engine_ctr)) {
                rc = phmac_kmac_update(req, false);
                if (rc == 0)
                        goto out;
        }

        /*
         * If sync operation failed or key expired or there are already
         * requests enqueued via engine, fallback to async. Mark tfm as
         * using engine to serialize requests.
         */
        if (rc == 0 || rc == -EKEYEXPIRED) {
                req_ctx->async_op = OP_UPDATE;
                atomic_inc(&tfm_ctx->via_engine_ctr);
                rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req);
                if (rc != -EINPROGRESS)
                        atomic_dec(&tfm_ctx->via_engine_ctr);
        }

        if (rc != -EINPROGRESS) {
                hwh_advance(hwh, rc);
                memzero_explicit(kmac_ctx, sizeof(*kmac_ctx));
        }

out:
        pr_debug("rc=%d\n", rc);
        return rc;
}

static int phmac_final(struct ahash_request *req)
{
        struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
        struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx;
        int rc = 0;

        /* Try synchronous operation if no active engine usage */
        if (!atomic_read(&tfm_ctx->via_engine_ctr)) {
                rc = phmac_kmac_final(req, false);
                if (rc == 0)
                        goto out;
        }

        /*
         * If sync operation failed or key expired or there are already
         * requests enqueued via engine, fallback to async. Mark tfm as
         * using engine to serialize requests.
         */
        if (rc == 0 || rc == -EKEYEXPIRED) {
                req_ctx->async_op = OP_FINAL;
                atomic_inc(&tfm_ctx->via_engine_ctr);
                rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req);
                if (rc != -EINPROGRESS)
                        atomic_dec(&tfm_ctx->via_engine_ctr);
        }

out:
        if (rc != -EINPROGRESS)
                memzero_explicit(kmac_ctx, sizeof(*kmac_ctx));
        pr_debug("rc=%d\n", rc);
        return rc;
}

static int phmac_finup(struct ahash_request *req)
{
        struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
        struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx;
        struct hash_walk_helper *hwh = &req_ctx->hwh;
        int rc;

        /* prep the walk in the request context */
        rc = hwh_prepare(req, hwh);
        if (rc)
                goto out;

        req_ctx->async_op = OP_FINUP;

        /* Try synchronous operations if no active engine usage */
        if (!atomic_read(&tfm_ctx->via_engine_ctr)) {
                rc = phmac_kmac_update(req, false);
                if (rc == 0)
                        req_ctx->async_op = OP_FINAL;
        }
        if (!rc && req_ctx->async_op == OP_FINAL &&
            !atomic_read(&tfm_ctx->via_engine_ctr)) {
                rc = phmac_kmac_final(req, false);
                if (rc == 0)
                        goto out;
        }

        /*
         * If sync operation failed or key expired or there are already
         * requests enqueued via engine, fallback to async. Mark tfm as
         * using engine to serialize requests.
         */
        if (rc == 0 || rc == -EKEYEXPIRED) {
                /* req->async_op has been set to either OP_FINUP or OP_FINAL */
                atomic_inc(&tfm_ctx->via_engine_ctr);
                rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req);
                if (rc != -EINPROGRESS)
                        atomic_dec(&tfm_ctx->via_engine_ctr);
        }

        if (rc != -EINPROGRESS)
                hwh_advance(hwh, rc);

out:
        if (rc != -EINPROGRESS)
                memzero_explicit(kmac_ctx, sizeof(*kmac_ctx));
        pr_debug("rc=%d\n", rc);
        return rc;
}

static int phmac_digest(struct ahash_request *req)
{
        int rc;

        rc = phmac_init(req);
        if (rc)
                goto out;

        rc = phmac_finup(req);

out:
        pr_debug("rc=%d\n", rc);
        return rc;
}

static int phmac_setkey(struct crypto_ahash *tfm,
                        const u8 *key, unsigned int keylen)
{
        struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
        unsigned int ds = crypto_ahash_digestsize(tfm);
        unsigned int bs = crypto_ahash_blocksize(tfm);
        bool tested = crypto_ahash_tested(tfm);
        unsigned int tmpkeylen;
        u8 *tmpkey = NULL;
        int rc = 0;

        if (!tested) {
                /*
                 * selftest running: key is a raw hmac clear key and needs
                 * to get embedded into a 'clear key token' in order to have
                 * it correctly processed by the pkey module.
                 */
                tmpkeylen = sizeof(struct hmac_clrkey_token) + bs;
                tmpkey = kzalloc(tmpkeylen, GFP_KERNEL);
                if (!tmpkey) {
                        rc = -ENOMEM;
                        goto out;
                }
                rc = make_clrkey_token(key, keylen, ds, tmpkey);
                if (rc)
                        goto out;
                keylen = tmpkeylen;
                key = tmpkey;
        }

        /* copy raw key into tfm context */
        rc = phmac_tfm_ctx_setkey(tfm_ctx, key, keylen);
        if (rc)
                goto out;

        /* convert raw key into protected key */
        rc = phmac_convert_key(tfm_ctx, tested);
        if (rc)
                goto out;

        /* set function code in tfm context, check for valid pk type */
        switch (ds) {
        case SHA224_DIGEST_SIZE:
                if (tfm_ctx->pk.type != PKEY_KEYTYPE_HMAC_512)
                        rc = -EINVAL;
                else
                        tfm_ctx->fc = CPACF_KMAC_PHMAC_SHA_224;
                break;
        case SHA256_DIGEST_SIZE:
                if (tfm_ctx->pk.type != PKEY_KEYTYPE_HMAC_512)
                        rc = -EINVAL;
                else
                        tfm_ctx->fc = CPACF_KMAC_PHMAC_SHA_256;
                break;
        case SHA384_DIGEST_SIZE:
                if (tfm_ctx->pk.type != PKEY_KEYTYPE_HMAC_1024)
                        rc = -EINVAL;
                else
                        tfm_ctx->fc = CPACF_KMAC_PHMAC_SHA_384;
                break;
        case SHA512_DIGEST_SIZE:
                if (tfm_ctx->pk.type != PKEY_KEYTYPE_HMAC_1024)
                        rc = -EINVAL;
                else
                        tfm_ctx->fc = CPACF_KMAC_PHMAC_SHA_512;
                break;
        default:
                tfm_ctx->fc = 0;
                rc = -EINVAL;
        }

out:
        kfree(tmpkey);
        pr_debug("rc=%d\n", rc);
        return rc;
}

static int phmac_export(struct ahash_request *req, void *out)
{
        struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
        struct kmac_sha2_ctx *ctx = &req_ctx->kmac_ctx;

        memcpy(out, ctx, sizeof(*ctx));

        return 0;
}

static int phmac_import(struct ahash_request *req, const void *in)
{
        struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
        struct kmac_sha2_ctx *ctx = &req_ctx->kmac_ctx;

        memset(req_ctx, 0, sizeof(*req_ctx));
        memcpy(ctx, in, sizeof(*ctx));

        return 0;
}

static int phmac_init_tfm(struct crypto_ahash *tfm)
{
        struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);

        memset(tfm_ctx, 0, sizeof(*tfm_ctx));
        spin_lock_init(&tfm_ctx->pk_lock);

        crypto_ahash_set_reqsize(tfm, sizeof(struct phmac_req_ctx));

        return 0;
}

static void phmac_exit_tfm(struct crypto_ahash *tfm)
{
        struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);

        memzero_explicit(tfm_ctx->keybuf, sizeof(tfm_ctx->keybuf));
        memzero_explicit(&tfm_ctx->pk, sizeof(tfm_ctx->pk));
}

static int phmac_do_one_request(struct crypto_engine *engine, void *areq)
{
        struct ahash_request *req = ahash_request_cast(areq);
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
        struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
        struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx;
        struct hash_walk_helper *hwh = &req_ctx->hwh;
        int rc = -EINVAL;

        /*
         * Three kinds of requests come in here:
         * 1. req->async_op == OP_UPDATE with req->nbytes > 0
         * 2. req->async_op == OP_FINUP with req->nbytes > 0
         * 3. req->async_op == OP_FINAL
         * For update and finup the hwh walk has already been prepared
         * by the caller. For final there is no hwh walk needed.
         */

        switch (req_ctx->async_op) {
        case OP_UPDATE:
        case OP_FINUP:
                rc = phmac_kmac_update(req, true);
                if (rc == -EKEYEXPIRED) {
                        /*
                         * Protected key expired, conversion is in process.
                         * Trigger a re-schedule of this request by returning
                         * -ENOSPC ("hardware queue full") to the crypto engine.
                         * To avoid immediately re-invocation of this callback,
                         * tell scheduler to voluntarily give up the CPU here.
                         */
                        pr_debug("rescheduling request\n");
                        cond_resched();
                        return -ENOSPC;
                } else if (rc) {
                        hwh_advance(hwh, rc);
                        goto out;
                }
                if (req_ctx->async_op == OP_UPDATE)
                        break;
                req_ctx->async_op = OP_FINAL;
                fallthrough;
        case OP_FINAL:
                rc = phmac_kmac_final(req, true);
                if (rc == -EKEYEXPIRED) {
                        /*
                         * Protected key expired, conversion is in process.
                         * Trigger a re-schedule of this request by returning
                         * -ENOSPC ("hardware queue full") to the crypto engine.
                         * To avoid immediately re-invocation of this callback,
                         * tell scheduler to voluntarily give up the CPU here.
                         */
                        pr_debug("rescheduling request\n");
                        cond_resched();
                        return -ENOSPC;
                }
                break;
        default:
                /* unknown/unsupported/unimplemented asynch op */
                return -EOPNOTSUPP;
        }

out:
        if (rc || req_ctx->async_op == OP_FINAL)
                memzero_explicit(kmac_ctx, sizeof(*kmac_ctx));
        pr_debug("request complete with rc=%d\n", rc);
        local_bh_disable();
        atomic_dec(&tfm_ctx->via_engine_ctr);
        crypto_finalize_hash_request(engine, req, rc);
        local_bh_enable();
        return rc;
}

#define S390_ASYNC_PHMAC_ALG(x)                                         \
{                                                                       \
        .base = {                                                       \
                .init     = phmac_init,                                 \
                .update   = phmac_update,                               \
                .final    = phmac_final,                                \
                .finup    = phmac_finup,                                \
                .digest   = phmac_digest,                               \
                .setkey   = phmac_setkey,                               \
                .import   = phmac_import,                               \
                .export   = phmac_export,                               \
                .init_tfm = phmac_init_tfm,                             \
                .exit_tfm = phmac_exit_tfm,                             \
                .halg = {                                               \
                        .digestsize = SHA##x##_DIGEST_SIZE,             \
                        .statesize  = sizeof(struct kmac_sha2_ctx),     \
                        .base = {                                       \
                                .cra_name = "phmac(sha" #x ")",         \
                                .cra_driver_name = "phmac_s390_sha" #x, \
                                .cra_blocksize = SHA##x##_BLOCK_SIZE,   \
                                .cra_priority = 400,                    \
                                .cra_flags = CRYPTO_ALG_ASYNC |         \
                                             CRYPTO_ALG_NO_FALLBACK,    \
                                .cra_ctxsize = sizeof(struct phmac_tfm_ctx), \
                                .cra_module = THIS_MODULE,              \
                        },                                              \
                },                                                      \
        },                                                              \
        .op = {                                                         \
                .do_one_request = phmac_do_one_request,                 \
        },                                                              \
}

static struct phmac_alg {
        unsigned int fc;
        struct ahash_engine_alg alg;
        bool registered;
} phmac_algs[] = {
        {
                .fc = CPACF_KMAC_PHMAC_SHA_224,
                .alg = S390_ASYNC_PHMAC_ALG(224),
        }, {
                .fc = CPACF_KMAC_PHMAC_SHA_256,
                .alg = S390_ASYNC_PHMAC_ALG(256),
        }, {
                .fc = CPACF_KMAC_PHMAC_SHA_384,
                .alg = S390_ASYNC_PHMAC_ALG(384),
        }, {
                .fc = CPACF_KMAC_PHMAC_SHA_512,
                .alg = S390_ASYNC_PHMAC_ALG(512),
        }
};

static struct miscdevice phmac_dev = {
        .name   = "phmac",
        .minor  = MISC_DYNAMIC_MINOR,
};

static void s390_phmac_exit(void)
{
        struct phmac_alg *phmac;
        int i;

        if (phmac_crypto_engine) {
                crypto_engine_stop(phmac_crypto_engine);
                crypto_engine_exit(phmac_crypto_engine);
        }

        for (i = ARRAY_SIZE(phmac_algs) - 1; i >= 0; i--) {
                phmac = &phmac_algs[i];
                if (phmac->registered)
                        crypto_engine_unregister_ahash(&phmac->alg);
        }

        misc_deregister(&phmac_dev);
}

static int __init s390_phmac_init(void)
{
        struct phmac_alg *phmac;
        int i, rc;

        /* for selftest cpacf klmd subfunction is needed */
        if (!cpacf_query_func(CPACF_KLMD, CPACF_KLMD_SHA_256))
                return -ENODEV;
        if (!cpacf_query_func(CPACF_KLMD, CPACF_KLMD_SHA_512))
                return -ENODEV;

        /* register a simple phmac pseudo misc device */
        rc = misc_register(&phmac_dev);
        if (rc)
                return rc;

        /* with this pseudo device alloc and start a crypto engine */
        phmac_crypto_engine =
                crypto_engine_alloc_init_and_set(phmac_dev.this_device,
                                                 true, false, MAX_QLEN);
        if (!phmac_crypto_engine) {
                rc = -ENOMEM;
                goto out_err;
        }
        rc = crypto_engine_start(phmac_crypto_engine);
        if (rc) {
                crypto_engine_exit(phmac_crypto_engine);
                phmac_crypto_engine = NULL;
                goto out_err;
        }

        for (i = 0; i < ARRAY_SIZE(phmac_algs); i++) {
                phmac = &phmac_algs[i];
                if (!cpacf_query_func(CPACF_KMAC, phmac->fc))
                        continue;
                rc = crypto_engine_register_ahash(&phmac->alg);
                if (rc)
                        goto out_err;
                phmac->registered = true;
                pr_debug("%s registered\n", phmac->alg.base.halg.base.cra_name);
        }

        return 0;

out_err:
        s390_phmac_exit();
        return rc;
}

module_init(s390_phmac_init);
module_exit(s390_phmac_exit);

MODULE_ALIAS_CRYPTO("phmac(sha224)");
MODULE_ALIAS_CRYPTO("phmac(sha256)");
MODULE_ALIAS_CRYPTO("phmac(sha384)");
MODULE_ALIAS_CRYPTO("phmac(sha512)");

MODULE_DESCRIPTION("S390 HMAC driver for protected keys");
MODULE_LICENSE("GPL");