// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2017 Marvell
 *
 * Antoine Tenart <antoine.tenart@free-electrons.com>
 */

#include <linux/clk.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/dmapool.h>
#include <linux/firmware.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/of_platform.h>
#include <linux/of_irq.h>
#include <linux/pci.h>
#include <linux/platform_device.h>
#include <linux/workqueue.h>

#include <crypto/internal/aead.h>
#include <crypto/internal/hash.h>
#include <crypto/internal/skcipher.h>

#include "safexcel.h"

static u32 max_rings = EIP197_MAX_RINGS;
module_param(max_rings, uint, 0644);
MODULE_PARM_DESC(max_rings, "Maximum number of rings to use.");

static void eip197_trc_cache_setupvirt(struct safexcel_crypto_priv *priv)
{
        int i;

        /*
         * Map all interfaces/rings to register index 0
         * so they can share contexts. Without this, the EIP197 will
         * assume each interface/ring to be in its own memory domain
         * i.e. have its own subset of UNIQUE memory addresses.
         * Which would cause records with the SAME memory address to
         * use DIFFERENT cache buffers, causing both poor cache utilization
         * AND serious coherence/invalidation issues.
         */
        for (i = 0; i < 4; i++)
                writel(0, priv->base + EIP197_FLUE_IFC_LUT(i));

        /*
         * Initialize other virtualization regs for cache
         * These may not be in their reset state ...
         */
        for (i = 0; i < priv->config.rings; i++) {
                writel(0, priv->base + EIP197_FLUE_CACHEBASE_LO(i));
                writel(0, priv->base + EIP197_FLUE_CACHEBASE_HI(i));
                writel(EIP197_FLUE_CONFIG_MAGIC,
                       priv->base + EIP197_FLUE_CONFIG(i));
        }
        writel(0, priv->base + EIP197_FLUE_OFFSETS);
        writel(0, priv->base + EIP197_FLUE_ARC4_OFFSET);
}

static void eip197_trc_cache_banksel(struct safexcel_crypto_priv *priv,
                                     u32 addrmid, int *actbank)
{
        u32 val;
        int curbank;

        curbank = addrmid >> 16;
        if (curbank != *actbank) {
                val = readl(priv->base + EIP197_CS_RAM_CTRL);
                val = (val & ~EIP197_CS_BANKSEL_MASK) |
                      (curbank << EIP197_CS_BANKSEL_OFS);
                writel(val, priv->base + EIP197_CS_RAM_CTRL);
                *actbank = curbank;
        }
}

static u32 eip197_trc_cache_probe(struct safexcel_crypto_priv *priv,
                                  int maxbanks, u32 probemask, u32 stride)
{
        u32 val, addrhi, addrlo, addrmid, addralias, delta, marker;
        int actbank;

        /*
         * And probe the actual size of the physically attached cache data RAM
         * Using a binary subdivision algorithm downto 32 byte cache lines.
         */
        addrhi = 1 << (16 + maxbanks);
        addrlo = 0;
        actbank = min(maxbanks - 1, 0);
        while ((addrhi - addrlo) > stride) {
                /* write marker to lowest address in top half */
                addrmid = (addrhi + addrlo) >> 1;
                marker = (addrmid ^ 0xabadbabe) & probemask; /* Unique */
                eip197_trc_cache_banksel(priv, addrmid, &actbank);
                writel(marker,
                        priv->base + EIP197_CLASSIFICATION_RAMS +
                        (addrmid & 0xffff));

                /* write invalid markers to possible aliases */
                delta = 1 << __fls(addrmid);
                while (delta >= stride) {
                        addralias = addrmid - delta;
                        eip197_trc_cache_banksel(priv, addralias, &actbank);
                        writel(~marker,
                               priv->base + EIP197_CLASSIFICATION_RAMS +
                               (addralias & 0xffff));
                        delta >>= 1;
                }

                /* read back marker from top half */
                eip197_trc_cache_banksel(priv, addrmid, &actbank);
                val = readl(priv->base + EIP197_CLASSIFICATION_RAMS +
                            (addrmid & 0xffff));

                if ((val & probemask) == marker)
                        /* read back correct, continue with top half */
                        addrlo = addrmid;
                else
                        /* not read back correct, continue with bottom half */
                        addrhi = addrmid;
        }
        return addrhi;
}

static void eip197_trc_cache_clear(struct safexcel_crypto_priv *priv,
                                   int cs_rc_max, int cs_ht_wc)
{
        int i;
        u32 htable_offset, val, offset;

        /* Clear all records in administration RAM */
        for (i = 0; i < cs_rc_max; i++) {
                offset = EIP197_CLASSIFICATION_RAMS + i * EIP197_CS_RC_SIZE;

                writel(EIP197_CS_RC_NEXT(EIP197_RC_NULL) |
                       EIP197_CS_RC_PREV(EIP197_RC_NULL),
                       priv->base + offset);

                val = EIP197_CS_RC_NEXT(i + 1) | EIP197_CS_RC_PREV(i - 1);
                if (i == 0)
                        val |= EIP197_CS_RC_PREV(EIP197_RC_NULL);
                else if (i == cs_rc_max - 1)
                        val |= EIP197_CS_RC_NEXT(EIP197_RC_NULL);
                writel(val, priv->base + offset + 4);
                /* must also initialize the address key due to ECC! */
                writel(0, priv->base + offset + 8);
                writel(0, priv->base + offset + 12);
        }

        /* Clear the hash table entries */
        htable_offset = cs_rc_max * EIP197_CS_RC_SIZE;
        for (i = 0; i < cs_ht_wc; i++)
                writel(GENMASK(29, 0),
                       priv->base + EIP197_CLASSIFICATION_RAMS +
                       htable_offset + i * sizeof(u32));
}

static int eip197_trc_cache_init(struct safexcel_crypto_priv *priv)
{
        u32 val, dsize, asize;
        int cs_rc_max, cs_ht_wc, cs_trc_rec_wc, cs_trc_lg_rec_wc;
        int cs_rc_abs_max, cs_ht_sz;
        int maxbanks;

        /* Setup (dummy) virtualization for cache */
        eip197_trc_cache_setupvirt(priv);

        /*
         * Enable the record cache memory access and
         * probe the bank select width
         */
        val = readl(priv->base + EIP197_CS_RAM_CTRL);
        val &= ~EIP197_TRC_ENABLE_MASK;
        val |= EIP197_TRC_ENABLE_0 | EIP197_CS_BANKSEL_MASK;
        writel(val, priv->base + EIP197_CS_RAM_CTRL);
        val = readl(priv->base + EIP197_CS_RAM_CTRL);
        maxbanks = ((val&EIP197_CS_BANKSEL_MASK)>>EIP197_CS_BANKSEL_OFS) + 1;

        /* Clear all ECC errors */
        writel(0, priv->base + EIP197_TRC_ECCCTRL);

        /*
         * Make sure the cache memory is accessible by taking record cache into
         * reset. Need data memory access here, not admin access.
         */
        val = readl(priv->base + EIP197_TRC_PARAMS);
        val |= EIP197_TRC_PARAMS_SW_RESET | EIP197_TRC_PARAMS_DATA_ACCESS;
        writel(val, priv->base + EIP197_TRC_PARAMS);

        /* Probed data RAM size in bytes */
        dsize = eip197_trc_cache_probe(priv, maxbanks, 0xffffffff, 32);

        /*
         * Now probe the administration RAM size pretty much the same way
         * Except that only the lower 30 bits are writable and we don't need
         * bank selects
         */
        val = readl(priv->base + EIP197_TRC_PARAMS);
        /* admin access now */
        val &= ~(EIP197_TRC_PARAMS_DATA_ACCESS | EIP197_CS_BANKSEL_MASK);
        writel(val, priv->base + EIP197_TRC_PARAMS);

        /* Probed admin RAM size in admin words */
        asize = eip197_trc_cache_probe(priv, 0, 0x3fffffff, 16) >> 4;

        /* Clear any ECC errors detected while probing! */
        writel(0, priv->base + EIP197_TRC_ECCCTRL);

        /* Sanity check probing results */
        if (dsize < EIP197_MIN_DSIZE || asize < EIP197_MIN_ASIZE) {
                dev_err(priv->dev, "Record cache probing failed (%d,%d).",
                        dsize, asize);
                return -ENODEV;
        }

        /*
         * Determine optimal configuration from RAM sizes
         * Note that we assume that the physical RAM configuration is sane
         * Therefore, we don't do any parameter error checking here ...
         */

        /* For now, just use a single record format covering everything */
        cs_trc_rec_wc = EIP197_CS_TRC_REC_WC;
        cs_trc_lg_rec_wc = EIP197_CS_TRC_REC_WC;

        /*
         * Step #1: How many records will physically fit?
         * Hard upper limit is 1023!
         */
        cs_rc_abs_max = min_t(uint, ((dsize >> 2) / cs_trc_lg_rec_wc), 1023);
        /* Step #2: Need at least 2 words in the admin RAM per record */
        cs_rc_max = min_t(uint, cs_rc_abs_max, (asize >> 1));
        /* Step #3: Determine log2 of hash table size */
        cs_ht_sz = __fls(asize - cs_rc_max) - 2;
        /* Step #4: determine current size of hash table in dwords */
        cs_ht_wc = 16 << cs_ht_sz; /* dwords, not admin words */
        /* Step #5: add back excess words and see if we can fit more records */
        cs_rc_max = min_t(uint, cs_rc_abs_max, asize - (cs_ht_wc >> 2));

        /* Clear the cache RAMs */
        eip197_trc_cache_clear(priv, cs_rc_max, cs_ht_wc);

        /* Disable the record cache memory access */
        val = readl(priv->base + EIP197_CS_RAM_CTRL);
        val &= ~EIP197_TRC_ENABLE_MASK;
        writel(val, priv->base + EIP197_CS_RAM_CTRL);

        /* Write head and tail pointers of the record free chain */
        val = EIP197_TRC_FREECHAIN_HEAD_PTR(0) |
              EIP197_TRC_FREECHAIN_TAIL_PTR(cs_rc_max - 1);
        writel(val, priv->base + EIP197_TRC_FREECHAIN);

        /* Configure the record cache #1 */
        val = EIP197_TRC_PARAMS2_RC_SZ_SMALL(cs_trc_rec_wc) |
              EIP197_TRC_PARAMS2_HTABLE_PTR(cs_rc_max);
        writel(val, priv->base + EIP197_TRC_PARAMS2);

        /* Configure the record cache #2 */
        val = EIP197_TRC_PARAMS_RC_SZ_LARGE(cs_trc_lg_rec_wc) |
              EIP197_TRC_PARAMS_BLK_TIMER_SPEED(1) |
              EIP197_TRC_PARAMS_HTABLE_SZ(cs_ht_sz);
        writel(val, priv->base + EIP197_TRC_PARAMS);

        dev_info(priv->dev, "TRC init: %dd,%da (%dr,%dh)\n",
                 dsize, asize, cs_rc_max, cs_ht_wc + cs_ht_wc);
        return 0;
}

static void eip197_init_firmware(struct safexcel_crypto_priv *priv)
{
        int pe, i;
        u32 val;

        for (pe = 0; pe < priv->config.pes; pe++) {
                /* Configure the token FIFO's */
                writel(3, EIP197_PE(priv) + EIP197_PE_ICE_PUTF_CTRL(pe));
                writel(0, EIP197_PE(priv) + EIP197_PE_ICE_PPTF_CTRL(pe));

                /* Clear the ICE scratchpad memory */
                val = readl(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe));
                val |= EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER |
                       EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN |
                       EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS |
                       EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS;
                writel(val, EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe));

                /* clear the scratchpad RAM using 32 bit writes only */
                for (i = 0; i < EIP197_NUM_OF_SCRATCH_BLOCKS; i++)
                        writel(0, EIP197_PE(priv) +
                                  EIP197_PE_ICE_SCRATCH_RAM(pe) + (i << 2));

                /* Reset the IFPP engine to make its program mem accessible */
                writel(EIP197_PE_ICE_x_CTRL_SW_RESET |
                       EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR |
                       EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR,
                       EIP197_PE(priv) + EIP197_PE_ICE_FPP_CTRL(pe));

                /* Reset the IPUE engine to make its program mem accessible */
                writel(EIP197_PE_ICE_x_CTRL_SW_RESET |
                       EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR |
                       EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR,
                       EIP197_PE(priv) + EIP197_PE_ICE_PUE_CTRL(pe));

                /* Enable access to all IFPP program memories */
                writel(EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN,
                       EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));

                /* bypass the OCE, if present */
                if (priv->flags & EIP197_OCE)
                        writel(EIP197_DEBUG_OCE_BYPASS, EIP197_PE(priv) +
                                                        EIP197_PE_DEBUG(pe));
        }

}

static int eip197_write_firmware(struct safexcel_crypto_priv *priv,
                                  const struct firmware *fw)
{
        u32 val;
        int i;

        /* Write the firmware */
        for (i = 0; i < fw->size / sizeof(u32); i++) {
                if (priv->data->fw_little_endian)
                        val = le32_to_cpu(((const __le32 *)fw->data)[i]);
                else
                        val = be32_to_cpu(((const __be32 *)fw->data)[i]);

                writel(val,
                       priv->base + EIP197_CLASSIFICATION_RAMS +
                       i * sizeof(val));
        }

        /* Exclude final 2 NOPs from size */
        return i - EIP197_FW_TERMINAL_NOPS;
}

/*
 * If FW is actual production firmware, then poll for its initialization
 * to complete and check if it is good for the HW, otherwise just return OK.
 */
static bool poll_fw_ready(struct safexcel_crypto_priv *priv, int fpp)
{
        int pe, pollcnt;
        u32 base, pollofs;

        if (fpp)
                pollofs  = EIP197_FW_FPP_READY;
        else
                pollofs  = EIP197_FW_PUE_READY;

        for (pe = 0; pe < priv->config.pes; pe++) {
                base = EIP197_PE_ICE_SCRATCH_RAM(pe);
                pollcnt = EIP197_FW_START_POLLCNT;
                while (pollcnt &&
                       (readl_relaxed(EIP197_PE(priv) + base +
                              pollofs) != 1)) {
                        pollcnt--;
                }
                if (!pollcnt) {
                        dev_err(priv->dev, "FW(%d) for PE %d failed to start\n",
                                fpp, pe);
                        return false;
                }
        }
        return true;
}

static bool eip197_start_firmware(struct safexcel_crypto_priv *priv,
                                  int ipuesz, int ifppsz, int minifw)
{
        int pe;
        u32 val;

        for (pe = 0; pe < priv->config.pes; pe++) {
                /* Disable access to all program memory */
                writel(0, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));

                /* Start IFPP microengines */
                if (minifw)
                        val = 0;
                else
                        val = EIP197_PE_ICE_UENG_START_OFFSET((ifppsz - 1) &
                                        EIP197_PE_ICE_UENG_INIT_ALIGN_MASK) |
                                EIP197_PE_ICE_UENG_DEBUG_RESET;
                writel(val, EIP197_PE(priv) + EIP197_PE_ICE_FPP_CTRL(pe));

                /* Start IPUE microengines */
                if (minifw)
                        val = 0;
                else
                        val = EIP197_PE_ICE_UENG_START_OFFSET((ipuesz - 1) &
                                        EIP197_PE_ICE_UENG_INIT_ALIGN_MASK) |
                                EIP197_PE_ICE_UENG_DEBUG_RESET;
                writel(val, EIP197_PE(priv) + EIP197_PE_ICE_PUE_CTRL(pe));
        }

        /* For miniFW startup, there is no initialization, so always succeed */
        if (minifw)
                return true;

        /* Wait until all the firmwares have properly started up */
        if (!poll_fw_ready(priv, 1))
                return false;
        if (!poll_fw_ready(priv, 0))
                return false;

        return true;
}

static int eip197_load_firmwares(struct safexcel_crypto_priv *priv)
{
        const char *fw_name[] = {"ifpp.bin", "ipue.bin"};
        const struct firmware *fw[FW_NB];
        char fw_path[37], *dir = NULL;
        int i, j, ret = 0, pe;
        int ipuesz, ifppsz, minifw = 0;

        if (priv->data->version == EIP197D_MRVL)
                dir = "eip197d";
        else if (priv->data->version == EIP197B_MRVL ||
                 priv->data->version == EIP197_DEVBRD)
                dir = "eip197b";
        else if (priv->data->version == EIP197C_MXL)
                dir = "eip197c";
        else
                return -ENODEV;

retry_fw:
        for (i = 0; i < FW_NB; i++) {
                snprintf(fw_path, 37, "inside-secure/%s/%s", dir, fw_name[i]);
                ret = firmware_request_nowarn(&fw[i], fw_path, priv->dev);
                if (ret) {
                        if (minifw || priv->data->version != EIP197B_MRVL)
                                goto release_fw;

                        /* Fallback to the old firmware location for the
                         * EIP197b.
                         */
                        ret = firmware_request_nowarn(&fw[i], fw_name[i],
                                                      priv->dev);
                        if (ret)
                                goto release_fw;
                }
        }

        eip197_init_firmware(priv);

        ifppsz = eip197_write_firmware(priv, fw[FW_IFPP]);

        /* Enable access to IPUE program memories */
        for (pe = 0; pe < priv->config.pes; pe++)
                writel(EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN,
                       EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));

        ipuesz = eip197_write_firmware(priv, fw[FW_IPUE]);

        if (eip197_start_firmware(priv, ipuesz, ifppsz, minifw)) {
                dev_dbg(priv->dev, "Firmware loaded successfully\n");
                return 0;
        }

        ret = -ENODEV;

release_fw:
        for (j = 0; j < i; j++)
                release_firmware(fw[j]);

        if (!minifw) {
                /* Retry with minifw path */
                dev_dbg(priv->dev, "Firmware set not (fully) present or init failed, falling back to BCLA mode\n");
                dir = "eip197_minifw";
                minifw = 1;
                goto retry_fw;
        }

        dev_err(priv->dev, "Firmware load failed.\n");

        return ret;
}

static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv)
{
        u32 cd_size_rnd, val;
        int i, cd_fetch_cnt;

        cd_size_rnd  = (priv->config.cd_size +
                        (BIT(priv->hwconfig.hwdataw) - 1)) >>
                       priv->hwconfig.hwdataw;
        /* determine number of CD's we can fetch into the CD FIFO as 1 block */
        if (priv->flags & SAFEXCEL_HW_EIP197) {
                /* EIP197: try to fetch enough in 1 go to keep all pipes busy */
                cd_fetch_cnt = (1 << priv->hwconfig.hwcfsize) / cd_size_rnd;
                cd_fetch_cnt = min_t(uint, cd_fetch_cnt,
                                     (priv->config.pes * EIP197_FETCH_DEPTH));
        } else {
                /* for the EIP97, just fetch all that fits minus 1 */
                cd_fetch_cnt = ((1 << priv->hwconfig.hwcfsize) /
                                cd_size_rnd) - 1;
        }
        /*
         * Since we're using command desc's way larger than formally specified,
         * we need to check whether we can fit even 1 for low-end EIP196's!
         */
        if (!cd_fetch_cnt) {
                dev_err(priv->dev, "Unable to fit even 1 command desc!\n");
                return -ENODEV;
        }

        for (i = 0; i < priv->config.rings; i++) {
                /* ring base address */
                writel(lower_32_bits(priv->ring[i].cdr.base_dma),
                       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
                writel(upper_32_bits(priv->ring[i].cdr.base_dma),
                       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);

                writel(EIP197_xDR_DESC_MODE_64BIT | EIP197_CDR_DESC_MODE_ADCP |
                       (priv->config.cd_offset << 14) | priv->config.cd_size,
                       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE);
                writel(((cd_fetch_cnt *
                         (cd_size_rnd << priv->hwconfig.hwdataw)) << 16) |
                       (cd_fetch_cnt * (priv->config.cd_offset / sizeof(u32))),
                       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG);

                /* Configure DMA tx control */
                val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
                val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
                writel(val, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DMA_CFG);

                /* clear any pending interrupt */
                writel(GENMASK(5, 0),
                       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT);
        }

        return 0;
}

static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv)
{
        u32 rd_size_rnd, val;
        int i, rd_fetch_cnt;

        /* determine number of RD's we can fetch into the FIFO as one block */
        rd_size_rnd = (EIP197_RD64_FETCH_SIZE +
                       (BIT(priv->hwconfig.hwdataw) - 1)) >>
                      priv->hwconfig.hwdataw;
        if (priv->flags & SAFEXCEL_HW_EIP197) {
                /* EIP197: try to fetch enough in 1 go to keep all pipes busy */
                rd_fetch_cnt = (1 << priv->hwconfig.hwrfsize) / rd_size_rnd;
                rd_fetch_cnt = min_t(uint, rd_fetch_cnt,
                                     (priv->config.pes * EIP197_FETCH_DEPTH));
        } else {
                /* for the EIP97, just fetch all that fits minus 1 */
                rd_fetch_cnt = ((1 << priv->hwconfig.hwrfsize) /
                                rd_size_rnd) - 1;
        }

        for (i = 0; i < priv->config.rings; i++) {
                /* ring base address */
                writel(lower_32_bits(priv->ring[i].rdr.base_dma),
                       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
                writel(upper_32_bits(priv->ring[i].rdr.base_dma),
                       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);

                writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.rd_offset << 14) |
                       priv->config.rd_size,
                       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE);

                writel(((rd_fetch_cnt *
                         (rd_size_rnd << priv->hwconfig.hwdataw)) << 16) |
                       (rd_fetch_cnt * (priv->config.rd_offset / sizeof(u32))),
                       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG);

                /* Configure DMA tx control */
                val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
                val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
                val |= EIP197_HIA_xDR_WR_RES_BUF | EIP197_HIA_xDR_WR_CTRL_BUF;
                writel(val,
                       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DMA_CFG);

                /* clear any pending interrupt */
                writel(GENMASK(7, 0),
                       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT);

                /* enable ring interrupt */
                val = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
                val |= EIP197_RDR_IRQ(i);
                writel(val, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
        }

        return 0;
}

static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
{
        u32 val;
        int i, ret, pe, opbuflo, opbufhi;

        dev_dbg(priv->dev, "HW init: using %d pipe(s) and %d ring(s)\n",
                priv->config.pes, priv->config.rings);

        /*
         * For EIP197's only set maximum number of TX commands to 2^5 = 32
         * Skip for the EIP97 as it does not have this field.
         */
        if (priv->flags & SAFEXCEL_HW_EIP197) {
                val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
                val |= EIP197_MST_CTRL_TX_MAX_CMD(5);
                writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
        }

        /* Configure wr/rd cache values */
        writel(EIP197_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) |
               EIP197_MST_CTRL_WD_CACHE(WR_CACHE_4BITS),
               EIP197_HIA_GEN_CFG(priv) + EIP197_MST_CTRL);

        /* Interrupts reset */

        /* Disable all global interrupts */
        writel(0, EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ENABLE_CTRL);

        /* Clear any pending interrupt */
        writel(GENMASK(31, 0), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK);

        /* Processing Engine configuration */
        for (pe = 0; pe < priv->config.pes; pe++) {
                /* Data Fetch Engine configuration */

                /* Reset all DFE threads */
                writel(EIP197_DxE_THR_CTRL_RESET_PE,
                       EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));

                if (priv->flags & EIP197_PE_ARB)
                        /* Reset HIA input interface arbiter (if present) */
                        writel(EIP197_HIA_RA_PE_CTRL_RESET,
                               EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe));

                /* DMA transfer size to use */
                val = EIP197_HIA_DFE_CFG_DIS_DEBUG;
                val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(6) |
                       EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(9);
                val |= EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(6) |
                       EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(7);
                val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS);
                val |= EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS);
                writel(val, EIP197_HIA_DFE(priv) + EIP197_HIA_DFE_CFG(pe));

                /* Leave the DFE threads reset state */
                writel(0, EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));

                /* Configure the processing engine thresholds */
                writel(EIP197_PE_IN_xBUF_THRES_MIN(6) |
                       EIP197_PE_IN_xBUF_THRES_MAX(9),
                       EIP197_PE(priv) + EIP197_PE_IN_DBUF_THRES(pe));
                writel(EIP197_PE_IN_xBUF_THRES_MIN(6) |
                       EIP197_PE_IN_xBUF_THRES_MAX(7),
                       EIP197_PE(priv) + EIP197_PE_IN_TBUF_THRES(pe));

                if (priv->flags & SAFEXCEL_HW_EIP197)
                        /* enable HIA input interface arbiter and rings */
                        writel(EIP197_HIA_RA_PE_CTRL_EN |
                               GENMASK(priv->config.rings - 1, 0),
                               EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe));

                /* Data Store Engine configuration */

                /* Reset all DSE threads */
                writel(EIP197_DxE_THR_CTRL_RESET_PE,
                       EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));

                /* Wait for all DSE threads to complete */
                while ((readl(EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_STAT(pe)) &
                        GENMASK(15, 12)) != GENMASK(15, 12))
                        ;

                /* DMA transfer size to use */
                if (priv->hwconfig.hwnumpes > 4) {
                        opbuflo = 9;
                        opbufhi = 10;
                } else {
                        opbuflo = 7;
                        opbufhi = 8;
                }
                val = EIP197_HIA_DSE_CFG_DIS_DEBUG;
                val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(opbuflo) |
                       EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(opbufhi);
                val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS);
                val |= EIP197_HIA_DSE_CFG_ALWAYS_BUFFERABLE;
                /* FIXME: instability issues can occur for EIP97 but disabling
                 * it impacts performance.
                 */
                if (priv->flags & SAFEXCEL_HW_EIP197)
                        val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR;
                writel(val, EIP197_HIA_DSE(priv) + EIP197_HIA_DSE_CFG(pe));

                /* Leave the DSE threads reset state */
                writel(0, EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));

                /* Configure the processing engine thresholds */
                writel(EIP197_PE_OUT_DBUF_THRES_MIN(opbuflo) |
                       EIP197_PE_OUT_DBUF_THRES_MAX(opbufhi),
                       EIP197_PE(priv) + EIP197_PE_OUT_DBUF_THRES(pe));

                /* Processing Engine configuration */

                /* Token & context configuration */
                val = EIP197_PE_EIP96_TOKEN_CTRL_CTX_UPDATES |
                      EIP197_PE_EIP96_TOKEN_CTRL_NO_TOKEN_WAIT |
                      EIP197_PE_EIP96_TOKEN_CTRL_ENABLE_TIMEOUT;
                writel(val, EIP197_PE(priv) + EIP197_PE_EIP96_TOKEN_CTRL(pe));

                /* H/W capabilities selection: just enable everything */
                writel(EIP197_FUNCTION_ALL,
                       EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION_EN(pe));
                writel(EIP197_FUNCTION_ALL,
                       EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION2_EN(pe));
        }

        /* Command Descriptor Rings prepare */
        for (i = 0; i < priv->config.rings; i++) {
                /* Clear interrupts for this ring */
                writel(GENMASK(31, 0),
                       EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CLR(i));

                /* Disable external triggering */
                writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG);

                /* Clear the pending prepared counter */
                writel(EIP197_xDR_PREP_CLR_COUNT,
                       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT);

                /* Clear the pending processed counter */
                writel(EIP197_xDR_PROC_CLR_COUNT,
                       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT);

                writel(0,
                       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR);
                writel(0,
                       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR);

                writel((EIP197_DEFAULT_RING_SIZE * priv->config.cd_offset),
                       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_SIZE);
        }

        /* Result Descriptor Ring prepare */
        for (i = 0; i < priv->config.rings; i++) {
                /* Disable external triggering*/
                writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG);

                /* Clear the pending prepared counter */
                writel(EIP197_xDR_PREP_CLR_COUNT,
                       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT);

                /* Clear the pending processed counter */
                writel(EIP197_xDR_PROC_CLR_COUNT,
                       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT);

                writel(0,
                       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR);
                writel(0,
                       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR);

                /* Ring size */
                writel((EIP197_DEFAULT_RING_SIZE * priv->config.rd_offset),
                       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_SIZE);
        }

        for (pe = 0; pe < priv->config.pes; pe++) {
                /* Enable command descriptor rings */
                writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
                       EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));

                /* Enable result descriptor rings */
                writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
                       EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));
        }

        /* Clear any HIA interrupt */
        writel(GENMASK(30, 20), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK);

        if (priv->flags & EIP197_SIMPLE_TRC) {
                writel(EIP197_STRC_CONFIG_INIT |
                       EIP197_STRC_CONFIG_LARGE_REC(EIP197_CS_TRC_REC_WC) |
                       EIP197_STRC_CONFIG_SMALL_REC(EIP197_CS_TRC_REC_WC),
                       priv->base + EIP197_STRC_CONFIG);
                writel(EIP197_PE_EIP96_TOKEN_CTRL2_CTX_DONE,
                       EIP197_PE(priv) + EIP197_PE_EIP96_TOKEN_CTRL2(0));
        } else if (priv->flags & SAFEXCEL_HW_EIP197) {
                ret = eip197_trc_cache_init(priv);
                if (ret)
                        return ret;
        }

        if (priv->flags & EIP197_ICE) {
                ret = eip197_load_firmwares(priv);
                if (ret)
                        return ret;
        }

        return safexcel_hw_setup_cdesc_rings(priv) ?:
               safexcel_hw_setup_rdesc_rings(priv) ?:
               0;
}

/* Called with ring's lock taken */
static void safexcel_try_push_requests(struct safexcel_crypto_priv *priv,
                                       int ring)
{
        int coal = min_t(int, priv->ring[ring].requests, EIP197_MAX_BATCH_SZ);

        if (!coal)
                return;

        /* Configure when we want an interrupt */
        writel(EIP197_HIA_RDR_THRESH_PKT_MODE |
               EIP197_HIA_RDR_THRESH_PROC_PKT(coal),
               EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_THRESH);
}

void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring)
{
        struct crypto_async_request *req, *backlog;
        struct safexcel_context *ctx;
        int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results;

        /* If a request wasn't properly dequeued because of a lack of resources,
         * proceeded it first,
         */
        req = priv->ring[ring].req;
        backlog = priv->ring[ring].backlog;
        if (req)
                goto handle_req;

        while (true) {
                spin_lock_bh(&priv->ring[ring].queue_lock);
                backlog = crypto_get_backlog(&priv->ring[ring].queue);
                req = crypto_dequeue_request(&priv->ring[ring].queue);
                spin_unlock_bh(&priv->ring[ring].queue_lock);

                if (!req) {
                        priv->ring[ring].req = NULL;
                        priv->ring[ring].backlog = NULL;
                        goto finalize;
                }

handle_req:
                ctx = crypto_tfm_ctx(req->tfm);
                ret = ctx->send(req, ring, &commands, &results);
                if (ret)
                        goto request_failed;

                if (backlog)
                        crypto_request_complete(backlog, -EINPROGRESS);

                /* In case the send() helper did not issue any command to push
                 * to the engine because the input data was cached, continue to
                 * dequeue other requests as this is valid and not an error.
                 */
                if (!commands && !results)
                        continue;

                cdesc += commands;
                rdesc += results;
                nreq++;
        }

request_failed:
        /* Not enough resources to handle all the requests. Bail out and save
         * the request and the backlog for the next dequeue call (per-ring).
         */
        priv->ring[ring].req = req;
        priv->ring[ring].backlog = backlog;

finalize:
        if (!nreq)
                return;

        spin_lock_bh(&priv->ring[ring].lock);

        priv->ring[ring].requests += nreq;

        if (!priv->ring[ring].busy) {
                safexcel_try_push_requests(priv, ring);
                priv->ring[ring].busy = true;
        }

        spin_unlock_bh(&priv->ring[ring].lock);

        /* let the RDR know we have pending descriptors */
        writel((rdesc * priv->config.rd_offset),
               EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT);

        /* let the CDR know we have pending descriptors */
        writel((cdesc * priv->config.cd_offset),
               EIP197_HIA_CDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT);
}

inline int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv,
                                       void *rdp)
{
        struct safexcel_result_desc *rdesc = rdp;
        struct result_data_desc *result_data = rdp + priv->config.res_offset;

        if (likely((!rdesc->last_seg) || /* Rest only valid if last seg! */
                   ((!rdesc->descriptor_overflow) &&
                    (!rdesc->buffer_overflow) &&
                    (!result_data->error_code))))
                return 0;

        if (rdesc->descriptor_overflow)
                dev_err(priv->dev, "Descriptor overflow detected");

        if (rdesc->buffer_overflow)
                dev_err(priv->dev, "Buffer overflow detected");

        if (result_data->error_code & 0x4066) {
                /* Fatal error (bits 1,2,5,6 & 14) */
                dev_err(priv->dev,
                        "result descriptor error (%x)",
                        result_data->error_code);

                return -EIO;
        } else if (result_data->error_code &
                   (BIT(7) | BIT(4) | BIT(3) | BIT(0))) {
                /*
                 * Give priority over authentication fails:
                 * Blocksize, length & overflow errors,
                 * something wrong with the input!
                 */
                return -EINVAL;
        } else if (result_data->error_code & BIT(9)) {
                /* Authentication failed */
                return -EBADMSG;
        }

        /* All other non-fatal errors */
        return -EINVAL;
}

inline void safexcel_rdr_req_set(struct safexcel_crypto_priv *priv,
                                 int ring,
                                 struct safexcel_result_desc *rdesc,
                                 struct crypto_async_request *req)
{
        int i = safexcel_ring_rdr_rdesc_index(priv, ring, rdesc);

        priv->ring[ring].rdr_req[i] = req;
}

inline struct crypto_async_request *
safexcel_rdr_req_get(struct safexcel_crypto_priv *priv, int ring)
{
        int i = safexcel_ring_first_rdr_index(priv, ring);

        return priv->ring[ring].rdr_req[i];
}

void safexcel_complete(struct safexcel_crypto_priv *priv, int ring)
{
        struct safexcel_command_desc *cdesc;

        /* Acknowledge the command descriptors */
        do {
                cdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].cdr);
                if (IS_ERR(cdesc)) {
                        dev_err(priv->dev,
                                "Could not retrieve the command descriptor\n");
                        return;
                }
        } while (!cdesc->last_seg);
}

int safexcel_invalidate_cache(struct crypto_async_request *async,
                              struct safexcel_crypto_priv *priv,
                              dma_addr_t ctxr_dma, int ring)
{
        struct safexcel_command_desc *cdesc;
        struct safexcel_result_desc *rdesc;
        struct safexcel_token  *dmmy;
        int ret = 0;

        /* Prepare command descriptor */
        cdesc = safexcel_add_cdesc(priv, ring, true, true, 0, 0, 0, ctxr_dma,
                                   &dmmy);
        if (IS_ERR(cdesc))
                return PTR_ERR(cdesc);

        cdesc->control_data.type = EIP197_TYPE_EXTENDED;
        cdesc->control_data.options = 0;
        cdesc->control_data.context_lo &= ~EIP197_CONTEXT_SIZE_MASK;
        cdesc->control_data.control0 = CONTEXT_CONTROL_INV_TR;

        /* Prepare result descriptor */
        rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0);

        if (IS_ERR(rdesc)) {
                ret = PTR_ERR(rdesc);
                goto cdesc_rollback;
        }

        safexcel_rdr_req_set(priv, ring, rdesc, async);

        return ret;

cdesc_rollback:
        safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);

        return ret;
}

static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv *priv,
                                                     int ring)
{
        struct crypto_async_request *req;
        struct safexcel_context *ctx;
        int ret, i, nreq, ndesc, tot_descs, handled = 0;
        bool should_complete;

handle_results:
        tot_descs = 0;

        nreq = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT);
        nreq >>= EIP197_xDR_PROC_xD_PKT_OFFSET;
        nreq &= EIP197_xDR_PROC_xD_PKT_MASK;
        if (!nreq)
                goto requests_left;

        for (i = 0; i < nreq; i++) {
                req = safexcel_rdr_req_get(priv, ring);

                ctx = crypto_tfm_ctx(req->tfm);
                ndesc = ctx->handle_result(priv, ring, req,
                                           &should_complete, &ret);
                if (ndesc < 0) {
                        dev_err(priv->dev, "failed to handle result (%d)\n",
                                ndesc);
                        goto acknowledge;
                }

                if (should_complete) {
                        local_bh_disable();
                        crypto_request_complete(req, ret);
                        local_bh_enable();
                }

                tot_descs += ndesc;
                handled++;
        }

acknowledge:
        if (i)
                writel(EIP197_xDR_PROC_xD_PKT(i) |
                       (tot_descs * priv->config.rd_offset),
                       EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT);

        /* If the number of requests overflowed the counter, try to proceed more
         * requests.
         */
        if (nreq == EIP197_xDR_PROC_xD_PKT_MASK)
                goto handle_results;

requests_left:
        spin_lock_bh(&priv->ring[ring].lock);

        priv->ring[ring].requests -= handled;
        safexcel_try_push_requests(priv, ring);

        if (!priv->ring[ring].requests)
                priv->ring[ring].busy = false;

        spin_unlock_bh(&priv->ring[ring].lock);
}

static void safexcel_dequeue_work(struct work_struct *work)
{
        struct safexcel_work_data *data =
                        container_of(work, struct safexcel_work_data, work);

        safexcel_dequeue(data->priv, data->ring);
}

struct safexcel_ring_irq_data {
        struct safexcel_crypto_priv *priv;
        int ring;
};

static irqreturn_t safexcel_irq_ring(int irq, void *data)
{
        struct safexcel_ring_irq_data *irq_data = data;
        struct safexcel_crypto_priv *priv = irq_data->priv;
        int ring = irq_data->ring, rc = IRQ_NONE;
        u32 status, stat;

        status = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLED_STAT(ring));
        if (!status)
                return rc;

        /* RDR interrupts */
        if (status & EIP197_RDR_IRQ(ring)) {
                stat = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT);

                if (unlikely(stat & EIP197_xDR_ERR)) {
                        /*
                         * Fatal error, the RDR is unusable and must be
                         * reinitialized. This should not happen under
                         * normal circumstances.
                         */
                        dev_err(priv->dev, "RDR: fatal error.\n");
                } else if (likely(stat & EIP197_xDR_THRESH)) {
                        rc = IRQ_WAKE_THREAD;
                }

                /* ACK the interrupts */
                writel(stat & 0xff,
                       EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT);
        }

        /* ACK the interrupts */
        writel(status, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ACK(ring));

        return rc;
}

static irqreturn_t safexcel_irq_ring_thread(int irq, void *data)
{
        struct safexcel_ring_irq_data *irq_data = data;
        struct safexcel_crypto_priv *priv = irq_data->priv;
        int ring = irq_data->ring;

        safexcel_handle_result_descriptor(priv, ring);

        queue_work(priv->ring[ring].workqueue,
                   &priv->ring[ring].work_data.work);

        return IRQ_HANDLED;
}

static int safexcel_request_ring_irq(void *pdev, int irqid,
                                     int is_pci_dev,
                                     int ring_id,
                                     irq_handler_t handler,
                                     irq_handler_t threaded_handler,
                                     struct safexcel_ring_irq_data *ring_irq_priv)
{
        int ret, irq, cpu;
        struct device *dev;

        if (IS_ENABLED(CONFIG_PCI) && is_pci_dev) {
                struct pci_dev *pci_pdev = pdev;

                dev = &pci_pdev->dev;
                irq = pci_irq_vector(pci_pdev, irqid);
                if (irq < 0) {
                        dev_err(dev, "unable to get device MSI IRQ %d (err %d)\n",
                                irqid, irq);
                        return irq;
                }
        } else if (IS_ENABLED(CONFIG_OF)) {
                struct platform_device *plf_pdev = pdev;
                char irq_name[6] = {0}; /* "ringX\0" */

                snprintf(irq_name, 6, "ring%d", irqid);
                dev = &plf_pdev->dev;
                irq = platform_get_irq_byname(plf_pdev, irq_name);

                if (irq < 0)
                        return irq;
        } else {
                return -ENXIO;
        }

        ret = devm_request_threaded_irq(dev, irq, handler,
                                        threaded_handler, IRQF_ONESHOT,
                                        dev_name(dev), ring_irq_priv);
        if (ret) {
                dev_err(dev, "unable to request IRQ %d\n", irq);
                return ret;
        }

        /* Set affinity */
        cpu = cpumask_local_spread(ring_id, NUMA_NO_NODE);
        irq_set_affinity_hint(irq, get_cpu_mask(cpu));

        return irq;
}

static struct safexcel_alg_template *safexcel_algs[] = {
        &safexcel_alg_ecb_des,
        &safexcel_alg_cbc_des,
        &safexcel_alg_ecb_des3_ede,
        &safexcel_alg_cbc_des3_ede,
        &safexcel_alg_ecb_aes,
        &safexcel_alg_cbc_aes,
        &safexcel_alg_ctr_aes,
        &safexcel_alg_md5,
        &safexcel_alg_sha1,
        &safexcel_alg_sha224,
        &safexcel_alg_sha256,
        &safexcel_alg_sha384,
        &safexcel_alg_sha512,
        &safexcel_alg_hmac_md5,
        &safexcel_alg_hmac_sha1,
        &safexcel_alg_hmac_sha224,
        &safexcel_alg_hmac_sha256,
        &safexcel_alg_hmac_sha384,
        &safexcel_alg_hmac_sha512,
        &safexcel_alg_authenc_hmac_sha1_cbc_aes,
        &safexcel_alg_authenc_hmac_sha224_cbc_aes,
        &safexcel_alg_authenc_hmac_sha256_cbc_aes,
        &safexcel_alg_authenc_hmac_sha384_cbc_aes,
        &safexcel_alg_authenc_hmac_sha512_cbc_aes,
        &safexcel_alg_authenc_hmac_sha1_cbc_des3_ede,
        &safexcel_alg_authenc_hmac_sha1_ctr_aes,
        &safexcel_alg_authenc_hmac_sha224_ctr_aes,
        &safexcel_alg_authenc_hmac_sha256_ctr_aes,
        &safexcel_alg_authenc_hmac_sha384_ctr_aes,
        &safexcel_alg_authenc_hmac_sha512_ctr_aes,
        &safexcel_alg_xts_aes,
        &safexcel_alg_gcm,
        &safexcel_alg_ccm,
        &safexcel_alg_cbcmac,
        &safexcel_alg_xcbcmac,
        &safexcel_alg_cmac,
        &safexcel_alg_chacha20,
        &safexcel_alg_chachapoly,
        &safexcel_alg_chachapoly_esp,
        &safexcel_alg_sm3,
        &safexcel_alg_hmac_sm3,
        &safexcel_alg_ecb_sm4,
        &safexcel_alg_cbc_sm4,
        &safexcel_alg_ctr_sm4,
        &safexcel_alg_authenc_hmac_sha1_cbc_sm4,
        &safexcel_alg_authenc_hmac_sm3_cbc_sm4,
        &safexcel_alg_authenc_hmac_sha1_ctr_sm4,
        &safexcel_alg_authenc_hmac_sm3_ctr_sm4,
        &safexcel_alg_sha3_224,
        &safexcel_alg_sha3_256,
        &safexcel_alg_sha3_384,
        &safexcel_alg_sha3_512,
        &safexcel_alg_hmac_sha3_224,
        &safexcel_alg_hmac_sha3_256,
        &safexcel_alg_hmac_sha3_384,
        &safexcel_alg_hmac_sha3_512,
        &safexcel_alg_authenc_hmac_sha1_cbc_des,
        &safexcel_alg_authenc_hmac_sha256_cbc_des3_ede,
        &safexcel_alg_authenc_hmac_sha224_cbc_des3_ede,
        &safexcel_alg_authenc_hmac_sha512_cbc_des3_ede,
        &safexcel_alg_authenc_hmac_sha384_cbc_des3_ede,
        &safexcel_alg_authenc_hmac_sha256_cbc_des,
        &safexcel_alg_authenc_hmac_sha224_cbc_des,
        &safexcel_alg_authenc_hmac_sha512_cbc_des,
        &safexcel_alg_authenc_hmac_sha384_cbc_des,
        &safexcel_alg_rfc4106_gcm,
        &safexcel_alg_rfc4543_gcm,
        &safexcel_alg_rfc4309_ccm,
};

static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv)
{
        int i, j, ret = 0;

        for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
                safexcel_algs[i]->priv = priv;

                /* Do we have all required base algorithms available? */
                if ((safexcel_algs[i]->algo_mask & priv->hwconfig.algo_flags) !=
                    safexcel_algs[i]->algo_mask)
                        /* No, so don't register this ciphersuite */
                        continue;

                if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
                        ret = crypto_register_skcipher(&safexcel_algs[i]->alg.skcipher);
                else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD)
                        ret = crypto_register_aead(&safexcel_algs[i]->alg.aead);
                else
                        ret = crypto_register_ahash(&safexcel_algs[i]->alg.ahash);

                if (ret)
                        goto fail;
        }

        return 0;

fail:
        for (j = 0; j < i; j++) {
                /* Do we have all required base algorithms available? */
                if ((safexcel_algs[j]->algo_mask & priv->hwconfig.algo_flags) !=
                    safexcel_algs[j]->algo_mask)
                        /* No, so don't unregister this ciphersuite */
                        continue;

                if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
                        crypto_unregister_skcipher(&safexcel_algs[j]->alg.skcipher);
                else if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_AEAD)
                        crypto_unregister_aead(&safexcel_algs[j]->alg.aead);
                else
                        crypto_unregister_ahash(&safexcel_algs[j]->alg.ahash);
        }

        return ret;
}

static void safexcel_unregister_algorithms(struct safexcel_crypto_priv *priv)
{
        int i;

        for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
                /* Do we have all required base algorithms available? */
                if ((safexcel_algs[i]->algo_mask & priv->hwconfig.algo_flags) !=
                    safexcel_algs[i]->algo_mask)
                        /* No, so don't unregister this ciphersuite */
                        continue;

                if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
                        crypto_unregister_skcipher(&safexcel_algs[i]->alg.skcipher);
                else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD)
                        crypto_unregister_aead(&safexcel_algs[i]->alg.aead);
                else
                        crypto_unregister_ahash(&safexcel_algs[i]->alg.ahash);
        }
}

static void safexcel_configure(struct safexcel_crypto_priv *priv)
{
        u32 mask = BIT(priv->hwconfig.hwdataw) - 1;

        priv->config.pes = priv->hwconfig.hwnumpes;
        priv->config.rings = min_t(u32, priv->hwconfig.hwnumrings, max_rings);
        /* Cannot currently support more rings than we have ring AICs! */
        priv->config.rings = min_t(u32, priv->config.rings,
                                        priv->hwconfig.hwnumraic);

        priv->config.cd_size = EIP197_CD64_FETCH_SIZE;
        priv->config.cd_offset = (priv->config.cd_size + mask) & ~mask;
        priv->config.cdsh_offset = (EIP197_MAX_TOKENS + mask) & ~mask;

        /* res token is behind the descr, but ofs must be rounded to buswdth */
        priv->config.res_offset = (EIP197_RD64_FETCH_SIZE + mask) & ~mask;
        /* now the size of the descr is this 1st part plus the result struct */
        priv->config.rd_size    = priv->config.res_offset +
                                  EIP197_RD64_RESULT_SIZE;
        priv->config.rd_offset = (priv->config.rd_size + mask) & ~mask;

        /* convert dwords to bytes */
        priv->config.cd_offset *= sizeof(u32);
        priv->config.cdsh_offset *= sizeof(u32);
        priv->config.rd_offset *= sizeof(u32);
        priv->config.res_offset *= sizeof(u32);
}

static void safexcel_init_register_offsets(struct safexcel_crypto_priv *priv)
{
        struct safexcel_register_offsets *offsets = &priv->offsets;

        if (priv->flags & SAFEXCEL_HW_EIP197) {
                offsets->hia_aic        = EIP197_HIA_AIC_BASE;
                offsets->hia_aic_g      = EIP197_HIA_AIC_G_BASE;
                offsets->hia_aic_r      = EIP197_HIA_AIC_R_BASE;
                offsets->hia_aic_xdr    = EIP197_HIA_AIC_xDR_BASE;
                offsets->hia_dfe        = EIP197_HIA_DFE_BASE;
                offsets->hia_dfe_thr    = EIP197_HIA_DFE_THR_BASE;
                offsets->hia_dse        = EIP197_HIA_DSE_BASE;
                offsets->hia_dse_thr    = EIP197_HIA_DSE_THR_BASE;
                offsets->hia_gen_cfg    = EIP197_HIA_GEN_CFG_BASE;
                offsets->pe             = EIP197_PE_BASE;
                offsets->global         = EIP197_GLOBAL_BASE;
        } else {
                offsets->hia_aic        = EIP97_HIA_AIC_BASE;
                offsets->hia_aic_g      = EIP97_HIA_AIC_G_BASE;
                offsets->hia_aic_r      = EIP97_HIA_AIC_R_BASE;
                offsets->hia_aic_xdr    = EIP97_HIA_AIC_xDR_BASE;
                offsets->hia_dfe        = EIP97_HIA_DFE_BASE;
                offsets->hia_dfe_thr    = EIP97_HIA_DFE_THR_BASE;
                offsets->hia_dse        = EIP97_HIA_DSE_BASE;
                offsets->hia_dse_thr    = EIP97_HIA_DSE_THR_BASE;
                offsets->hia_gen_cfg    = EIP97_HIA_GEN_CFG_BASE;
                offsets->pe             = EIP97_PE_BASE;
                offsets->global         = EIP97_GLOBAL_BASE;
        }
}

/*
 * Generic part of probe routine, shared by platform and PCI driver
 *
 * Assumes IO resources have been mapped, private data mem has been allocated,
 * clocks have been enabled, device pointer has been assigned etc.
 *
 */
static int safexcel_probe_generic(void *pdev,
                                  struct safexcel_crypto_priv *priv,
                                  int is_pci_dev)
{
        struct device *dev = priv->dev;
        u32 peid, version, mask, val, hiaopt, hwopt, peopt;
        int i, ret, hwctg;

        priv->context_pool = dmam_pool_create("safexcel-context", dev,
                                              sizeof(struct safexcel_context_record),
                                              1, 0);
        if (!priv->context_pool)
                return -ENOMEM;

        /*
         * First try the EIP97 HIA version regs
         * For the EIP197, this is guaranteed to NOT return any of the test
         * values
         */
        version = readl(priv->base + EIP97_HIA_AIC_BASE + EIP197_HIA_VERSION);

        mask = 0;  /* do not swap */
        if (EIP197_REG_LO16(version) == EIP197_HIA_VERSION_LE) {
                priv->hwconfig.hiaver = EIP197_VERSION_MASK(version);
        } else if (EIP197_REG_HI16(version) == EIP197_HIA_VERSION_BE) {
                /* read back byte-swapped, so complement byte swap bits */
                mask = EIP197_MST_CTRL_BYTE_SWAP_BITS;
                priv->hwconfig.hiaver = EIP197_VERSION_SWAP(version);
        } else {
                /* So it wasn't an EIP97 ... maybe it's an EIP197? */
                version = readl(priv->base + EIP197_HIA_AIC_BASE +
                                EIP197_HIA_VERSION);
                if (EIP197_REG_LO16(version) == EIP197_HIA_VERSION_LE) {
                        priv->hwconfig.hiaver = EIP197_VERSION_MASK(version);
                        priv->flags |= SAFEXCEL_HW_EIP197;
                } else if (EIP197_REG_HI16(version) ==
                           EIP197_HIA_VERSION_BE) {
                        /* read back byte-swapped, so complement swap bits */
                        mask = EIP197_MST_CTRL_BYTE_SWAP_BITS;
                        priv->hwconfig.hiaver = EIP197_VERSION_SWAP(version);
                        priv->flags |= SAFEXCEL_HW_EIP197;
                } else {
                        return -ENODEV;
                }
        }

        /* Now initialize the reg offsets based on the probing info so far */
        safexcel_init_register_offsets(priv);

        /*
         * If the version was read byte-swapped, we need to flip the device
         * swapping Keep in mind here, though, that what we write will also be
         * byte-swapped ...
         */
        if (mask) {
                val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
                val = val ^ (mask >> 24); /* toggle byte swap bits */
                writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
        }

        /*
         * We're not done probing yet! We may fall through to here if no HIA
         * was found at all. So, with the endianness presumably correct now and
         * the offsets setup, *really* probe for the EIP97/EIP197.
         */
        version = readl(EIP197_GLOBAL(priv) + EIP197_VERSION);
        if (((priv->flags & SAFEXCEL_HW_EIP197) &&
             (EIP197_REG_LO16(version) != EIP197_VERSION_LE) &&
             (EIP197_REG_LO16(version) != EIP196_VERSION_LE)) ||
            ((!(priv->flags & SAFEXCEL_HW_EIP197) &&
             (EIP197_REG_LO16(version) != EIP97_VERSION_LE)))) {
                /*
                 * We did not find the device that matched our initial probing
                 * (or our initial probing failed) Report appropriate error.
                 */
                dev_err(priv->dev, "Probing for EIP97/EIP19x failed - no such device (read %08x)\n",
                        version);
                return -ENODEV;
        }

        priv->hwconfig.hwver = EIP197_VERSION_MASK(version);
        hwctg = version >> 28;
        peid = version & 255;

        /* Detect EIP206 processing pipe */
        version = readl(EIP197_PE(priv) + + EIP197_PE_VERSION(0));
        if (EIP197_REG_LO16(version) != EIP206_VERSION_LE) {
                dev_err(priv->dev, "EIP%d: EIP206 not detected\n", peid);
                return -ENODEV;
        }
        priv->hwconfig.ppver = EIP197_VERSION_MASK(version);

        /* Detect EIP96 packet engine and version */
        version = readl(EIP197_PE(priv) + EIP197_PE_EIP96_VERSION(0));
        if (EIP197_REG_LO16(version) != EIP96_VERSION_LE) {
                dev_err(dev, "EIP%d: EIP96 not detected.\n", peid);
                return -ENODEV;
        }
        priv->hwconfig.pever = EIP197_VERSION_MASK(version);

        hwopt = readl(EIP197_GLOBAL(priv) + EIP197_OPTIONS);
        hiaopt = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_OPTIONS);

        priv->hwconfig.icever = 0;
        priv->hwconfig.ocever = 0;
        priv->hwconfig.psever = 0;
        if (priv->flags & SAFEXCEL_HW_EIP197) {
                /* EIP197 */
                peopt = readl(EIP197_PE(priv) + EIP197_PE_OPTIONS(0));

                priv->hwconfig.hwdataw  = (hiaopt >> EIP197_HWDATAW_OFFSET) &
                                          EIP197_HWDATAW_MASK;
                priv->hwconfig.hwcfsize = ((hiaopt >> EIP197_CFSIZE_OFFSET) &
                                           EIP197_CFSIZE_MASK) +
                                          EIP197_CFSIZE_ADJUST;
                priv->hwconfig.hwrfsize = ((hiaopt >> EIP197_RFSIZE_OFFSET) &
                                           EIP197_RFSIZE_MASK) +
                                          EIP197_RFSIZE_ADJUST;
                priv->hwconfig.hwnumpes = (hiaopt >> EIP197_N_PES_OFFSET) &
                                          EIP197_N_PES_MASK;
                priv->hwconfig.hwnumrings = (hiaopt >> EIP197_N_RINGS_OFFSET) &
                                            EIP197_N_RINGS_MASK;
                if (hiaopt & EIP197_HIA_OPT_HAS_PE_ARB)
                        priv->flags |= EIP197_PE_ARB;
                if (EIP206_OPT_ICE_TYPE(peopt) == 1) {
                        priv->flags |= EIP197_ICE;
                        /* Detect ICE EIP207 class. engine and version */
                        version = readl(EIP197_PE(priv) +
                                  EIP197_PE_ICE_VERSION(0));
                        if (EIP197_REG_LO16(version) != EIP207_VERSION_LE) {
                                dev_err(dev, "EIP%d: ICE EIP207 not detected.\n",
                                        peid);
                                return -ENODEV;
                        }
                        priv->hwconfig.icever = EIP197_VERSION_MASK(version);
                }
                if (EIP206_OPT_OCE_TYPE(peopt) == 1) {
                        priv->flags |= EIP197_OCE;
                        /* Detect EIP96PP packet stream editor and version */
                        version = readl(EIP197_PE(priv) + EIP197_PE_PSE_VERSION(0));
                        if (EIP197_REG_LO16(version) != EIP96_VERSION_LE) {
                                dev_err(dev, "EIP%d: EIP96PP not detected.\n", peid);
                                return -ENODEV;
                        }
                        priv->hwconfig.psever = EIP197_VERSION_MASK(version);
                        /* Detect OCE EIP207 class. engine and version */
                        version = readl(EIP197_PE(priv) +
                                  EIP197_PE_ICE_VERSION(0));
                        if (EIP197_REG_LO16(version) != EIP207_VERSION_LE) {
                                dev_err(dev, "EIP%d: OCE EIP207 not detected.\n",
                                        peid);
                                return -ENODEV;
                        }
                        priv->hwconfig.ocever = EIP197_VERSION_MASK(version);
                }
                /* If not a full TRC, then assume simple TRC */
                if (!(hwopt & EIP197_OPT_HAS_TRC))
                        priv->flags |= EIP197_SIMPLE_TRC;
                /* EIP197 always has SOME form of TRC */
                priv->flags |= EIP197_TRC_CACHE;
        } else {
                /* EIP97 */
                priv->hwconfig.hwdataw  = (hiaopt >> EIP197_HWDATAW_OFFSET) &
                                          EIP97_HWDATAW_MASK;
                priv->hwconfig.hwcfsize = (hiaopt >> EIP97_CFSIZE_OFFSET) &
                                          EIP97_CFSIZE_MASK;
                priv->hwconfig.hwrfsize = (hiaopt >> EIP97_RFSIZE_OFFSET) &
                                          EIP97_RFSIZE_MASK;
                priv->hwconfig.hwnumpes = 1; /* by definition */
                priv->hwconfig.hwnumrings = (hiaopt >> EIP197_N_RINGS_OFFSET) &
                                            EIP197_N_RINGS_MASK;
        }

        /* Scan for ring AIC's */
        for (i = 0; i < EIP197_MAX_RING_AIC; i++) {
                version = readl(EIP197_HIA_AIC_R(priv) +
                                EIP197_HIA_AIC_R_VERSION(i));
                if (EIP197_REG_LO16(version) != EIP201_VERSION_LE)
                        break;
        }
        priv->hwconfig.hwnumraic = i;
        /* Low-end EIP196 may not have any ring AIC's ... */
        if (!priv->hwconfig.hwnumraic) {
                dev_err(priv->dev, "No ring interrupt controller present!\n");
                return -ENODEV;
        }

        /* Get supported algorithms from EIP96 transform engine */
        priv->hwconfig.algo_flags = readl(EIP197_PE(priv) +
                                    EIP197_PE_EIP96_OPTIONS(0));

        /* Print single info line describing what we just detected */
        dev_info(priv->dev, "EIP%d:%x(%d,%d,%d,%d)-HIA:%x(%d,%d,%d),PE:%x/%x(alg:%08x)/%x/%x/%x\n",
                 peid, priv->hwconfig.hwver, hwctg, priv->hwconfig.hwnumpes,
                 priv->hwconfig.hwnumrings, priv->hwconfig.hwnumraic,
                 priv->hwconfig.hiaver, priv->hwconfig.hwdataw,
                 priv->hwconfig.hwcfsize, priv->hwconfig.hwrfsize,
                 priv->hwconfig.ppver, priv->hwconfig.pever,
                 priv->hwconfig.algo_flags, priv->hwconfig.icever,
                 priv->hwconfig.ocever, priv->hwconfig.psever);

        safexcel_configure(priv);

        if (IS_ENABLED(CONFIG_PCI) && priv->data->version == EIP197_DEVBRD) {
                /*
                 * Request MSI vectors for global + 1 per ring -
                 * or just 1 for older dev images
                 */
                struct pci_dev *pci_pdev = pdev;

                ret = pci_alloc_irq_vectors(pci_pdev,
                                            priv->config.rings + 1,
                                            priv->config.rings + 1,
                                            PCI_IRQ_MSI | PCI_IRQ_MSIX);
                if (ret < 0) {
                        dev_err(dev, "Failed to allocate PCI MSI interrupts\n");
                        return ret;
                }
        }

        /* Register the ring IRQ handlers and configure the rings */
        priv->ring = devm_kcalloc(dev, priv->config.rings,
                                  sizeof(*priv->ring),
                                  GFP_KERNEL);
        if (!priv->ring)
                return -ENOMEM;

        for (i = 0; i < priv->config.rings; i++) {
                char wq_name[9] = {0};
                int irq;
                struct safexcel_ring_irq_data *ring_irq;

                ret = safexcel_init_ring_descriptors(priv,
                                                     &priv->ring[i].cdr,
                                                     &priv->ring[i].rdr);
                if (ret) {
                        dev_err(dev, "Failed to initialize rings\n");
                        goto err_cleanup_rings;
                }

                priv->ring[i].rdr_req = devm_kcalloc(dev,
                        EIP197_DEFAULT_RING_SIZE,
                        sizeof(*priv->ring[i].rdr_req),
                        GFP_KERNEL);
                if (!priv->ring[i].rdr_req) {
                        ret = -ENOMEM;
                        goto err_cleanup_rings;
                }

                ring_irq = devm_kzalloc(dev, sizeof(*ring_irq), GFP_KERNEL);
                if (!ring_irq) {
                        ret = -ENOMEM;
                        goto err_cleanup_rings;
                }

                ring_irq->priv = priv;
                ring_irq->ring = i;

                irq = safexcel_request_ring_irq(pdev,
                                                EIP197_IRQ_NUMBER(i, is_pci_dev),
                                                is_pci_dev,
                                                i,
                                                safexcel_irq_ring,
                                                safexcel_irq_ring_thread,
                                                ring_irq);
                if (irq < 0) {
                        dev_err(dev, "Failed to get IRQ ID for ring %d\n", i);
                        ret = irq;
                        goto err_cleanup_rings;
                }

                priv->ring[i].irq = irq;
                priv->ring[i].work_data.priv = priv;
                priv->ring[i].work_data.ring = i;
                INIT_WORK(&priv->ring[i].work_data.work,
                          safexcel_dequeue_work);

                snprintf(wq_name, 9, "wq_ring%d", i);
                priv->ring[i].workqueue =
                        create_singlethread_workqueue(wq_name);
                if (!priv->ring[i].workqueue) {
                        ret = -ENOMEM;
                        goto err_cleanup_rings;
                }

                priv->ring[i].requests = 0;
                priv->ring[i].busy = false;

                crypto_init_queue(&priv->ring[i].queue,
                                  EIP197_DEFAULT_RING_SIZE);

                spin_lock_init(&priv->ring[i].lock);
                spin_lock_init(&priv->ring[i].queue_lock);
        }

        atomic_set(&priv->ring_used, 0);

        ret = safexcel_hw_init(priv);
        if (ret) {
                dev_err(dev, "HW init failed (%d)\n", ret);
                goto err_cleanup_rings;
        }

        ret = safexcel_register_algorithms(priv);
        if (ret) {
                dev_err(dev, "Failed to register algorithms (%d)\n", ret);
                goto err_cleanup_rings;
        }

        return 0;

err_cleanup_rings:
        for (i = 0; i < priv->config.rings; i++) {
                if (priv->ring[i].irq)
                        irq_set_affinity_hint(priv->ring[i].irq, NULL);
                if (priv->ring[i].workqueue)
                        destroy_workqueue(priv->ring[i].workqueue);
        }

        return ret;
}

static void safexcel_hw_reset_rings(struct safexcel_crypto_priv *priv)
{
        int i;

        for (i = 0; i < priv->config.rings; i++) {
                /* clear any pending interrupt */
                writel(GENMASK(5, 0), EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT);
                writel(GENMASK(7, 0), EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT);

                /* Reset the CDR base address */
                writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
                writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);

                /* Reset the RDR base address */
                writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
                writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
        }
}

/* for Device Tree platform driver */

static int safexcel_probe(struct platform_device *pdev)
{
        struct device *dev = &pdev->dev;
        struct safexcel_crypto_priv *priv;
        int ret;

        priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
        if (!priv)
                return -ENOMEM;

        priv->dev = dev;
        priv->data = (struct safexcel_priv_data *)of_device_get_match_data(dev);

        platform_set_drvdata(pdev, priv);

        priv->base = devm_platform_ioremap_resource(pdev, 0);
        if (IS_ERR(priv->base)) {
                dev_err(dev, "failed to get resource\n");
                return PTR_ERR(priv->base);
        }

        priv->clk = devm_clk_get(&pdev->dev, NULL);
        ret = PTR_ERR_OR_ZERO(priv->clk);
        /* The clock isn't mandatory */
        if  (ret != -ENOENT) {
                if (ret)
                        return ret;

                ret = clk_prepare_enable(priv->clk);
                if (ret) {
                        dev_err(dev, "unable to enable clk (%d)\n", ret);
                        return ret;
                }
        }

        priv->reg_clk = devm_clk_get(&pdev->dev, "reg");
        ret = PTR_ERR_OR_ZERO(priv->reg_clk);
        /* The clock isn't mandatory */
        if  (ret != -ENOENT) {
                if (ret)
                        goto err_core_clk;

                ret = clk_prepare_enable(priv->reg_clk);
                if (ret) {
                        dev_err(dev, "unable to enable reg clk (%d)\n", ret);
                        goto err_core_clk;
                }
        }

        ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
        if (ret)
                goto err_reg_clk;

        /* Generic EIP97/EIP197 device probing */
        ret = safexcel_probe_generic(pdev, priv, 0);
        if (ret)
                goto err_reg_clk;

        return 0;

err_reg_clk:
        clk_disable_unprepare(priv->reg_clk);
err_core_clk:
        clk_disable_unprepare(priv->clk);
        return ret;
}

static void safexcel_remove(struct platform_device *pdev)
{
        struct safexcel_crypto_priv *priv = platform_get_drvdata(pdev);
        int i;

        safexcel_unregister_algorithms(priv);
        safexcel_hw_reset_rings(priv);

        clk_disable_unprepare(priv->reg_clk);
        clk_disable_unprepare(priv->clk);

        for (i = 0; i < priv->config.rings; i++) {
                irq_set_affinity_hint(priv->ring[i].irq, NULL);
                destroy_workqueue(priv->ring[i].workqueue);
        }
}

static const struct safexcel_priv_data eip97ies_mrvl_data = {
        .version = EIP97IES_MRVL,
};

static const struct safexcel_priv_data eip197b_mrvl_data = {
        .version = EIP197B_MRVL,
};

static const struct safexcel_priv_data eip197d_mrvl_data = {
        .version = EIP197D_MRVL,
};

static const struct safexcel_priv_data eip197_devbrd_data = {
        .version = EIP197_DEVBRD,
};

static const struct safexcel_priv_data eip197c_mxl_data = {
        .version = EIP197C_MXL,
        .fw_little_endian = true,
};

static const struct of_device_id safexcel_of_match_table[] = {
        {
                .compatible = "inside-secure,safexcel-eip97ies",
                .data = &eip97ies_mrvl_data,
        },
        {
                .compatible = "inside-secure,safexcel-eip197b",
                .data = &eip197b_mrvl_data,
        },
        {
                .compatible = "inside-secure,safexcel-eip197d",
                .data = &eip197d_mrvl_data,
        },
        {
                .compatible = "inside-secure,safexcel-eip197c-mxl",
                .data = &eip197c_mxl_data,
        },
        /* For backward compatibility and intended for generic use */
        {
                .compatible = "inside-secure,safexcel-eip97",
                .data = &eip97ies_mrvl_data,
        },
        {
                .compatible = "inside-secure,safexcel-eip197",
                .data = &eip197b_mrvl_data,
        },
        {},
};

MODULE_DEVICE_TABLE(of, safexcel_of_match_table);

static struct platform_driver  crypto_safexcel = {
        .probe          = safexcel_probe,
        .remove         = safexcel_remove,
        .driver         = {
                .name   = "crypto-safexcel",
                .of_match_table = safexcel_of_match_table,
        },
};

/* PCIE devices - i.e. Inside Secure development boards */

static int safexcel_pci_probe(struct pci_dev *pdev,
                               const struct pci_device_id *ent)
{
        struct device *dev = &pdev->dev;
        struct safexcel_crypto_priv *priv;
        void __iomem *pciebase;
        int rc;
        u32 val;

        dev_dbg(dev, "Probing PCIE device: vendor %04x, device %04x, subv %04x, subdev %04x, ctxt %lx\n",
                ent->vendor, ent->device, ent->subvendor,
                ent->subdevice, ent->driver_data);

        priv = kzalloc_obj(*priv);
        if (!priv)
                return -ENOMEM;

        priv->dev = dev;
        priv->data = (struct safexcel_priv_data *)ent->driver_data;

        pci_set_drvdata(pdev, priv);

        /* enable the device */
        rc = pcim_enable_device(pdev);
        if (rc) {
                dev_err(dev, "Failed to enable PCI device\n");
                return rc;
        }

        /* take ownership of PCI BAR0 */
        rc = pcim_iomap_regions(pdev, 1, "crypto_safexcel");
        if (rc) {
                dev_err(dev, "Failed to map IO region for BAR0\n");
                return rc;
        }
        priv->base = pcim_iomap_table(pdev)[0];

        if (priv->data->version == EIP197_DEVBRD) {
                dev_dbg(dev, "Device identified as FPGA based development board - applying HW reset\n");

                rc = pcim_iomap_regions(pdev, 4, "crypto_safexcel");
                if (rc) {
                        dev_err(dev, "Failed to map IO region for BAR4\n");
                        return rc;
                }

                pciebase = pcim_iomap_table(pdev)[2];
                val = readl(pciebase + EIP197_XLX_IRQ_BLOCK_ID_ADDR);
                if ((val >> 16) == EIP197_XLX_IRQ_BLOCK_ID_VALUE) {
                        dev_dbg(dev, "Detected Xilinx PCIE IRQ block version %d, multiple MSI support enabled\n",
                                (val & 0xff));

                        /* Setup MSI identity map mapping */
                        writel(EIP197_XLX_USER_VECT_LUT0_IDENT,
                               pciebase + EIP197_XLX_USER_VECT_LUT0_ADDR);
                        writel(EIP197_XLX_USER_VECT_LUT1_IDENT,
                               pciebase + EIP197_XLX_USER_VECT_LUT1_ADDR);
                        writel(EIP197_XLX_USER_VECT_LUT2_IDENT,
                               pciebase + EIP197_XLX_USER_VECT_LUT2_ADDR);
                        writel(EIP197_XLX_USER_VECT_LUT3_IDENT,
                               pciebase + EIP197_XLX_USER_VECT_LUT3_ADDR);

                        /* Enable all device interrupts */
                        writel(GENMASK(31, 0),
                               pciebase + EIP197_XLX_USER_INT_ENB_MSK);
                } else {
                        dev_err(dev, "Unrecognised IRQ block identifier %x\n",
                                val);
                        return -ENODEV;
                }

                /* HW reset FPGA dev board */
                /* assert reset */
                writel(1, priv->base + EIP197_XLX_GPIO_BASE);
                wmb(); /* maintain strict ordering for accesses here */
                /* deassert reset */
                writel(0, priv->base + EIP197_XLX_GPIO_BASE);
                wmb(); /* maintain strict ordering for accesses here */
        }

        /* enable bus mastering */
        pci_set_master(pdev);

        /* Generic EIP97/EIP197 device probing */
        rc = safexcel_probe_generic(pdev, priv, 1);
        return rc;
}

static void safexcel_pci_remove(struct pci_dev *pdev)
{
        struct safexcel_crypto_priv *priv = pci_get_drvdata(pdev);
        int i;

        safexcel_unregister_algorithms(priv);

        for (i = 0; i < priv->config.rings; i++)
                destroy_workqueue(priv->ring[i].workqueue);

        safexcel_hw_reset_rings(priv);
}

static const struct pci_device_id safexcel_pci_ids[] = {
        {
                PCI_DEVICE_SUB(PCI_VENDOR_ID_XILINX, 0x9038,
                               0x16ae, 0xc522),
                .driver_data = (kernel_ulong_t)&eip197_devbrd_data,
        },
        {},
};

MODULE_DEVICE_TABLE(pci, safexcel_pci_ids);

static struct pci_driver safexcel_pci_driver = {
        .name          = "crypto-safexcel",
        .id_table      = safexcel_pci_ids,
        .probe         = safexcel_pci_probe,
        .remove        = safexcel_pci_remove,
};

static int __init safexcel_init(void)
{
        int ret;

        /* Register PCI driver */
        ret = pci_register_driver(&safexcel_pci_driver);

        /* Register platform driver */
        if (IS_ENABLED(CONFIG_OF) && !ret) {
                ret = platform_driver_register(&crypto_safexcel);
                if (ret)
                        pci_unregister_driver(&safexcel_pci_driver);
        }

        return ret;
}

static void __exit safexcel_exit(void)
{
        /* Unregister platform driver */
        if (IS_ENABLED(CONFIG_OF))
                platform_driver_unregister(&crypto_safexcel);

        /* Unregister PCI driver if successfully registered before */
        pci_unregister_driver(&safexcel_pci_driver);
}

module_init(safexcel_init);
module_exit(safexcel_exit);

MODULE_AUTHOR("Antoine Tenart <antoine.tenart@free-electrons.com>");
MODULE_AUTHOR("Ofer Heifetz <oferh@marvell.com>");
MODULE_AUTHOR("Igal Liberman <igall@marvell.com>");
MODULE_DESCRIPTION("Support for SafeXcel cryptographic engines: EIP97 & EIP197");
MODULE_LICENSE("GPL v2");
MODULE_IMPORT_NS("CRYPTO_INTERNAL");

MODULE_FIRMWARE("ifpp.bin");
MODULE_FIRMWARE("ipue.bin");
MODULE_FIRMWARE("inside-secure/eip197b/ifpp.bin");
MODULE_FIRMWARE("inside-secure/eip197b/ipue.bin");
MODULE_FIRMWARE("inside-secure/eip197d/ifpp.bin");
MODULE_FIRMWARE("inside-secure/eip197d/ipue.bin");
MODULE_FIRMWARE("inside-secure/eip197_minifw/ifpp.bin");
MODULE_FIRMWARE("inside-secure/eip197_minifw/ipue.bin");