#include <crypto/internal/acompress.h>
#include <crypto/scatterwalk.h>
#include <linux/cryptouser.h>
#include <linux/cpumask.h>
#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/percpu.h>
#include <linux/scatterlist.h>
#include <linux/sched.h>
#include <linux/seq_file.h>
#include <linux/smp.h>
#include <linux/spinlock.h>
#include <linux/string.h>
#include <linux/workqueue.h>
#include <net/netlink.h>
#include "compress.h"
struct crypto_scomp;
enum {
ACOMP_WALK_SLEEP = 1 << 0,
ACOMP_WALK_SRC_LINEAR = 1 << 1,
ACOMP_WALK_DST_LINEAR = 1 << 2,
};
static const struct crypto_type crypto_acomp_type;
static void acomp_reqchain_done(void *data, int err);
static inline struct acomp_alg *__crypto_acomp_alg(struct crypto_alg *alg)
{
return container_of(alg, struct acomp_alg, calg.base);
}
static inline struct acomp_alg *crypto_acomp_alg(struct crypto_acomp *tfm)
{
return __crypto_acomp_alg(crypto_acomp_tfm(tfm)->__crt_alg);
}
static int __maybe_unused crypto_acomp_report(
struct sk_buff *skb, struct crypto_alg *alg)
{
struct crypto_report_acomp racomp;
memset(&racomp, 0, sizeof(racomp));
strscpy(racomp.type, "acomp", sizeof(racomp.type));
return nla_put(skb, CRYPTOCFGA_REPORT_ACOMP, sizeof(racomp), &racomp);
}
static void __maybe_unused crypto_acomp_show(struct seq_file *m,
struct crypto_alg *alg)
{
seq_puts(m, "type : acomp\n");
}
static void crypto_acomp_exit_tfm(struct crypto_tfm *tfm)
{
struct crypto_acomp *acomp = __crypto_acomp_tfm(tfm);
struct acomp_alg *alg = crypto_acomp_alg(acomp);
if (alg->exit)
alg->exit(acomp);
if (acomp_is_async(acomp))
crypto_free_acomp(crypto_acomp_fb(acomp));
}
static int crypto_acomp_init_tfm(struct crypto_tfm *tfm)
{
struct crypto_acomp *acomp = __crypto_acomp_tfm(tfm);
struct acomp_alg *alg = crypto_acomp_alg(acomp);
struct crypto_acomp *fb = NULL;
int err;
if (tfm->__crt_alg->cra_type != &crypto_acomp_type)
return crypto_init_scomp_ops_async(tfm);
if (acomp_is_async(acomp)) {
fb = crypto_alloc_acomp(crypto_acomp_alg_name(acomp), 0,
CRYPTO_ALG_ASYNC);
if (IS_ERR(fb))
return PTR_ERR(fb);
err = -EINVAL;
if (crypto_acomp_reqsize(fb) > MAX_SYNC_COMP_REQSIZE)
goto out_free_fb;
tfm->fb = crypto_acomp_tfm(fb);
}
acomp->compress = alg->compress;
acomp->decompress = alg->decompress;
acomp->reqsize = alg->base.cra_reqsize;
acomp->base.exit = crypto_acomp_exit_tfm;
if (!alg->init)
return 0;
err = alg->init(acomp);
if (err)
goto out_free_fb;
return 0;
out_free_fb:
crypto_free_acomp(fb);
return err;
}
static unsigned int crypto_acomp_extsize(struct crypto_alg *alg)
{
int extsize = crypto_alg_extsize(alg);
if (alg->cra_type != &crypto_acomp_type)
extsize += sizeof(struct crypto_scomp *);
return extsize;
}
static const struct crypto_type crypto_acomp_type = {
.extsize = crypto_acomp_extsize,
.init_tfm = crypto_acomp_init_tfm,
#ifdef CONFIG_PROC_FS
.show = crypto_acomp_show,
#endif
#if IS_ENABLED(CONFIG_CRYPTO_USER)
.report = crypto_acomp_report,
#endif
.maskclear = ~CRYPTO_ALG_TYPE_MASK,
.maskset = CRYPTO_ALG_TYPE_ACOMPRESS_MASK,
.type = CRYPTO_ALG_TYPE_ACOMPRESS,
.tfmsize = offsetof(struct crypto_acomp, base),
.algsize = offsetof(struct acomp_alg, base),
};
struct crypto_acomp *crypto_alloc_acomp(const char *alg_name, u32 type,
u32 mask)
{
return crypto_alloc_tfm(alg_name, &crypto_acomp_type, type, mask);
}
EXPORT_SYMBOL_GPL(crypto_alloc_acomp);
struct crypto_acomp *crypto_alloc_acomp_node(const char *alg_name, u32 type,
u32 mask, int node)
{
return crypto_alloc_tfm_node(alg_name, &crypto_acomp_type, type, mask,
node);
}
EXPORT_SYMBOL_GPL(crypto_alloc_acomp_node);
static void acomp_save_req(struct acomp_req *req, crypto_completion_t cplt)
{
struct acomp_req_chain *state = &req->chain;
state->compl = req->base.complete;
state->data = req->base.data;
req->base.complete = cplt;
req->base.data = state;
}
static void acomp_restore_req(struct acomp_req *req)
{
struct acomp_req_chain *state = req->base.data;
req->base.complete = state->compl;
req->base.data = state->data;
}
static void acomp_reqchain_virt(struct acomp_req *req)
{
struct acomp_req_chain *state = &req->chain;
unsigned int slen = req->slen;
unsigned int dlen = req->dlen;
if (state->flags & CRYPTO_ACOMP_REQ_SRC_VIRT)
acomp_request_set_src_dma(req, state->src, slen);
if (state->flags & CRYPTO_ACOMP_REQ_DST_VIRT)
acomp_request_set_dst_dma(req, state->dst, dlen);
}
static void acomp_virt_to_sg(struct acomp_req *req)
{
struct acomp_req_chain *state = &req->chain;
state->flags = req->base.flags & (CRYPTO_ACOMP_REQ_SRC_VIRT |
CRYPTO_ACOMP_REQ_DST_VIRT);
if (acomp_request_src_isvirt(req)) {
unsigned int slen = req->slen;
const u8 *svirt = req->svirt;
state->src = svirt;
sg_init_one(&state->ssg, svirt, slen);
acomp_request_set_src_sg(req, &state->ssg, slen);
}
if (acomp_request_dst_isvirt(req)) {
unsigned int dlen = req->dlen;
u8 *dvirt = req->dvirt;
state->dst = dvirt;
sg_init_one(&state->dsg, dvirt, dlen);
acomp_request_set_dst_sg(req, &state->dsg, dlen);
}
}
static int acomp_do_nondma(struct acomp_req *req, bool comp)
{
ACOMP_FBREQ_ON_STACK(fbreq, req);
int err;
if (comp)
err = crypto_acomp_compress(fbreq);
else
err = crypto_acomp_decompress(fbreq);
req->dlen = fbreq->dlen;
return err;
}
static int acomp_do_one_req(struct acomp_req *req, bool comp)
{
if (acomp_request_isnondma(req))
return acomp_do_nondma(req, comp);
acomp_virt_to_sg(req);
return comp ? crypto_acomp_reqtfm(req)->compress(req) :
crypto_acomp_reqtfm(req)->decompress(req);
}
static int acomp_reqchain_finish(struct acomp_req *req, int err)
{
acomp_reqchain_virt(req);
acomp_restore_req(req);
return err;
}
static void acomp_reqchain_done(void *data, int err)
{
struct acomp_req *req = data;
crypto_completion_t compl;
compl = req->chain.compl;
data = req->chain.data;
if (err == -EINPROGRESS)
goto notify;
err = acomp_reqchain_finish(req, err);
notify:
compl(data, err);
}
static int acomp_do_req_chain(struct acomp_req *req, bool comp)
{
int err;
acomp_save_req(req, acomp_reqchain_done);
err = acomp_do_one_req(req, comp);
if (err == -EBUSY || err == -EINPROGRESS)
return err;
return acomp_reqchain_finish(req, err);
}
int crypto_acomp_compress(struct acomp_req *req)
{
struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);
if (acomp_req_on_stack(req) && acomp_is_async(tfm))
return -EAGAIN;
if (crypto_acomp_req_virt(tfm) || acomp_request_issg(req))
return crypto_acomp_reqtfm(req)->compress(req);
return acomp_do_req_chain(req, true);
}
EXPORT_SYMBOL_GPL(crypto_acomp_compress);
int crypto_acomp_decompress(struct acomp_req *req)
{
struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);
if (acomp_req_on_stack(req) && acomp_is_async(tfm))
return -EAGAIN;
if (crypto_acomp_req_virt(tfm) || acomp_request_issg(req))
return crypto_acomp_reqtfm(req)->decompress(req);
return acomp_do_req_chain(req, false);
}
EXPORT_SYMBOL_GPL(crypto_acomp_decompress);
void comp_prepare_alg(struct comp_alg_common *alg)
{
struct crypto_alg *base = &alg->base;
base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK;
}
int crypto_register_acomp(struct acomp_alg *alg)
{
struct crypto_alg *base = &alg->calg.base;
comp_prepare_alg(&alg->calg);
base->cra_type = &crypto_acomp_type;
base->cra_flags |= CRYPTO_ALG_TYPE_ACOMPRESS;
return crypto_register_alg(base);
}
EXPORT_SYMBOL_GPL(crypto_register_acomp);
void crypto_unregister_acomp(struct acomp_alg *alg)
{
crypto_unregister_alg(&alg->base);
}
EXPORT_SYMBOL_GPL(crypto_unregister_acomp);
int crypto_register_acomps(struct acomp_alg *algs, int count)
{
int i, ret;
for (i = 0; i < count; i++) {
ret = crypto_register_acomp(&algs[i]);
if (ret) {
crypto_unregister_acomps(algs, i);
return ret;
}
}
return 0;
}
EXPORT_SYMBOL_GPL(crypto_register_acomps);
void crypto_unregister_acomps(struct acomp_alg *algs, int count)
{
int i;
for (i = count - 1; i >= 0; --i)
crypto_unregister_acomp(&algs[i]);
}
EXPORT_SYMBOL_GPL(crypto_unregister_acomps);
static void acomp_stream_workfn(struct work_struct *work)
{
struct crypto_acomp_streams *s =
container_of(work, struct crypto_acomp_streams, stream_work);
struct crypto_acomp_stream __percpu *streams = s->streams;
int cpu;
for_each_cpu(cpu, &s->stream_want) {
struct crypto_acomp_stream *ps;
void *ctx;
ps = per_cpu_ptr(streams, cpu);
if (ps->ctx)
continue;
ctx = s->alloc_ctx();
if (IS_ERR(ctx))
break;
spin_lock_bh(&ps->lock);
ps->ctx = ctx;
spin_unlock_bh(&ps->lock);
cpumask_clear_cpu(cpu, &s->stream_want);
}
}
void crypto_acomp_free_streams(struct crypto_acomp_streams *s)
{
struct crypto_acomp_stream __percpu *streams = s->streams;
void (*free_ctx)(void *);
int i;
s->streams = NULL;
if (!streams)
return;
cancel_work_sync(&s->stream_work);
free_ctx = s->free_ctx;
for_each_possible_cpu(i) {
struct crypto_acomp_stream *ps = per_cpu_ptr(streams, i);
if (!ps->ctx)
continue;
free_ctx(ps->ctx);
}
free_percpu(streams);
}
EXPORT_SYMBOL_GPL(crypto_acomp_free_streams);
int crypto_acomp_alloc_streams(struct crypto_acomp_streams *s)
{
struct crypto_acomp_stream __percpu *streams;
struct crypto_acomp_stream *ps;
unsigned int i;
void *ctx;
if (s->streams)
return 0;
streams = alloc_percpu(struct crypto_acomp_stream);
if (!streams)
return -ENOMEM;
ctx = s->alloc_ctx();
if (IS_ERR(ctx)) {
free_percpu(streams);
return PTR_ERR(ctx);
}
i = cpumask_first(cpu_possible_mask);
ps = per_cpu_ptr(streams, i);
ps->ctx = ctx;
for_each_possible_cpu(i) {
ps = per_cpu_ptr(streams, i);
spin_lock_init(&ps->lock);
}
s->streams = streams;
INIT_WORK(&s->stream_work, acomp_stream_workfn);
return 0;
}
EXPORT_SYMBOL_GPL(crypto_acomp_alloc_streams);
struct crypto_acomp_stream *_crypto_acomp_lock_stream_bh(
struct crypto_acomp_streams *s)
{
struct crypto_acomp_stream __percpu *streams = s->streams;
int cpu = raw_smp_processor_id();
struct crypto_acomp_stream *ps;
ps = per_cpu_ptr(streams, cpu);
spin_lock_bh(&ps->lock);
if (likely(ps->ctx))
return ps;
spin_unlock(&ps->lock);
cpumask_set_cpu(cpu, &s->stream_want);
schedule_work(&s->stream_work);
ps = per_cpu_ptr(streams, cpumask_first(cpu_possible_mask));
spin_lock(&ps->lock);
return ps;
}
EXPORT_SYMBOL_GPL(_crypto_acomp_lock_stream_bh);
void acomp_walk_done_src(struct acomp_walk *walk, int used)
{
walk->slen -= used;
if ((walk->flags & ACOMP_WALK_SRC_LINEAR))
scatterwalk_advance(&walk->in, used);
else
scatterwalk_done_src(&walk->in, used);
if ((walk->flags & ACOMP_WALK_SLEEP))
cond_resched();
}
EXPORT_SYMBOL_GPL(acomp_walk_done_src);
void acomp_walk_done_dst(struct acomp_walk *walk, int used)
{
walk->dlen -= used;
if ((walk->flags & ACOMP_WALK_DST_LINEAR))
scatterwalk_advance(&walk->out, used);
else
scatterwalk_done_dst(&walk->out, used);
if ((walk->flags & ACOMP_WALK_SLEEP))
cond_resched();
}
EXPORT_SYMBOL_GPL(acomp_walk_done_dst);
int acomp_walk_next_src(struct acomp_walk *walk)
{
unsigned int slen = walk->slen;
unsigned int max = UINT_MAX;
if (!preempt_model_preemptible() && (walk->flags & ACOMP_WALK_SLEEP))
max = PAGE_SIZE;
if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) {
walk->in.__addr = (void *)(((u8 *)walk->in.sg) +
walk->in.offset);
return min(slen, max);
}
return slen ? scatterwalk_next(&walk->in, slen) : 0;
}
EXPORT_SYMBOL_GPL(acomp_walk_next_src);
int acomp_walk_next_dst(struct acomp_walk *walk)
{
unsigned int dlen = walk->dlen;
unsigned int max = UINT_MAX;
if (!preempt_model_preemptible() && (walk->flags & ACOMP_WALK_SLEEP))
max = PAGE_SIZE;
if ((walk->flags & ACOMP_WALK_DST_LINEAR)) {
walk->out.__addr = (void *)(((u8 *)walk->out.sg) +
walk->out.offset);
return min(dlen, max);
}
return dlen ? scatterwalk_next(&walk->out, dlen) : 0;
}
EXPORT_SYMBOL_GPL(acomp_walk_next_dst);
int acomp_walk_virt(struct acomp_walk *__restrict walk,
struct acomp_req *__restrict req, bool atomic)
{
struct scatterlist *src = req->src;
struct scatterlist *dst = req->dst;
walk->slen = req->slen;
walk->dlen = req->dlen;
if (!walk->slen || !walk->dlen)
return -EINVAL;
walk->flags = 0;
if ((req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) && !atomic)
walk->flags |= ACOMP_WALK_SLEEP;
if ((req->base.flags & CRYPTO_ACOMP_REQ_SRC_VIRT))
walk->flags |= ACOMP_WALK_SRC_LINEAR;
if ((req->base.flags & CRYPTO_ACOMP_REQ_DST_VIRT))
walk->flags |= ACOMP_WALK_DST_LINEAR;
if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) {
walk->in.sg = (void *)req->svirt;
walk->in.offset = 0;
} else
scatterwalk_start(&walk->in, src);
if ((walk->flags & ACOMP_WALK_DST_LINEAR)) {
walk->out.sg = (void *)req->dvirt;
walk->out.offset = 0;
} else
scatterwalk_start(&walk->out, dst);
return 0;
}
EXPORT_SYMBOL_GPL(acomp_walk_virt);
struct acomp_req *acomp_request_clone(struct acomp_req *req,
size_t total, gfp_t gfp)
{
struct acomp_req *nreq;
nreq = container_of(crypto_request_clone(&req->base, total, gfp),
struct acomp_req, base);
if (nreq == req)
return req;
if (req->src == &req->chain.ssg)
nreq->src = &nreq->chain.ssg;
if (req->dst == &req->chain.dsg)
nreq->dst = &nreq->chain.dsg;
return nreq;
}
EXPORT_SYMBOL_GPL(acomp_request_clone);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Asynchronous compression type");
