#include <sys/param.h>
#include <sys/event.h>
#include <sys/ioctl.h>
#include <sys/malloc.h>
#include <sys/timeout.h>
#include <sys/atomic.h>
#include <sys/task.h>
#include <sys/msgbuf.h>
#include <sys/mount.h>
#include <sys/syscallargs.h>
#include <sys/syslimits.h>
#include <crypto/sha2.h>
#define KEYSTREAM_ONLY
#include <crypto/chacha_private.h>
#include <uvm/uvm_extern.h>
#define POOLWORDS 2048
#define POOLBYTES (POOLWORDS*4)
#define POOLMASK (POOLWORDS - 1)
#define POOL_TAP1 1638
#define POOL_TAP2 1231
#define POOL_TAP3 819
#define POOL_TAP4 411
#define QEVLEN 128
#define QEVCONSUME 8
#define KEYSZ 32
#define IVSZ 8
#define BLOCKSZ 64
#define RSBUFSZ (16*BLOCKSZ)
#define EBUFSIZE KEYSZ + IVSZ
struct rand_event {
u_int re_time;
u_int re_val;
} rnd_event_space[QEVLEN];
u_int rnd_event_cons;
u_int rnd_event_prod;
int rnd_cold = 1;
int rnd_slowextract = 1;
void rnd_reinit(void *v);
void rnd_init(void *);
static u_int32_t entropy_pool[POOLWORDS];
u_int32_t entropy_pool0[POOLWORDS] __attribute__((section(".openbsd.randomdata")));
void dequeue_randomness(void *);
void add_entropy_words(const u_int32_t *, u_int);
void extract_entropy(u_int8_t *)
__attribute__((__bounded__(__minbytes__,1,EBUFSIZE)));
struct timeout rnd_timeout = TIMEOUT_INITIALIZER(dequeue_randomness, NULL);
int filt_randomread(struct knote *, long);
void filt_randomdetach(struct knote *);
int filt_randomwrite(struct knote *, long);
static void _rs_seed(u_char *, size_t);
static void _rs_clearseed(const void *p, size_t s);
const struct filterops randomread_filtops = {
.f_flags = FILTEROP_ISFD,
.f_attach = NULL,
.f_detach = filt_randomdetach,
.f_event = filt_randomread,
};
const struct filterops randomwrite_filtops = {
.f_flags = FILTEROP_ISFD,
.f_attach = NULL,
.f_detach = filt_randomdetach,
.f_event = filt_randomwrite,
};
static void
add_event_data(u_int val)
{
struct rand_event *rep;
int e;
e = (atomic_inc_int_nv(&rnd_event_prod) - 1) & (QEVLEN-1);
rep = &rnd_event_space[e];
rep->re_time += cpu_rnd_messybits();
rep->re_val += val;
}
void
enqueue_randomness(u_int val)
{
add_event_data(val);
if (rnd_cold) {
dequeue_randomness(NULL);
rnd_init(NULL);
if (!cold)
rnd_cold = 0;
} else if (!timeout_pending(&rnd_timeout) &&
(rnd_event_prod - rnd_event_cons) > QEVCONSUME) {
rnd_slowextract = min(rnd_slowextract * 2, 5000);
timeout_add_msec(&rnd_timeout, rnd_slowextract * 10);
}
}
void
add_entropy_words(const u_int32_t *buf, u_int n)
{
static const u_int32_t twist_table[8] = {
0x00000000, 0x3b6e20c8, 0x76dc4190, 0x4db26158,
0xedb88320, 0xd6d6a3e8, 0x9b64c2b0, 0xa00ae278
};
static u_int entropy_add_ptr;
static u_char entropy_input_rotate;
for (; n--; buf++) {
u_int32_t w = (*buf << entropy_input_rotate) |
(*buf >> ((32 - entropy_input_rotate) & 31));
u_int i = entropy_add_ptr =
(entropy_add_ptr - 1) & POOLMASK;
entropy_input_rotate =
(entropy_input_rotate + (i ? 7 : 14)) & 31;
w ^= entropy_pool[(i + POOL_TAP1) & POOLMASK] ^
entropy_pool[(i + POOL_TAP2) & POOLMASK] ^
entropy_pool[(i + POOL_TAP3) & POOLMASK] ^
entropy_pool[(i + POOL_TAP4) & POOLMASK] ^
entropy_pool[(i + 1) & POOLMASK] ^
entropy_pool[i];
entropy_pool[i] = (w >> 3) ^ twist_table[w & 7];
}
}
void
dequeue_randomness(void *v)
{
u_int32_t buf[2];
u_int startp, startc, i;
startp = rnd_event_prod - QEVCONSUME;
for (i = 0; i < QEVCONSUME; i++) {
u_int e = (startp + i) & (QEVLEN-1);
buf[0] = rnd_event_space[e].re_time;
buf[1] = rnd_event_space[e].re_val;
add_entropy_words(buf, 2);
}
startc = atomic_add_int_nv(&rnd_event_cons, QEVCONSUME) - QEVCONSUME;
for (i = 0; i < QEVCONSUME; i++) {
u_int e = (startc + i) & (QEVLEN-1);
buf[0] = rnd_event_space[e].re_time;
buf[1] = rnd_event_space[e].re_val;
add_entropy_words(buf, 2);
}
}
void
extract_entropy(u_int8_t *buf)
{
static u_int32_t extract_pool[POOLWORDS];
u_char digest[SHA512_DIGEST_LENGTH];
SHA2_CTX shactx;
#if SHA512_DIGEST_LENGTH < EBUFSIZE
#error "need more bigger hash output"
#endif
memcpy(extract_pool, entropy_pool, sizeof(extract_pool));
SHA512Init(&shactx);
SHA512Update(&shactx, (u_int8_t *)extract_pool, sizeof(extract_pool));
SHA512Final(digest, &shactx);
memcpy(buf, digest, EBUFSIZE);
add_event_data(extract_pool[0]);
dequeue_randomness(NULL);
explicit_bzero(extract_pool, sizeof(extract_pool));
explicit_bzero(digest, sizeof(digest));
}
struct mutex rndlock = MUTEX_INITIALIZER(IPL_HIGH);
struct timeout rndreinit_timeout = TIMEOUT_INITIALIZER(rnd_reinit, NULL);
struct task rnd_task = TASK_INITIALIZER(rnd_init, NULL);
static chacha_ctx rs;
static u_char rs_buf[RSBUFSZ];
u_char rs_buf0[RSBUFSZ] __attribute__((section(".openbsd.randomdata")));
static size_t rs_have;
static size_t rs_count;
void
suspend_randomness(void)
{
struct timespec ts;
getnanotime(&ts);
enqueue_randomness(ts.tv_sec);
enqueue_randomness(ts.tv_nsec);
dequeue_randomness(NULL);
rs_count = 0;
arc4random_buf(entropy_pool, sizeof(entropy_pool));
}
void
resume_randomness(char *buf, size_t buflen)
{
struct timespec ts;
if (buf && buflen)
_rs_seed(buf, buflen);
getnanotime(&ts);
enqueue_randomness(ts.tv_sec);
enqueue_randomness(ts.tv_nsec);
dequeue_randomness(NULL);
rs_count = 0;
}
static inline void _rs_rekey(u_char *dat, size_t datlen);
static inline void
_rs_init(u_char *buf, size_t n)
{
KASSERT(n >= KEYSZ + IVSZ);
chacha_keysetup(&rs, buf, KEYSZ * 8);
chacha_ivsetup(&rs, buf + KEYSZ, NULL);
}
static void
_rs_seed(u_char *buf, size_t n)
{
_rs_rekey(buf, n);
rs_have = 0;
memset(rs_buf, 0, sizeof(rs_buf));
rs_count = 1600000;
}
static void
_rs_stir(int do_lock)
{
struct timespec ts;
u_int8_t buf[EBUFSIZE], *p;
int i;
extract_entropy(buf);
nanotime(&ts);
for (p = (u_int8_t *)&ts, i = 0; i < sizeof(ts); i++)
buf[i] ^= p[i];
if (do_lock)
mtx_enter(&rndlock);
_rs_seed(buf, sizeof(buf));
if (do_lock)
mtx_leave(&rndlock);
explicit_bzero(buf, sizeof(buf));
rnd_slowextract = 1;
}
static inline void
_rs_stir_if_needed(size_t len)
{
static int rs_initialized;
if (!rs_initialized) {
memcpy(entropy_pool, entropy_pool0, sizeof(entropy_pool));
memcpy(rs_buf, rs_buf0, sizeof(rs_buf));
_rs_init(rs_buf, KEYSZ + IVSZ);
rs_count = 1024 * 1024 * 1024;
rs_initialized = 1;
} else if (rs_count <= len)
_rs_stir(0);
else
rs_count -= len;
}
static void
_rs_clearseed(const void *p, size_t s)
{
struct kmem_dyn_mode kd_avoidalias;
vaddr_t va = trunc_page((vaddr_t)p);
vsize_t off = (vaddr_t)p - va;
vsize_t len;
vaddr_t rwva;
paddr_t pa;
while (s > 0) {
pmap_extract(pmap_kernel(), va, &pa);
memset(&kd_avoidalias, 0, sizeof(kd_avoidalias));
kd_avoidalias.kd_prefer = pa;
kd_avoidalias.kd_waitok = 1;
rwva = (vaddr_t)km_alloc(PAGE_SIZE, &kv_any, &kp_none,
&kd_avoidalias);
if (!rwva)
panic("_rs_clearseed");
pmap_kenter_pa(rwva, pa, PROT_READ | PROT_WRITE);
pmap_update(pmap_kernel());
len = MIN(s, PAGE_SIZE - off);
explicit_bzero((void *)(rwva + off), len);
pmap_kremove(rwva, PAGE_SIZE);
km_free((void *)rwva, PAGE_SIZE, &kv_any, &kp_none);
va += PAGE_SIZE;
s -= len;
off = 0;
}
}
static inline void
_rs_rekey(u_char *dat, size_t datlen)
{
#ifndef KEYSTREAM_ONLY
memset(rs_buf, 0, sizeof(rs_buf));
#endif
chacha_encrypt_bytes(&rs, rs_buf, rs_buf, sizeof(rs_buf));
if (dat) {
size_t i, m;
m = MIN(datlen, KEYSZ + IVSZ);
for (i = 0; i < m; i++)
rs_buf[i] ^= dat[i];
}
_rs_init(rs_buf, KEYSZ + IVSZ);
memset(rs_buf, 0, KEYSZ + IVSZ);
rs_have = sizeof(rs_buf) - KEYSZ - IVSZ;
}
static inline void
_rs_random_buf(void *_buf, size_t n)
{
u_char *buf = (u_char *)_buf;
size_t m;
_rs_stir_if_needed(n);
while (n > 0) {
if (rs_have > 0) {
m = MIN(n, rs_have);
memcpy(buf, rs_buf + sizeof(rs_buf) - rs_have, m);
memset(rs_buf + sizeof(rs_buf) - rs_have, 0, m);
buf += m;
n -= m;
rs_have -= m;
}
if (rs_have == 0)
_rs_rekey(NULL, 0);
}
}
static inline void
_rs_random_u32(u_int32_t *val)
{
_rs_stir_if_needed(sizeof(*val));
if (rs_have < sizeof(*val))
_rs_rekey(NULL, 0);
memcpy(val, rs_buf + sizeof(rs_buf) - rs_have, sizeof(*val));
memset(rs_buf + sizeof(rs_buf) - rs_have, 0, sizeof(*val));
rs_have -= sizeof(*val);
}
u_int32_t
arc4random(void)
{
u_int32_t ret;
mtx_enter(&rndlock);
_rs_random_u32(&ret);
mtx_leave(&rndlock);
return ret;
}
void
arc4random_buf(void *buf, size_t n)
{
mtx_enter(&rndlock);
_rs_random_buf(buf, n);
mtx_leave(&rndlock);
}
struct arc4random_ctx *
arc4random_ctx_new(void)
{
char keybuf[KEYSZ + IVSZ];
chacha_ctx *ctx = malloc(sizeof(chacha_ctx), M_TEMP, M_WAITOK);
arc4random_buf(keybuf, KEYSZ + IVSZ);
chacha_keysetup(ctx, keybuf, KEYSZ * 8);
chacha_ivsetup(ctx, keybuf + KEYSZ, NULL);
explicit_bzero(keybuf, sizeof(keybuf));
return (struct arc4random_ctx *)ctx;
}
void
arc4random_ctx_free(struct arc4random_ctx *ctx)
{
explicit_bzero(ctx, sizeof(chacha_ctx));
free(ctx, M_TEMP, sizeof(chacha_ctx));
}
void
arc4random_ctx_buf(struct arc4random_ctx *ctx, void *buf, size_t n)
{
#ifndef KEYSTREAM_ONLY
memset(buf, 0, n);
#endif
chacha_encrypt_bytes((chacha_ctx *)ctx, buf, buf, n);
}
u_int32_t
arc4random_uniform(u_int32_t upper_bound)
{
u_int32_t r, min;
if (upper_bound < 2)
return 0;
min = -upper_bound % upper_bound;
for (;;) {
r = arc4random();
if (r >= min)
break;
}
return r % upper_bound;
}
void
rnd_init(void *null)
{
_rs_stir(1);
}
void
rnd_reinit(void *v)
{
task_add(systq, &rnd_task);
timeout_add_sec(&rndreinit_timeout, 10 * 60);
}
void
random_start(int goodseed)
{
extern char etext[];
#if !defined(NO_PROPOLICE)
extern long __guard_local;
if (__guard_local == 0)
printf("warning: no entropy supplied by boot loader\n");
#endif
_rs_clearseed(entropy_pool0, sizeof(entropy_pool0));
_rs_clearseed(rs_buf0, sizeof(rs_buf0));
if (msgbufp->msg_magic == MSG_MAGIC)
add_entropy_words((u_int32_t *)msgbufp->msg_bufc,
msgbufp->msg_bufs / sizeof(u_int32_t));
add_entropy_words((u_int32_t *)etext - 32*1024,
8192/sizeof(u_int32_t));
dequeue_randomness(NULL);
rnd_init(NULL);
rnd_reinit(NULL);
if (goodseed)
printf("random: good seed from bootblocks\n");
else {
printf("random: boothowto does not indicate good seed\n");
}
}
int
randomopen(dev_t dev, int flag, int mode, struct proc *p)
{
return 0;
}
int
randomclose(dev_t dev, int flag, int mode, struct proc *p)
{
return 0;
}
#define RND_MAIN_MAX_BYTES 2048
int
randomread(dev_t dev, struct uio *uio, int ioflag)
{
struct arc4random_ctx *lctx = NULL;
size_t total = uio->uio_resid;
u_char *buf;
int ret = 0;
if (uio->uio_resid == 0)
return 0;
buf = malloc(POOLBYTES, M_TEMP, M_WAITOK);
if (total > RND_MAIN_MAX_BYTES)
lctx = arc4random_ctx_new();
while (ret == 0 && uio->uio_resid > 0) {
size_t n = ulmin(POOLBYTES, uio->uio_resid);
if (lctx != NULL)
arc4random_ctx_buf(lctx, buf, n);
else
arc4random_buf(buf, n);
ret = uiomove(buf, n, uio);
if (ret == 0 && uio->uio_resid > 0)
yield();
}
if (lctx != NULL)
arc4random_ctx_free(lctx);
explicit_bzero(buf, POOLBYTES);
free(buf, M_TEMP, POOLBYTES);
return ret;
}
int
randomwrite(dev_t dev, struct uio *uio, int flags)
{
int ret = 0, newdata = 0;
u_int32_t *buf;
if (uio->uio_resid == 0)
return 0;
buf = malloc(POOLBYTES, M_TEMP, M_WAITOK);
while (ret == 0 && uio->uio_resid > 0) {
size_t n = ulmin(POOLBYTES, uio->uio_resid);
ret = uiomove(buf, n, uio);
if (ret != 0)
break;
while (n % sizeof(u_int32_t))
((u_int8_t *)buf)[n++] = 0;
add_entropy_words(buf, n / 4);
if (uio->uio_resid > 0)
yield();
newdata = 1;
}
if (newdata)
rnd_init(NULL);
explicit_bzero(buf, POOLBYTES);
free(buf, M_TEMP, POOLBYTES);
return ret;
}
int
randomkqfilter(dev_t dev, struct knote *kn)
{
switch (kn->kn_filter) {
case EVFILT_READ:
kn->kn_fop = &randomread_filtops;
break;
case EVFILT_WRITE:
kn->kn_fop = &randomwrite_filtops;
break;
default:
return (EINVAL);
}
return (0);
}
void
filt_randomdetach(struct knote *kn)
{
}
int
filt_randomread(struct knote *kn, long hint)
{
kn->kn_data = RND_MAIN_MAX_BYTES;
return (1);
}
int
filt_randomwrite(struct knote *kn, long hint)
{
kn->kn_data = POOLBYTES;
return (1);
}
int
randomioctl(dev_t dev, u_long cmd, caddr_t data, int flag, struct proc *p)
{
switch (cmd) {
case FIOASYNC:
break;
default:
return ENOTTY;
}
return 0;
}
int
sys_getentropy(struct proc *p, void *v, register_t *retval)
{
struct sys_getentropy_args
*uap = v;
char buf[GETENTROPY_MAX];
int error;
if (SCARG(uap, nbyte) > sizeof(buf))
return (EINVAL);
arc4random_buf(buf, SCARG(uap, nbyte));
if ((error = copyout(buf, SCARG(uap, buf), SCARG(uap, nbyte))) != 0)
return (error);
explicit_bzero(buf, sizeof(buf));
*retval = 0;
return (0);
}