/*-
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Copyright (c) 2023 Chelsio Communications, Inc.
 * Written by: John Baldwin <jhb@FreeBSD.org>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include <sys/param.h>
#include <sys/mbuf.h>
#include <sys/memdesc.h>
#include <sys/systm.h>
#include <sys/uio.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <vm/vm_page.h>
#include <vm/vm_param.h>
#include <machine/bus.h>

/*
 * memdesc_copyback copies data from a source buffer into a buffer
 * described by a memory descriptor.
 */
static void
phys_copyback(vm_paddr_t pa, int off, int size, const void *src)
{
        const char *cp;
        u_int page_off;
        int todo;
        void *p;

        KASSERT(PMAP_HAS_DMAP, ("direct-map required"));

        cp = src;
        pa += off;
        page_off = pa & PAGE_MASK;
        while (size > 0) {
                todo = min(PAGE_SIZE - page_off, size);
                p = PHYS_TO_DMAP(pa);
                memcpy(p, cp, todo);
                size -= todo;
                cp += todo;
                pa += todo;
                page_off = 0;
        }
}

static void
vlist_copyback(struct bus_dma_segment *vlist, int sglist_cnt, int off,
    int size, const void *src)
{
        const char *p;
        int todo;

        while (vlist->ds_len <= off) {
                KASSERT(sglist_cnt > 1, ("out of sglist entries"));

                off -= vlist->ds_len;
                vlist++;
                sglist_cnt--;
        }

        p = src;
        while (size > 0) {
                KASSERT(sglist_cnt >= 1, ("out of sglist entries"));

                todo = size;
                if (todo > vlist->ds_len - off)
                        todo = vlist->ds_len - off;

                memcpy((char *)(uintptr_t)vlist->ds_addr + off, p, todo);
                off = 0;
                vlist++;
                sglist_cnt--;
                size -= todo;
                p += todo;
        }
}

static void
plist_copyback(struct bus_dma_segment *plist, int sglist_cnt, int off,
    int size, const void *src)
{
        const char *p;
        int todo;

        while (plist->ds_len <= off) {
                KASSERT(sglist_cnt > 1, ("out of sglist entries"));

                off -= plist->ds_len;
                plist++;
                sglist_cnt--;
        }

        p = src;
        while (size > 0) {
                KASSERT(sglist_cnt >= 1, ("out of sglist entries"));

                todo = size;
                if (todo > plist->ds_len - off)
                        todo = plist->ds_len - off;

                phys_copyback(plist->ds_addr, off, todo, p);
                off = 0;
                plist++;
                sglist_cnt--;
                size -= todo;
                p += todo;
        }
}

static void
vmpages_copyback(vm_page_t *m, int off, int size, const void *src)
{
        struct iovec iov[1];
        struct uio uio;
        int error __diagused;

        iov[0].iov_base = __DECONST(void *, src);
        iov[0].iov_len = size;
        uio.uio_iov = iov;
        uio.uio_iovcnt = 1;
        uio.uio_offset = 0;
        uio.uio_resid = size;
        uio.uio_segflg = UIO_SYSSPACE;
        uio.uio_rw = UIO_WRITE;
        error = uiomove_fromphys(m, off, size, &uio);
        KASSERT(error == 0 && uio.uio_resid == 0, ("copy failed"));
}

void
memdesc_copyback(struct memdesc *mem, int off, int size, const void *src)
{
        KASSERT(off >= 0, ("%s: invalid offset %d", __func__, off));
        KASSERT(size >= 0, ("%s: invalid size %d", __func__, off));

        switch (mem->md_type) {
        case MEMDESC_VADDR:
                KASSERT(off + size <= mem->md_len, ("copy out of bounds"));
                memcpy((char *)mem->u.md_vaddr + off, src, size);
                break;
        case MEMDESC_PADDR:
                KASSERT(off + size <= mem->md_len, ("copy out of bounds"));
                phys_copyback(mem->u.md_paddr, off, size, src);
                break;
        case MEMDESC_VLIST:
                vlist_copyback(mem->u.md_list, mem->md_nseg, off, size, src);
                break;
        case MEMDESC_PLIST:
                plist_copyback(mem->u.md_list, mem->md_nseg, off, size, src);
                break;
        case MEMDESC_UIO:
                panic("Use uiomove instead");
                break;
        case MEMDESC_MBUF:
                m_copyback(mem->u.md_mbuf, off, size, src);
                break;
        case MEMDESC_VMPAGES:
                KASSERT(off + size <= mem->md_len, ("copy out of bounds"));
                vmpages_copyback(mem->u.md_ma, mem->md_offset + off, size,
                    src);
                break;
        default:
                __assert_unreachable();
        }
}

/*
 * memdesc_copydata copies data from a buffer described by a memory
 * descriptor into a destination buffer.
 */
static void
phys_copydata(vm_paddr_t pa, int off, int size, void *dst)
{
        char *cp;
        u_int page_off;
        int todo;
        const void *p;

        KASSERT(PMAP_HAS_DMAP, ("direct-map required"));

        cp = dst;
        pa += off;
        page_off = pa & PAGE_MASK;
        while (size > 0) {
                todo = min(PAGE_SIZE - page_off, size);
                p = PHYS_TO_DMAP(pa);
                memcpy(cp, p, todo);
                size -= todo;
                cp += todo;
                pa += todo;
                page_off = 0;
        }
}

static void
vlist_copydata(struct bus_dma_segment *vlist, int sglist_cnt, int off,
    int size, void *dst)
{
        char *p;
        int todo;

        while (vlist->ds_len <= off) {
                KASSERT(sglist_cnt > 1, ("out of sglist entries"));

                off -= vlist->ds_len;
                vlist++;
                sglist_cnt--;
        }

        p = dst;
        while (size > 0) {
                KASSERT(sglist_cnt >= 1, ("out of sglist entries"));

                todo = size;
                if (todo > vlist->ds_len - off)
                        todo = vlist->ds_len - off;

                memcpy(p, (char *)(uintptr_t)vlist->ds_addr + off, todo);
                off = 0;
                vlist++;
                sglist_cnt--;
                size -= todo;
                p += todo;
        }
}

static void
plist_copydata(struct bus_dma_segment *plist, int sglist_cnt, int off,
    int size, void *dst)
{
        char *p;
        int todo;

        while (plist->ds_len <= off) {
                KASSERT(sglist_cnt > 1, ("out of sglist entries"));

                off -= plist->ds_len;
                plist++;
                sglist_cnt--;
        }

        p = dst;
        while (size > 0) {
                KASSERT(sglist_cnt >= 1, ("out of sglist entries"));

                todo = size;
                if (todo > plist->ds_len - off)
                        todo = plist->ds_len - off;

                phys_copydata(plist->ds_addr, off, todo, p);
                off = 0;
                plist++;
                sglist_cnt--;
                size -= todo;
                p += todo;
        }
}

static void
vmpages_copydata(vm_page_t *m, int off, int size, void *dst)
{
        struct iovec iov[1];
        struct uio uio;
        int error __diagused;

        iov[0].iov_base = dst;
        iov[0].iov_len = size;
        uio.uio_iov = iov;
        uio.uio_iovcnt = 1;
        uio.uio_offset = 0;
        uio.uio_resid = size;
        uio.uio_segflg = UIO_SYSSPACE;
        uio.uio_rw = UIO_READ;
        error = uiomove_fromphys(m, off, size, &uio);
        KASSERT(error == 0 && uio.uio_resid == 0, ("copy failed"));
}

void
memdesc_copydata(struct memdesc *mem, int off, int size, void *dst)
{
        KASSERT(off >= 0, ("%s: invalid offset %d", __func__, off));
        KASSERT(size >= 0, ("%s: invalid size %d", __func__, off));

        switch (mem->md_type) {
        case MEMDESC_VADDR:
                KASSERT(off + size <= mem->md_len, ("copy out of bounds"));
                memcpy(dst, (const char *)mem->u.md_vaddr + off, size);
                break;
        case MEMDESC_PADDR:
                KASSERT(off + size <= mem->md_len, ("copy out of bounds"));
                phys_copydata(mem->u.md_paddr, off, size, dst);
                break;
        case MEMDESC_VLIST:
                vlist_copydata(mem->u.md_list, mem->md_nseg, off, size, dst);
                break;
        case MEMDESC_PLIST:
                plist_copydata(mem->u.md_list, mem->md_nseg, off, size, dst);
                break;
        case MEMDESC_UIO:
                panic("Use uiomove instead");
                break;
        case MEMDESC_MBUF:
                m_copydata(mem->u.md_mbuf, off, size, dst);
                break;
        case MEMDESC_VMPAGES:
                KASSERT(off + size <= mem->md_len, ("copy out of bounds"));
                vmpages_copydata(mem->u.md_ma, mem->md_offset + off, size,
                    dst);
                break;
        default:
                __assert_unreachable();
        }
}

/*
 * memdesc_alloc_ext_mbufs allocates a chain of external mbufs backed
 * by the storage of a memory descriptor's data buffer.
 */
static struct mbuf *
vaddr_ext_mbuf(memdesc_alloc_ext_mbuf_t *ext_alloc, void *cb_arg, int how,
    void *buf, size_t len, size_t *actual_len)
{
        *actual_len = len;
        return (ext_alloc(cb_arg, how, buf, len));
}

static bool
can_append_paddr(struct mbuf *m, vm_paddr_t pa)
{
        u_int last_len;

        /* Can always append to an empty mbuf. */
        if (m->m_epg_npgs == 0)
                return (true);

        /* Can't append to a full mbuf. */
        if (m->m_epg_npgs == MBUF_PEXT_MAX_PGS)
                return (false);

        /* Can't append a non-page-aligned address to a non-empty mbuf. */
        if ((pa & PAGE_MASK) != 0)
                return (false);

        /* Can't append if the last page is not a full page. */
        last_len = m->m_epg_last_len;
        if (m->m_epg_npgs == 1)
                last_len += m->m_epg_1st_off;
        return (last_len == PAGE_SIZE);
}

/*
 * Returns amount of data added to an M_EXTPG mbuf.
 */
static size_t
append_paddr_range(struct mbuf *m, vm_paddr_t pa, size_t len)
{
        size_t appended;

        appended = 0;

        /* Append the first page. */
        if (m->m_epg_npgs == 0) {
                m->m_epg_pa[0] = trunc_page(pa);
                m->m_epg_npgs = 1;
                m->m_epg_1st_off = pa & PAGE_MASK;
                m->m_epg_last_len = PAGE_SIZE - m->m_epg_1st_off;
                if (m->m_epg_last_len > len)
                        m->m_epg_last_len = len;
                m->m_len = m->m_epg_last_len;
                len -= m->m_epg_last_len;
                pa += m->m_epg_last_len;
                appended += m->m_epg_last_len;
        }
        KASSERT(len == 0 || (pa & PAGE_MASK) == 0,
            ("PA not aligned before full pages"));

        /* Full pages. */
        while (len >= PAGE_SIZE && m->m_epg_npgs < MBUF_PEXT_MAX_PGS) {
                m->m_epg_pa[m->m_epg_npgs] = pa;
                m->m_epg_npgs++;
                m->m_epg_last_len = PAGE_SIZE;
                m->m_len += PAGE_SIZE;
                pa += PAGE_SIZE;
                len -= PAGE_SIZE;
                appended += PAGE_SIZE;
        }

        /* Final partial page. */
        if (len > 0 && m->m_epg_npgs < MBUF_PEXT_MAX_PGS) {
                KASSERT(len < PAGE_SIZE, ("final page is full page"));
                m->m_epg_pa[m->m_epg_npgs] = pa;
                m->m_epg_npgs++;
                m->m_epg_last_len = len;
                m->m_len += len;
                appended += len;
        }

        return (appended);
}

static struct mbuf *
paddr_ext_mbuf(memdesc_alloc_extpg_mbuf_t *extpg_alloc, void *cb_arg, int how,
    vm_paddr_t pa, size_t len, size_t *actual_len, bool can_truncate)
{
        struct mbuf *m, *tail;
        size_t appended;

        if (can_truncate) {
                vm_paddr_t end;

                /*
                 * Trim any partial page at the end, but not if it's
                 * the only page.
                 */
                end = trunc_page(pa + len);
                if (end > pa)
                        len = end - pa;
        }
        *actual_len = len;

        m = tail = extpg_alloc(cb_arg, how);
        if (m == NULL)
                return (NULL);
        while (len > 0) {
                if (!can_append_paddr(tail, pa)) {
                        MBUF_EXT_PGS_ASSERT_SANITY(tail);
                        tail->m_next = extpg_alloc(cb_arg, how);
                        if (tail->m_next == NULL)
                                goto error;
                        tail = tail->m_next;
                }

                appended = append_paddr_range(tail, pa, len);
                KASSERT(appended > 0, ("did not append anything"));
                KASSERT(appended <= len, ("appended too much"));

                pa += appended;
                len -= appended;
        }

        MBUF_EXT_PGS_ASSERT_SANITY(tail);
        return (m);
error:
        m_freem(m);
        return (NULL);
}

static struct mbuf *
vlist_ext_mbuf(memdesc_alloc_ext_mbuf_t *ext_alloc, void *cb_arg, int how,
    struct bus_dma_segment *vlist, u_int sglist_cnt, size_t offset,
    size_t len, size_t *actual_len)
{
        struct mbuf *m, *n, *tail;
        size_t todo;

        *actual_len = len;

        while (vlist->ds_len <= offset) {
                KASSERT(sglist_cnt > 1, ("out of sglist entries"));

                offset -= vlist->ds_len;
                vlist++;
                sglist_cnt--;
        }

        m = tail = NULL;
        while (len > 0) {
                KASSERT(sglist_cnt >= 1, ("out of sglist entries"));

                todo = len;
                if (todo > vlist->ds_len - offset)
                        todo = vlist->ds_len - offset;

                n = ext_alloc(cb_arg, how, (char *)(uintptr_t)vlist->ds_addr +
                    offset, todo);
                if (n == NULL)
                        goto error;

                if (m == NULL) {
                        m = n;
                        tail = m;
                } else {
                        tail->m_next = n;
                        tail = n;
                }

                offset = 0;
                vlist++;
                sglist_cnt--;
                len -= todo;
        }

        return (m);
error:
        m_freem(m);
        return (NULL);
}

static struct mbuf *
plist_ext_mbuf(memdesc_alloc_extpg_mbuf_t *extpg_alloc, void *cb_arg, int how,
    struct bus_dma_segment *plist, u_int sglist_cnt, size_t offset, size_t len,
    size_t *actual_len, bool can_truncate)
{
        vm_paddr_t pa;
        struct mbuf *m, *tail;
        size_t appended, totlen, todo;

        while (plist->ds_len <= offset) {
                KASSERT(sglist_cnt > 1, ("out of sglist entries"));

                offset -= plist->ds_len;
                plist++;
                sglist_cnt--;
        }

        totlen = 0;
        m = tail = extpg_alloc(cb_arg, how);
        if (m == NULL)
                return (NULL);
        while (len > 0) {
                KASSERT(sglist_cnt >= 1, ("out of sglist entries"));

                pa = plist->ds_addr + offset;
                todo = len;
                if (todo > plist->ds_len - offset)
                        todo = plist->ds_len - offset;

                /*
                 * If truncation is enabled, avoid sending a final
                 * partial page, but only if there is more data
                 * available in the current segment.  Also, at least
                 * some data must be sent, so only drop the final page
                 * for this segment if the segment spans multiple
                 * pages or some other data is already queued.
                 */
                else if (can_truncate) {
                        vm_paddr_t end;

                        end = trunc_page(pa + len);
                        if (end <= pa && totlen != 0) {
                                /*
                                 * This last segment is only a partial
                                 * page.
                                 */
                                len = 0;
                                break;
                        }
                        todo = end - pa;
                }

                offset = 0;
                len -= todo;
                totlen += todo;

                while (todo > 0) {
                        if (!can_append_paddr(tail, pa)) {
                                MBUF_EXT_PGS_ASSERT_SANITY(tail);
                                tail->m_next = extpg_alloc(cb_arg, how);
                                if (tail->m_next == NULL)
                                        goto error;
                                tail = tail->m_next;
                        }

                        appended = append_paddr_range(tail, pa, todo);
                        KASSERT(appended > 0, ("did not append anything"));

                        pa += appended;
                        todo -= appended;
                }
        }

        MBUF_EXT_PGS_ASSERT_SANITY(tail);
        *actual_len = totlen;
        return (m);
error:
        m_freem(m);
        return (NULL);
}

static struct mbuf *
vmpages_ext_mbuf(memdesc_alloc_extpg_mbuf_t *extpg_alloc, void *cb_arg, int how,
    vm_page_t *ma, size_t offset, size_t len, size_t *actual_len,
    bool can_truncate)
{
        struct mbuf *m, *tail;

        while (offset >= PAGE_SIZE) {
                ma++;
                offset -= PAGE_SIZE;
        }

        if (can_truncate) {
                size_t end;

                /*
                 * Trim any partial page at the end, but not if it's
                 * the only page.
                 */
                end = trunc_page(offset + len);
                if (end > offset)
                        len = end - offset;
        }
        *actual_len = len;

        m = tail = extpg_alloc(cb_arg, how);
        if (m == NULL)
                return (NULL);

        /* First page. */
        m->m_epg_pa[0] = VM_PAGE_TO_PHYS(*ma);
        ma++;
        m->m_epg_npgs = 1;
        m->m_epg_1st_off = offset;
        m->m_epg_last_len = PAGE_SIZE - offset;
        if (m->m_epg_last_len > len)
                m->m_epg_last_len = len;
        m->m_len = m->m_epg_last_len;
        len -= m->m_epg_last_len;

        /* Full pages. */
        while (len >= PAGE_SIZE) {
                if (tail->m_epg_npgs == MBUF_PEXT_MAX_PGS) {
                        MBUF_EXT_PGS_ASSERT_SANITY(tail);
                        tail->m_next = extpg_alloc(cb_arg, how);
                        if (tail->m_next == NULL)
                                goto error;
                        tail = tail->m_next;
                }

                tail->m_epg_pa[tail->m_epg_npgs] = VM_PAGE_TO_PHYS(*ma);
                ma++;
                tail->m_epg_npgs++;
                tail->m_epg_last_len = PAGE_SIZE;
                tail->m_len += PAGE_SIZE;
                len -= PAGE_SIZE;
        }

        /* Last partial page. */
        if (len > 0) {
                if (tail->m_epg_npgs == MBUF_PEXT_MAX_PGS) {
                        MBUF_EXT_PGS_ASSERT_SANITY(tail);
                        tail->m_next = extpg_alloc(cb_arg, how);
                        if (tail->m_next == NULL)
                                goto error;
                        tail = tail->m_next;
                }

                tail->m_epg_pa[tail->m_epg_npgs] = VM_PAGE_TO_PHYS(*ma);
                ma++;
                tail->m_epg_npgs++;
                tail->m_epg_last_len = len;
                tail->m_len += len;
        }

        MBUF_EXT_PGS_ASSERT_SANITY(tail);
        return (m);
error:
        m_freem(m);
        return (NULL);
}

/*
 * Somewhat similar to m_copym but optionally avoids a partial mbuf at
 * the end.
 */
static struct mbuf *
mbuf_subchain(struct mbuf *m0, size_t offset, size_t len,
    size_t *actual_len, bool can_truncate, int how)
{
        struct mbuf *m, *tail;
        size_t totlen;

        while (offset >= m0->m_len) {
                offset -= m0->m_len;
                m0 = m0->m_next;
        }

        /* Always return at least one mbuf. */
        totlen = m0->m_len - offset;
        if (totlen > len)
                totlen = len;

        m = m_get(how, MT_DATA);
        if (m == NULL)
                return (NULL);
        m->m_len = totlen;
        if (m0->m_flags & (M_EXT | M_EXTPG)) {
                m->m_data = m0->m_data + offset;
                mb_dupcl(m, m0);
        } else
                memcpy(mtod(m, void *), mtodo(m0, offset), m->m_len);

        tail = m;
        m0 = m0->m_next;
        len -= totlen;
        while (len > 0) {
                /*
                 * If truncation is enabled, don't send any partial
                 * mbufs besides the first one.
                 */
                if (can_truncate && m0->m_len > len)
                        break;

                tail->m_next = m_get(how, MT_DATA);
                if (tail->m_next == NULL)
                        goto error;
                tail = tail->m_next;
                tail->m_len = m0->m_len;
                if (m0->m_flags & (M_EXT | M_EXTPG)) {
                        tail->m_data = m0->m_data;
                        mb_dupcl(tail, m0);
                } else
                        memcpy(mtod(tail, void *), mtod(m0, void *),
                            tail->m_len);

                totlen += tail->m_len;
                m0 = m0->m_next;
                len -= tail->m_len;
        }
        *actual_len = totlen;
        return (m);
error:
        m_freem(m);
        return (NULL);
}

struct mbuf *
memdesc_alloc_ext_mbufs(struct memdesc *mem,
    memdesc_alloc_ext_mbuf_t *ext_alloc,
    memdesc_alloc_extpg_mbuf_t *extpg_alloc, void *cb_arg, int how,
    size_t offset, size_t len, size_t *actual_len, bool can_truncate)
{
        struct mbuf *m;
        size_t done;

        switch (mem->md_type) {
        case MEMDESC_VADDR:
                m = vaddr_ext_mbuf(ext_alloc, cb_arg, how,
                    (char *)mem->u.md_vaddr + offset, len, &done);
                break;
        case MEMDESC_PADDR:
                m = paddr_ext_mbuf(extpg_alloc, cb_arg, how, mem->u.md_paddr +
                    offset, len, &done, can_truncate);
                break;
        case MEMDESC_VLIST:
                m = vlist_ext_mbuf(ext_alloc, cb_arg, how, mem->u.md_list,
                    mem->md_nseg, offset, len, &done);
                break;
        case MEMDESC_PLIST:
                m = plist_ext_mbuf(extpg_alloc, cb_arg, how, mem->u.md_list,
                    mem->md_nseg, offset, len, &done, can_truncate);
                break;
        case MEMDESC_UIO:
                panic("uio not supported");
        case MEMDESC_MBUF:
                m = mbuf_subchain(mem->u.md_mbuf, offset, len, &done,
                    can_truncate, how);
                break;
        case MEMDESC_VMPAGES:
                m = vmpages_ext_mbuf(extpg_alloc, cb_arg, how, mem->u.md_ma,
                    mem->md_offset + offset, len, &done, can_truncate);
                break;
        default:
                __assert_unreachable();
        }
        if (m == NULL)
                return (NULL);

        if (can_truncate) {
                KASSERT(done <= len, ("chain too long"));
        } else {
                KASSERT(done == len, ("short chain with no limit"));
        }
        KASSERT(m_length(m, NULL) == done, ("length mismatch"));
        if (actual_len != NULL)
                *actual_len = done;
        return (m);
}