/*
 * Copyright (c) 2017, Mellanox Technologies inc.  All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <rdma/rdma_user_ioctl.h>
#include <rdma/uverbs_ioctl.h>
#include "rdma_core.h"
#include "uverbs.h"

struct bundle_alloc_head {
        struct_group_tagged(bundle_alloc_head_hdr, hdr,
                struct bundle_alloc_head *next;
        );
        u8 data[];
};

struct bundle_priv {
        /* Must be first */
        struct bundle_alloc_head_hdr alloc_head;
        struct bundle_alloc_head *allocated_mem;
        size_t internal_avail;
        size_t internal_used;

        struct radix_tree_root *radix;
        const struct uverbs_api_ioctl_method *method_elm;
        void __rcu **radix_slots;
        unsigned long radix_slots_len;
        u32 method_key;

        struct ib_uverbs_attr __user *user_attrs;
        struct ib_uverbs_attr *uattrs;

        DECLARE_BITMAP(uobj_finalize, UVERBS_API_ATTR_BKEY_LEN);
        DECLARE_BITMAP(spec_finalize, UVERBS_API_ATTR_BKEY_LEN);
        DECLARE_BITMAP(uobj_hw_obj_valid, UVERBS_API_ATTR_BKEY_LEN);

        /*
         * Must be last. bundle ends in a flex array which overlaps
         * internal_buffer.
         */
        struct uverbs_attr_bundle_hdr bundle;
        u64 internal_buffer[32];
};

/*
 * Each method has an absolute minimum amount of memory it needs to allocate,
 * precompute that amount and determine if the onstack memory can be used or
 * if allocation is need.
 */
void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm,
                              unsigned int num_attrs)
{
        struct bundle_priv *pbundle;
        struct uverbs_attr_bundle *bundle;
        size_t bundle_size =
                offsetof(struct bundle_priv, internal_buffer) +
                sizeof(*bundle->attrs) * method_elm->key_bitmap_len +
                sizeof(*pbundle->uattrs) * num_attrs;

        method_elm->use_stack = bundle_size <= sizeof(*pbundle);
        method_elm->bundle_size =
                ALIGN(bundle_size + 256, sizeof(*pbundle->internal_buffer));

        /* Do not want order-2 allocations for this. */
        WARN_ON_ONCE(method_elm->bundle_size > PAGE_SIZE);
}

/**
 * _uverbs_alloc() - Quickly allocate memory for use with a bundle
 * @bundle: The bundle
 * @size: Number of bytes to allocate
 * @flags: Allocator flags
 *
 * The bundle allocator is intended for allocations that are connected with
 * processing the system call related to the bundle. The allocated memory is
 * always freed once the system call completes, and cannot be freed any other
 * way.
 *
 * This tries to use a small pool of pre-allocated memory for performance.
 */
__malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size,
                             gfp_t flags)
{
        struct bundle_priv *pbundle =
                container_of(&bundle->hdr, struct bundle_priv, bundle);
        size_t new_used;
        void *res;

        if (check_add_overflow(size, pbundle->internal_used, &new_used))
                return ERR_PTR(-EOVERFLOW);

        if (new_used > pbundle->internal_avail) {
                struct bundle_alloc_head *buf;

                buf = kvmalloc_flex(*buf, data, size, flags);
                if (!buf)
                        return ERR_PTR(-ENOMEM);
                buf->next = pbundle->allocated_mem;
                pbundle->allocated_mem = buf;
                return buf->data;
        }

        res = (void *)pbundle->internal_buffer + pbundle->internal_used;
        pbundle->internal_used =
                ALIGN(new_used, sizeof(*pbundle->internal_buffer));
        if (want_init_on_alloc(flags))
                memset(res, 0, size);
        return res;
}
EXPORT_SYMBOL(_uverbs_alloc);

static bool uverbs_is_attr_cleared(const struct ib_uverbs_attr *uattr,
                                   u16 len)
{
        if (uattr->len > sizeof_field(struct ib_uverbs_attr, data))
                return ib_is_buffer_cleared(u64_to_user_ptr(uattr->data) + len,
                                            uattr->len - len);

        return !memchr_inv((const void *)&uattr->data + len,
                           0, uattr->len - len);
}

static int uverbs_set_output(const struct uverbs_attr_bundle *bundle,
                             const struct uverbs_attr *attr)
{
        struct bundle_priv *pbundle =
                container_of(&bundle->hdr, struct bundle_priv, bundle);
        u16 flags;

        flags = pbundle->uattrs[attr->ptr_attr.uattr_idx].flags |
                UVERBS_ATTR_F_VALID_OUTPUT;
        if (put_user(flags,
                     &pbundle->user_attrs[attr->ptr_attr.uattr_idx].flags))
                return -EFAULT;
        return 0;
}

static int uverbs_process_idrs_array(struct bundle_priv *pbundle,
                                     const struct uverbs_api_attr *attr_uapi,
                                     struct uverbs_objs_arr_attr *attr,
                                     struct ib_uverbs_attr *uattr,
                                     u32 attr_bkey)
{
        struct uverbs_attr_bundle *bundle =
                container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);
        const struct uverbs_attr_spec *spec = &attr_uapi->spec;
        size_t array_len;
        u32 *idr_vals;
        int ret = 0;
        size_t i;

        if (uattr->attr_data.reserved)
                return -EINVAL;

        if (uattr->len % sizeof(u32))
                return -EINVAL;

        array_len = uattr->len / sizeof(u32);
        if (array_len < spec->u2.objs_arr.min_len ||
            array_len > spec->u2.objs_arr.max_len)
                return -EINVAL;

        attr->uobjects =
                uverbs_alloc(bundle,
                             array_size(array_len, sizeof(*attr->uobjects)));
        if (IS_ERR(attr->uobjects))
                return PTR_ERR(attr->uobjects);

        /*
         * Since idr is 4B and *uobjects is >= 4B, we can use attr->uobjects
         * to store idrs array and avoid additional memory allocation. The
         * idrs array is offset to the end of the uobjects array so we will be
         * able to read idr and replace with a pointer.
         */
        idr_vals = (u32 *)(attr->uobjects + array_len) - array_len;

        if (uattr->len > sizeof(uattr->data)) {
                ret = copy_from_user(idr_vals, u64_to_user_ptr(uattr->data),
                                     uattr->len);
                if (ret)
                        return -EFAULT;
        } else {
                memcpy(idr_vals, &uattr->data, uattr->len);
        }

        for (i = 0; i != array_len; i++) {
                attr->uobjects[i] = uverbs_get_uobject_from_file(
                        spec->u2.objs_arr.obj_type, spec->u2.objs_arr.access,
                        idr_vals[i], bundle);
                if (IS_ERR(attr->uobjects[i])) {
                        ret = PTR_ERR(attr->uobjects[i]);
                        break;
                }
        }

        attr->len = i;
        __set_bit(attr_bkey, pbundle->spec_finalize);
        return ret;
}

static void uverbs_free_idrs_array(const struct uverbs_api_attr *attr_uapi,
                                   struct uverbs_objs_arr_attr *attr,
                                   bool commit,
                                   struct uverbs_attr_bundle *attrs)
{
        const struct uverbs_attr_spec *spec = &attr_uapi->spec;
        size_t i;

        for (i = 0; i != attr->len; i++)
                uverbs_finalize_object(attr->uobjects[i],
                                       spec->u2.objs_arr.access, false, commit,
                                       attrs);
}

static int uverbs_process_attr(struct bundle_priv *pbundle,
                               const struct uverbs_api_attr *attr_uapi,
                               struct ib_uverbs_attr *uattr, u32 attr_bkey)
{
        const struct uverbs_attr_spec *spec = &attr_uapi->spec;
        struct uverbs_attr_bundle *bundle =
                container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);
        struct uverbs_attr *e = &bundle->attrs[attr_bkey];
        const struct uverbs_attr_spec *val_spec = spec;
        struct uverbs_obj_attr *o_attr;

        switch (spec->type) {
        case UVERBS_ATTR_TYPE_ENUM_IN:
                if (uattr->attr_data.enum_data.elem_id >= spec->u.enum_def.num_elems)
                        return -EOPNOTSUPP;

                if (uattr->attr_data.enum_data.reserved)
                        return -EINVAL;

                val_spec = &spec->u2.enum_def.ids[uattr->attr_data.enum_data.elem_id];

                /* Currently we only support PTR_IN based enums */
                if (val_spec->type != UVERBS_ATTR_TYPE_PTR_IN)
                        return -EOPNOTSUPP;

                e->ptr_attr.enum_id = uattr->attr_data.enum_data.elem_id;
                fallthrough;
        case UVERBS_ATTR_TYPE_PTR_IN:
                /* Ensure that any data provided by userspace beyond the known
                 * struct is zero. Userspace that knows how to use some future
                 * longer struct will fail here if used with an old kernel and
                 * non-zero content, making ABI compat/discovery simpler.
                 */
                if (uattr->len > val_spec->u.ptr.len &&
                    val_spec->zero_trailing &&
                    !uverbs_is_attr_cleared(uattr, val_spec->u.ptr.len))
                        return -EOPNOTSUPP;

                fallthrough;
        case UVERBS_ATTR_TYPE_PTR_OUT:
                if (uattr->len < val_spec->u.ptr.min_len ||
                    (!val_spec->zero_trailing &&
                     uattr->len > val_spec->u.ptr.len))
                        return -EINVAL;

                if (spec->type != UVERBS_ATTR_TYPE_ENUM_IN &&
                    uattr->attr_data.reserved)
                        return -EINVAL;

                e->ptr_attr.uattr_idx = uattr - pbundle->uattrs;
                e->ptr_attr.len = uattr->len;

                if (val_spec->alloc_and_copy && !uverbs_attr_ptr_is_inline(e)) {
                        void *p;

                        p = uverbs_alloc(bundle, uattr->len);
                        if (IS_ERR(p))
                                return PTR_ERR(p);

                        e->ptr_attr.ptr = p;

                        if (copy_from_user(p, u64_to_user_ptr(uattr->data),
                                           uattr->len))
                                return -EFAULT;
                } else {
                        e->ptr_attr.data = uattr->data;
                }
                break;

        case UVERBS_ATTR_TYPE_IDR:
        case UVERBS_ATTR_TYPE_FD:
                if (uattr->attr_data.reserved)
                        return -EINVAL;

                if (uattr->len != 0)
                        return -EINVAL;

                o_attr = &e->obj_attr;
                o_attr->attr_elm = attr_uapi;

                /*
                 * The type of uattr->data is u64 for UVERBS_ATTR_TYPE_IDR and
                 * s64 for UVERBS_ATTR_TYPE_FD. We can cast the u64 to s64
                 * here without caring about truncation as we know that the
                 * IDR implementation today rejects negative IDs
                 */
                o_attr->uobject = uverbs_get_uobject_from_file(
                        spec->u.obj.obj_type, spec->u.obj.access,
                        uattr->data_s64, bundle);
                if (IS_ERR(o_attr->uobject))
                        return PTR_ERR(o_attr->uobject);
                __set_bit(attr_bkey, pbundle->uobj_finalize);

                if (spec->u.obj.access == UVERBS_ACCESS_NEW) {
                        unsigned int uattr_idx = uattr - pbundle->uattrs;
                        s64 id = o_attr->uobject->id;

                        /* Copy the allocated id to the user-space */
                        if (put_user(id, &pbundle->user_attrs[uattr_idx].data))
                                return -EFAULT;
                }

                break;

        case UVERBS_ATTR_TYPE_RAW_FD:
                if (uattr->attr_data.reserved || uattr->len != 0 ||
                    uattr->data_s64 < INT_MIN || uattr->data_s64 > INT_MAX)
                        return -EINVAL;
                /* _uverbs_get_const_signed() is the accessor */
                e->ptr_attr.data = uattr->data_s64;
                break;

        case UVERBS_ATTR_TYPE_IDRS_ARRAY:
                return uverbs_process_idrs_array(pbundle, attr_uapi,
                                                 &e->objs_arr_attr, uattr,
                                                 attr_bkey);
        default:
                return -EOPNOTSUPP;
        }

        return 0;
}

/*
 * We search the radix tree with the method prefix and now we want to fast
 * search the suffix bits to get a particular attribute pointer. It is not
 * totally clear to me if this breaks the radix tree encasulation or not, but
 * it uses the iter data to determine if the method iter points at the same
 * chunk that will store the attribute, if so it just derefs it directly. By
 * construction in most kernel configs the method and attrs will all fit in a
 * single radix chunk, so in most cases this will have no search. Other cases
 * this falls back to a full search.
 */
static void __rcu **uapi_get_attr_for_method(struct bundle_priv *pbundle,
                                             u32 attr_key)
{
        void __rcu **slot;

        if (likely(attr_key < pbundle->radix_slots_len)) {
                void *entry;

                slot = pbundle->radix_slots + attr_key;
                entry = rcu_dereference_raw(*slot);
                if (likely(!radix_tree_is_internal_node(entry) && entry))
                        return slot;
        }

        return radix_tree_lookup_slot(pbundle->radix,
                                      pbundle->method_key | attr_key);
}

static int uverbs_set_attr(struct bundle_priv *pbundle,
                           struct ib_uverbs_attr *uattr)
{
        u32 attr_key = uapi_key_attr(uattr->attr_id);
        u32 attr_bkey = uapi_bkey_attr(attr_key);
        const struct uverbs_api_attr *attr;
        void __rcu **slot;
        int ret;

        slot = uapi_get_attr_for_method(pbundle, attr_key);
        if (!slot) {
                /*
                 * Kernel does not support the attribute but user-space says it
                 * is mandatory
                 */
                if (uattr->flags & UVERBS_ATTR_F_MANDATORY)
                        return -EPROTONOSUPPORT;
                return 0;
        }
        attr = rcu_dereference_protected(*slot, true);

        /* Reject duplicate attributes from user-space */
        if (test_bit(attr_bkey, pbundle->bundle.attr_present))
                return -EINVAL;

        ret = uverbs_process_attr(pbundle, attr, uattr, attr_bkey);
        if (ret)
                return ret;

        __set_bit(attr_bkey, pbundle->bundle.attr_present);

        return 0;
}

static int ib_uverbs_run_method(struct bundle_priv *pbundle,
                                unsigned int num_attrs)
{
        int (*handler)(struct uverbs_attr_bundle *attrs);
        struct uverbs_attr_bundle *bundle =
                container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);
        size_t uattrs_size = array_size(sizeof(*pbundle->uattrs), num_attrs);
        unsigned int destroy_bkey = pbundle->method_elm->destroy_bkey;
        unsigned int i;
        int ret;

        /* See uverbs_disassociate_api() */
        handler = srcu_dereference(
                pbundle->method_elm->handler,
                &pbundle->bundle.ufile->device->disassociate_srcu);
        if (!handler)
                return -EIO;

        pbundle->uattrs = uverbs_alloc(bundle, uattrs_size);
        if (IS_ERR(pbundle->uattrs))
                return PTR_ERR(pbundle->uattrs);
        if (copy_from_user(pbundle->uattrs, pbundle->user_attrs, uattrs_size))
                return -EFAULT;

        for (i = 0; i != num_attrs; i++) {
                ret = uverbs_set_attr(pbundle, &pbundle->uattrs[i]);
                if (unlikely(ret))
                        return ret;
        }

        /* User space did not provide all the mandatory attributes */
        if (unlikely(!bitmap_subset(pbundle->method_elm->attr_mandatory,
                                    pbundle->bundle.attr_present,
                                    pbundle->method_elm->key_bitmap_len)))
                return -EINVAL;

        if (pbundle->method_elm->has_udata)
                uverbs_fill_udata(bundle, &pbundle->bundle.driver_udata,
                                  UVERBS_ATTR_UHW_IN, UVERBS_ATTR_UHW_OUT);
        else
                pbundle->bundle.driver_udata = (struct ib_udata){};

        if (destroy_bkey != UVERBS_API_ATTR_BKEY_LEN) {
                struct uverbs_obj_attr *destroy_attr = &bundle->attrs[destroy_bkey].obj_attr;

                ret = uobj_destroy(destroy_attr->uobject, bundle);
                if (ret)
                        return ret;
                __clear_bit(destroy_bkey, pbundle->uobj_finalize);

                ret = handler(bundle);
                uobj_put_destroy(destroy_attr->uobject);
        } else {
                ret = handler(bundle);
        }

        /*
         * Until the drivers are revised to use the bundle directly we have to
         * assume that the driver wrote to its UHW_OUT and flag userspace
         * appropriately.
         */
        if (!ret && pbundle->method_elm->has_udata) {
                const struct uverbs_attr *attr =
                        uverbs_attr_get(bundle, UVERBS_ATTR_UHW_OUT);

                if (!IS_ERR(attr))
                        ret = uverbs_set_output(bundle, attr);
        }

        /*
         * EPROTONOSUPPORT is ONLY to be returned if the ioctl framework can
         * not invoke the method because the request is not supported.  No
         * other cases should return this code.
         */
        if (WARN_ON_ONCE(ret == -EPROTONOSUPPORT))
                return -EINVAL;

        return ret;
}

static void bundle_destroy(struct bundle_priv *pbundle, bool commit)
{
        unsigned int key_bitmap_len = pbundle->method_elm->key_bitmap_len;
        struct uverbs_attr_bundle *bundle =
                container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);
        struct bundle_alloc_head *memblock;
        unsigned int i;

        /* fast path for simple uobjects */
        i = -1;
        while ((i = find_next_bit(pbundle->uobj_finalize, key_bitmap_len,
                                  i + 1)) < key_bitmap_len) {
                struct uverbs_attr *attr = &bundle->attrs[i];

                uverbs_finalize_object(
                        attr->obj_attr.uobject,
                        attr->obj_attr.attr_elm->spec.u.obj.access,
                        test_bit(i, pbundle->uobj_hw_obj_valid),
                        commit, bundle);
        }

        i = -1;
        while ((i = find_next_bit(pbundle->spec_finalize, key_bitmap_len,
                                  i + 1)) < key_bitmap_len) {
                struct uverbs_attr *attr = &bundle->attrs[i];
                const struct uverbs_api_attr *attr_uapi;
                void __rcu **slot;

                slot = uapi_get_attr_for_method(
                        pbundle,
                        pbundle->method_key | uapi_bkey_to_key_attr(i));
                if (WARN_ON(!slot))
                        continue;

                attr_uapi = rcu_dereference_protected(*slot, true);

                if (attr_uapi->spec.type == UVERBS_ATTR_TYPE_IDRS_ARRAY) {
                        uverbs_free_idrs_array(attr_uapi, &attr->objs_arr_attr,
                                               commit, bundle);
                }
        }

        for (memblock = pbundle->allocated_mem; memblock;) {
                struct bundle_alloc_head *tmp = memblock;

                memblock = memblock->next;
                kvfree(tmp);
        }
}

static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile,
                               struct ib_uverbs_ioctl_hdr *hdr,
                               struct ib_uverbs_attr __user *user_attrs)
{
        const struct uverbs_api_ioctl_method *method_elm;
        struct uverbs_api *uapi = ufile->device->uapi;
        struct radix_tree_iter attrs_iter;
        struct bundle_priv *pbundle;
        struct bundle_priv onstack;
        void __rcu **slot;
        int ret;

        if (unlikely(hdr->driver_id != uapi->driver_id))
                return -EINVAL;

        slot = radix_tree_iter_lookup(
                &uapi->radix, &attrs_iter,
                uapi_key_obj(hdr->object_id) |
                        uapi_key_ioctl_method(hdr->method_id));
        if (unlikely(!slot))
                return -EPROTONOSUPPORT;
        method_elm = rcu_dereference_protected(*slot, true);

        if (!method_elm->use_stack) {
                pbundle = kmalloc(method_elm->bundle_size, GFP_KERNEL);
                if (!pbundle)
                        return -ENOMEM;
                pbundle->internal_avail =
                        method_elm->bundle_size -
                        offsetof(struct bundle_priv, internal_buffer);
                pbundle->alloc_head.next = NULL;
                pbundle->allocated_mem = container_of(&pbundle->alloc_head,
                                                struct bundle_alloc_head, hdr);
        } else {
                pbundle = &onstack;
                pbundle->internal_avail = sizeof(pbundle->internal_buffer);
                pbundle->allocated_mem = NULL;
        }

        /* Space for the pbundle->bundle.attrs flex array */
        pbundle->method_elm = method_elm;
        pbundle->method_key = attrs_iter.index;
        pbundle->bundle.ufile = ufile;
        pbundle->bundle.context = NULL; /* only valid if bundle has uobject */
        pbundle->radix = &uapi->radix;
        pbundle->radix_slots = slot;
        pbundle->radix_slots_len = radix_tree_chunk_size(&attrs_iter);
        pbundle->user_attrs = user_attrs;

        pbundle->internal_used = ALIGN(pbundle->method_elm->key_bitmap_len *
                                               sizeof(*container_of(&pbundle->bundle,
                                                        struct uverbs_attr_bundle, hdr)->attrs),
                                               sizeof(*pbundle->internal_buffer));
        memset(pbundle->bundle.attr_present, 0,
               sizeof(pbundle->bundle.attr_present));
        memset(pbundle->uobj_finalize, 0, sizeof(pbundle->uobj_finalize));
        memset(pbundle->spec_finalize, 0, sizeof(pbundle->spec_finalize));
        memset(pbundle->uobj_hw_obj_valid, 0,
               sizeof(pbundle->uobj_hw_obj_valid));

        ret = ib_uverbs_run_method(pbundle, hdr->num_attrs);
        bundle_destroy(pbundle, ret == 0);
        return ret;
}

long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
{
        struct ib_uverbs_file *file = filp->private_data;
        struct ib_uverbs_ioctl_hdr __user *user_hdr =
                (struct ib_uverbs_ioctl_hdr __user *)arg;
        struct ib_uverbs_ioctl_hdr hdr;
        int srcu_key;
        int err;

        if (unlikely(cmd != RDMA_VERBS_IOCTL))
                return -ENOIOCTLCMD;

        err = copy_from_user(&hdr, user_hdr, sizeof(hdr));
        if (err)
                return -EFAULT;

        if (hdr.length > PAGE_SIZE ||
            hdr.length != struct_size(&hdr, attrs, hdr.num_attrs))
                return -EINVAL;

        if (hdr.reserved1 || hdr.reserved2)
                return -EPROTONOSUPPORT;

        srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
        err = ib_uverbs_cmd_verbs(file, &hdr, user_hdr->attrs);
        srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
        return err;
}

int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle,
                       size_t idx, u64 allowed_bits)
{
        const struct uverbs_attr *attr;
        u64 flags;

        attr = uverbs_attr_get(attrs_bundle, idx);
        /* Missing attribute means 0 flags */
        if (IS_ERR(attr)) {
                *to = 0;
                return 0;
        }

        /*
         * New userspace code should use 8 bytes to pass flags, but we
         * transparently support old userspaces that were using 4 bytes as
         * well.
         */
        if (attr->ptr_attr.len == 8)
                flags = attr->ptr_attr.data;
        else if (attr->ptr_attr.len == 4)
                flags = *(u32 *)&attr->ptr_attr.data;
        else
                return -EINVAL;

        if (flags & ~allowed_bits)
                return -EINVAL;

        *to = flags;
        return 0;
}
EXPORT_SYMBOL(uverbs_get_flags64);

int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle,
                       size_t idx, u64 allowed_bits)
{
        u64 flags;
        int ret;

        ret = uverbs_get_flags64(&flags, attrs_bundle, idx, allowed_bits);
        if (ret)
                return ret;

        if (flags > U32_MAX)
                return -EINVAL;
        *to = flags;

        return 0;
}
EXPORT_SYMBOL(uverbs_get_flags32);

/*
 * Fill a ib_udata struct (core or uhw) using the given attribute IDs.
 * This is primarily used to convert the UVERBS_ATTR_UHW() into the
 * ib_udata format used by the drivers.
 */
void uverbs_fill_udata(struct uverbs_attr_bundle *bundle,
                       struct ib_udata *udata, unsigned int attr_in,
                       unsigned int attr_out)
{
        struct bundle_priv *pbundle =
                container_of(&bundle->hdr, struct bundle_priv, bundle);
        struct uverbs_attr_bundle *bundle_aux =
                container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);
        const struct uverbs_attr *in =
                uverbs_attr_get(bundle_aux, attr_in);
        const struct uverbs_attr *out =
                uverbs_attr_get(bundle_aux, attr_out);

        if (!IS_ERR(in)) {
                udata->inlen = in->ptr_attr.len;
                if (uverbs_attr_ptr_is_inline(in))
                        udata->inbuf =
                                &pbundle->user_attrs[in->ptr_attr.uattr_idx]
                                         .data;
                else
                        udata->inbuf = u64_to_user_ptr(in->ptr_attr.data);
        } else {
                udata->inbuf = NULL;
                udata->inlen = 0;
        }

        if (!IS_ERR(out)) {
                udata->outbuf = u64_to_user_ptr(out->ptr_attr.data);
                udata->outlen = out->ptr_attr.len;
        } else {
                udata->outbuf = NULL;
                udata->outlen = 0;
        }
}

int uverbs_copy_to(const struct uverbs_attr_bundle *bundle, size_t idx,
                   const void *from, size_t size)
{
        const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
        size_t min_size;

        if (IS_ERR(attr))
                return PTR_ERR(attr);

        min_size = min_t(size_t, attr->ptr_attr.len, size);
        if (copy_to_user(u64_to_user_ptr(attr->ptr_attr.data), from, min_size))
                return -EFAULT;

        return uverbs_set_output(bundle, attr);
}
EXPORT_SYMBOL(uverbs_copy_to);


/*
 * This is only used if the caller has directly used copy_to_use to write the
 * data.  It signals to user space that the buffer is filled in.
 */
int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx)
{
        const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);

        if (IS_ERR(attr))
                return PTR_ERR(attr);

        return uverbs_set_output(bundle, attr);
}

int _uverbs_get_const_signed(s64 *to,
                             const struct uverbs_attr_bundle *attrs_bundle,
                             size_t idx, s64 lower_bound, u64 upper_bound,
                             s64  *def_val)
{
        const struct uverbs_attr *attr;

        attr = uverbs_attr_get(attrs_bundle, idx);
        if (IS_ERR(attr)) {
                if ((PTR_ERR(attr) != -ENOENT) || !def_val)
                        return PTR_ERR(attr);

                *to = *def_val;
        } else {
                *to = attr->ptr_attr.data;
        }

        if (*to < lower_bound || (*to > 0 && (u64)*to > upper_bound))
                return -EINVAL;

        return 0;
}
EXPORT_SYMBOL(_uverbs_get_const_signed);

int _uverbs_get_const_unsigned(u64 *to,
                               const struct uverbs_attr_bundle *attrs_bundle,
                               size_t idx, u64 upper_bound, u64 *def_val)
{
        const struct uverbs_attr *attr;

        attr = uverbs_attr_get(attrs_bundle, idx);
        if (IS_ERR(attr)) {
                if ((PTR_ERR(attr) != -ENOENT) || !def_val)
                        return PTR_ERR(attr);

                *to = *def_val;
        } else {
                *to = attr->ptr_attr.data;
        }

        if (*to > upper_bound)
                return -EINVAL;

        return 0;
}
EXPORT_SYMBOL(_uverbs_get_const_unsigned);

int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle,
                                  size_t idx, const void *from, size_t size)
{
        const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);

        if (IS_ERR(attr))
                return PTR_ERR(attr);

        if (size < attr->ptr_attr.len) {
                if (clear_user(u64_to_user_ptr(attr->ptr_attr.data) + size,
                               attr->ptr_attr.len - size))
                        return -EFAULT;
        }
        return uverbs_copy_to(bundle, idx, from, size);
}
EXPORT_SYMBOL(uverbs_copy_to_struct_or_zero);

/* Once called an abort will call through to the type's destroy_hw() */
void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle,
                                 u16 idx)
{
        struct bundle_priv *pbundle =
                container_of(&bundle->hdr, struct bundle_priv, bundle);

        __set_bit(uapi_bkey_attr(uapi_key_attr(idx)),
                  pbundle->uobj_hw_obj_valid);
}
EXPORT_SYMBOL(uverbs_finalize_uobj_create);