/*- * Copyright (c) 2018 VMware, Inc. * * SPDX-License-Identifier: (BSD-2-Clause OR GPL-2.0) */ /* This file implements defines and helper functions. */ #include <sys/cdefs.h> #include <sys/malloc.h> #include <sys/proc.h> #include <sys/uio.h> #include <machine/bus.h> #include "vmci.h" #include "vmci_defs.h" #include "vmci_kernel_defs.h" #include "vmci_kernel_if.h" #include "vmci_queue.h" struct vmci_queue_kernel_if { size_t num_pages; /* Num pages incl. header. */ struct vmci_dma_alloc *dmas; /* For dma alloc. */ }; /* *------------------------------------------------------------------------------ * * vmci_init_lock * * Initializes the lock. Must be called before use. * * Results: * Always VMCI_SUCCESS. * * Side effects: * Thread can block. * *------------------------------------------------------------------------------ */ int vmci_init_lock(vmci_lock *lock, char *name) { mtx_init(lock, name, NULL, MTX_DEF | MTX_NOWITNESS); return (VMCI_SUCCESS); } /* *------------------------------------------------------------------------------ * * vmci_cleanup_lock * * Cleanup the lock. Must be called before deallocating lock. * * Results: * None * * Side effects: * Deletes kernel lock state * *------------------------------------------------------------------------------ */ void vmci_cleanup_lock(vmci_lock *lock) { if (mtx_initialized(lock)) mtx_destroy(lock); } /* *------------------------------------------------------------------------------ * * vmci_grab_lock * * Grabs the given lock. * * Results: * None * * Side effects: * Thread can block. * *------------------------------------------------------------------------------ */ void vmci_grab_lock(vmci_lock *lock) { mtx_lock(lock); } /* *------------------------------------------------------------------------------ * * vmci_release_lock * * Releases the given lock. * * Results: * None * * Side effects: * A thread blocked on this lock may wake up. * *------------------------------------------------------------------------------ */ void vmci_release_lock(vmci_lock *lock) { mtx_unlock(lock); } /* *------------------------------------------------------------------------------ * * vmci_grab_lock_bh * * Grabs the given lock. * * Results: * None * * Side effects: * None. * *------------------------------------------------------------------------------ */ void vmci_grab_lock_bh(vmci_lock *lock) { mtx_lock(lock); } /* *------------------------------------------------------------------------------ * * vmci_release_lock_bh * * Releases the given lock. * * Results: * None * * Side effects: * None. * *------------------------------------------------------------------------------ */ void vmci_release_lock_bh(vmci_lock *lock) { mtx_unlock(lock); } /* *------------------------------------------------------------------------------ * * vmci_initialized_lock * * Returns whether a lock has been initialized. * * Results: * Return 1 if initialized or 0 if unininitialized. * * Side effects: * None * *------------------------------------------------------------------------------ */ int vmci_initialized_lock(vmci_lock *lock) { return mtx_initialized(lock); } /* *------------------------------------------------------------------------------ * * vmci_alloc_kernel_mem * * Allocate physically contiguous memory for the VMCI driver. * * Results: * The address allocated or NULL on error. * * * Side effects: * Memory may be allocated. * *------------------------------------------------------------------------------ */ void * vmci_alloc_kernel_mem(size_t size, int flags) { void *ptr; if ((flags & VMCI_MEMORY_ATOMIC) != 0) ptr = contigmalloc(size, M_DEVBUF, M_NOWAIT, 0, 0xFFFFFFFF, 8, 1024 * 1024); else ptr = contigmalloc(size, M_DEVBUF, M_WAITOK, 0, 0xFFFFFFFF, 8, 1024 * 1024); return (ptr); } /* *------------------------------------------------------------------------------ * * vmci_free_kernel_mem * * Free kernel memory allocated for the VMCI driver. * * Results: * None. * * Side effects: * Memory is freed. * *------------------------------------------------------------------------------ */ void vmci_free_kernel_mem(void *ptr, size_t size __unused) { free(ptr, M_DEVBUF); } /* *------------------------------------------------------------------------------ * * vmci_can_schedule_delayed_work -- * * Checks to see if the given platform supports delayed work callbacks. * * Results: * true if it does. false otherwise. * * Side effects: * None. * *------------------------------------------------------------------------------ */ bool vmci_can_schedule_delayed_work(void) { return (true); } /* *------------------------------------------------------------------------------ * * vmci_schedule_delayed_work -- * * Schedule the specified callback. * * Results: * Zero on success, error code otherwise. * * Side effects: * None. * *------------------------------------------------------------------------------ */ int vmci_schedule_delayed_work(vmci_work_fn *work_fn, void *data) { return (vmci_schedule_delayed_work_fn(work_fn, data)); } /* *------------------------------------------------------------------------------ * * vmci_create_event -- * * Results: * None. * * Side effects: * None. * *------------------------------------------------------------------------------ */ void vmci_create_event(vmci_event *event) { sema_init(event, 0, "vmci_event"); } /* *------------------------------------------------------------------------------ * * vmci_destroy_event -- * * Results: * None. * * Side effects: * None. * *------------------------------------------------------------------------------ */ void vmci_destroy_event(vmci_event *event) { if (mtx_owned(&event->sema_mtx)) sema_destroy(event); } /* *------------------------------------------------------------------------------ * * vmci_signal_event -- * * Results: * None. * * Side effects: * None. * *------------------------------------------------------------------------------ */ void vmci_signal_event(vmci_event *event) { sema_post(event); } /* *------------------------------------------------------------------------------ * * vmci_wait_on_event -- * * Results: * None. * * Side effects: * None. * *------------------------------------------------------------------------------ */ void vmci_wait_on_event(vmci_event *event, vmci_event_release_cb release_cb, void *client_data) { release_cb(client_data); sema_wait(event); } /* *------------------------------------------------------------------------------ * * vmci_mutex_init -- * * Initializes the mutex. Must be called before use. * * Results: * Success. * * Side effects: * None. * *------------------------------------------------------------------------------ */ int vmci_mutex_init(vmci_mutex *mutex, char *name) { mtx_init(mutex, name, NULL, MTX_DEF | MTX_NOWITNESS); return (VMCI_SUCCESS); } /* *------------------------------------------------------------------------------ * * vmci_mutex_destroy -- * * Destroys the mutex. * * Results: * None. * * Side effects: * None. * *------------------------------------------------------------------------------ */ void vmci_mutex_destroy(vmci_mutex *mutex) { mtx_destroy(mutex); } /* *------------------------------------------------------------------------------ * * vmci_mutex_acquire -- * * Acquires the mutex. * * Results: * None. * * Side effects: * Thread may block. * *------------------------------------------------------------------------------ */ void vmci_mutex_acquire(vmci_mutex *mutex) { mtx_lock(mutex); } /* *------------------------------------------------------------------------------ * * vmci_mutex_release -- * * Releases the mutex. * * Results: * None. * * Side effects: * May wake up the thread blocking on this mutex. * *------------------------------------------------------------------------------ */ void vmci_mutex_release(vmci_mutex *mutex) { mtx_unlock(mutex); } /* *------------------------------------------------------------------------------ * * vmci_mutex_initialized * * Returns whether a mutex has been initialized. * * Results: * Return 1 if initialized or 0 if unininitialized. * * Side effects: * None * *------------------------------------------------------------------------------ */ int vmci_mutex_initialized(vmci_mutex *mutex) { return mtx_initialized(mutex); } /* *------------------------------------------------------------------------------ * * vmci_alloc_queue -- * * Allocates kernel queue pages of specified size with IOMMU mappings, plus * space for the queue structure/kernel interface and the queue header. * * Results: * Pointer to the queue on success, NULL otherwise. * * Side effects: * Memory is allocated. * *------------------------------------------------------------------------------ */ void * vmci_alloc_queue(uint64_t size, uint32_t flags) { struct vmci_queue *queue; size_t i; const size_t num_pages = CEILING(size, PAGE_SIZE) + 1; const size_t dmas_size = num_pages * sizeof(struct vmci_dma_alloc); const size_t queue_size = sizeof(*queue) + sizeof(*(queue->kernel_if)) + dmas_size; /* Size should be enforced by vmci_qpair_alloc(), double-check here. */ if (size > VMCI_MAX_GUEST_QP_MEMORY) { ASSERT(false); return (NULL); } queue = malloc(queue_size, M_DEVBUF, M_NOWAIT); if (!queue) return (NULL); queue->q_header = NULL; queue->saved_header = NULL; queue->kernel_if = (struct vmci_queue_kernel_if *)(queue + 1); queue->kernel_if->num_pages = num_pages; queue->kernel_if->dmas = (struct vmci_dma_alloc *)(queue->kernel_if + 1); for (i = 0; i < num_pages; i++) { vmci_dma_malloc(PAGE_SIZE, 1, &queue->kernel_if->dmas[i]); if (!queue->kernel_if->dmas[i].dma_vaddr) { /* Size excl. the header. */ vmci_free_queue(queue, i * PAGE_SIZE); return (NULL); } } /* Queue header is the first page. */ queue->q_header = (void *)queue->kernel_if->dmas[0].dma_vaddr; return ((void *)queue); } /* *------------------------------------------------------------------------------ * * vmci_free_queue -- * * Frees kernel VA space for a given queue and its queue header, and frees * physical data pages. * * Results: * None. * * Side effects: * Memory is freed. * *------------------------------------------------------------------------------ */ void vmci_free_queue(void *q, uint64_t size) { struct vmci_queue *queue = q; if (queue) { const size_t num_pages = CEILING(size, PAGE_SIZE) + 1; uint64_t i; /* Given size doesn't include header, so add in a page here. */ for (i = 0; i < num_pages; i++) vmci_dma_free(&queue->kernel_if->dmas[i]); free(queue, M_DEVBUF); } } /* *------------------------------------------------------------------------------ * * vmci_alloc_ppn_set -- * * Allocates two list of PPNs --- one for the pages in the produce queue, * and the other for the pages in the consume queue. Intializes the list of * PPNs with the page frame numbers of the KVA for the two queues (and the * queue headers). * * Results: * Success or failure. * * Side effects: * Memory may be allocated. * *----------------------------------------------------------------------------- */ int vmci_alloc_ppn_set(void *prod_q, uint64_t num_produce_pages, void *cons_q, uint64_t num_consume_pages, struct ppn_set *ppn_set) { struct vmci_queue *consume_q = cons_q; struct vmci_queue *produce_q = prod_q; vmci_ppn_list consume_ppns; vmci_ppn_list produce_ppns; uint64_t i; if (!produce_q || !num_produce_pages || !consume_q || !num_consume_pages || !ppn_set) return (VMCI_ERROR_INVALID_ARGS); if (ppn_set->initialized) return (VMCI_ERROR_ALREADY_EXISTS); produce_ppns = vmci_alloc_kernel_mem(num_produce_pages * sizeof(*produce_ppns), VMCI_MEMORY_NORMAL); if (!produce_ppns) return (VMCI_ERROR_NO_MEM); consume_ppns = vmci_alloc_kernel_mem(num_consume_pages * sizeof(*consume_ppns), VMCI_MEMORY_NORMAL); if (!consume_ppns) { vmci_free_kernel_mem(produce_ppns, num_produce_pages * sizeof(*produce_ppns)); return (VMCI_ERROR_NO_MEM); } for (i = 0; i < num_produce_pages; i++) { unsigned long pfn; produce_ppns[i] = pfn = produce_q->kernel_if->dmas[i].dma_paddr >> PAGE_SHIFT; /* * Fail allocation if PFN isn't supported by hypervisor. */ if (sizeof(pfn) > sizeof(*produce_ppns) && pfn != produce_ppns[i]) goto ppn_error; } for (i = 0; i < num_consume_pages; i++) { unsigned long pfn; consume_ppns[i] = pfn = consume_q->kernel_if->dmas[i].dma_paddr >> PAGE_SHIFT; /* * Fail allocation if PFN isn't supported by hypervisor. */ if (sizeof(pfn) > sizeof(*consume_ppns) && pfn != consume_ppns[i]) goto ppn_error; } ppn_set->num_produce_pages = num_produce_pages; ppn_set->num_consume_pages = num_consume_pages; ppn_set->produce_ppns = produce_ppns; ppn_set->consume_ppns = consume_ppns; ppn_set->initialized = true; return (VMCI_SUCCESS); ppn_error: vmci_free_kernel_mem(produce_ppns, num_produce_pages * sizeof(*produce_ppns)); vmci_free_kernel_mem(consume_ppns, num_consume_pages * sizeof(*consume_ppns)); return (VMCI_ERROR_INVALID_ARGS); } /* *------------------------------------------------------------------------------ * * vmci_free_ppn_set -- * * Frees the two list of PPNs for a queue pair. * * Results: * None. * * Side effects: * None. * *------------------------------------------------------------------------------ */ void vmci_free_ppn_set(struct ppn_set *ppn_set) { ASSERT(ppn_set); if (ppn_set->initialized) { /* Do not call these functions on NULL inputs. */ ASSERT(ppn_set->produce_ppns && ppn_set->consume_ppns); vmci_free_kernel_mem(ppn_set->produce_ppns, ppn_set->num_produce_pages * sizeof(*ppn_set->produce_ppns)); vmci_free_kernel_mem(ppn_set->consume_ppns, ppn_set->num_consume_pages * sizeof(*ppn_set->consume_ppns)); } memset(ppn_set, 0, sizeof(*ppn_set)); } /* *------------------------------------------------------------------------------ * * vmci_populate_ppn_list -- * * Populates the list of PPNs in the hypercall structure with the PPNS * of the produce queue and the consume queue. * * Results: * VMCI_SUCCESS. * * Side effects: * None. * *------------------------------------------------------------------------------ */ int vmci_populate_ppn_list(uint8_t *call_buf, const struct ppn_set *ppn_set) { ASSERT(call_buf && ppn_set && ppn_set->initialized); memcpy(call_buf, ppn_set->produce_ppns, ppn_set->num_produce_pages * sizeof(*ppn_set->produce_ppns)); memcpy(call_buf + ppn_set->num_produce_pages * sizeof(*ppn_set->produce_ppns), ppn_set->consume_ppns, ppn_set->num_consume_pages * sizeof(*ppn_set->consume_ppns)); return (VMCI_SUCCESS); } /* *------------------------------------------------------------------------------ * * vmci_memcpy_{to,from}iovec -- * * These helper routines will copy the specified bytes to/from memory that's * specified as a struct iovec. The routines can not verify the correctness * of the struct iovec's contents. * * Results: * None. * * Side effects: * None. * *------------------------------------------------------------------------------ */ static inline void vmci_memcpy_toiovec(struct iovec *iov, uint8_t *src, size_t len) { while (len > 0) { if (iov->iov_len) { size_t to_copy = MIN(iov->iov_len, len); memcpy(iov->iov_base, src, to_copy); src += to_copy; len -= to_copy; iov->iov_base = (void *)((uintptr_t) iov->iov_base + to_copy); iov->iov_len -= to_copy; } iov++; } } static inline void vmci_memcpy_fromiovec(uint8_t *dst, struct iovec *iov, size_t len) { while (len > 0) { if (iov->iov_len) { size_t to_copy = MIN(iov->iov_len, len); memcpy(dst, iov->iov_base, to_copy); dst += to_copy; len -= to_copy; iov->iov_base = (void *)((uintptr_t) iov->iov_base + to_copy); iov->iov_len -= to_copy; } iov++; } } /* *------------------------------------------------------------------------------ * * __vmci_memcpy_to_queue -- * * Copies from a given buffer or iovector to a VMCI Queue. Assumes that * offset + size does not wrap around in the queue. * * Results: * Zero on success, negative error code on failure. * * Side effects: * None. * *------------------------------------------------------------------------------ */ #pragma GCC diagnostic ignored "-Wcast-qual" static int __vmci_memcpy_to_queue(struct vmci_queue *queue, uint64_t queue_offset, const void *src, size_t size, bool is_iovec) { struct vmci_queue_kernel_if *kernel_if = queue->kernel_if; size_t bytes_copied = 0; while (bytes_copied < size) { const uint64_t page_index = (queue_offset + bytes_copied) / PAGE_SIZE; const size_t page_offset = (queue_offset + bytes_copied) & (PAGE_SIZE - 1); void *va; size_t to_copy; /* Skip header. */ va = (void *)kernel_if->dmas[page_index + 1].dma_vaddr; ASSERT(va); /* * Fill up the page if we have enough payload, or else * copy the remaining bytes. */ to_copy = MIN(PAGE_SIZE - page_offset, size - bytes_copied); if (is_iovec) { struct iovec *iov = (struct iovec *)src; /* The iovec will track bytes_copied internally. */ vmci_memcpy_fromiovec((uint8_t *)va + page_offset, iov, to_copy); } else memcpy((uint8_t *)va + page_offset, (uint8_t *)src + bytes_copied, to_copy); bytes_copied += to_copy; } return (VMCI_SUCCESS); } /* *------------------------------------------------------------------------------ * * __vmci_memcpy_from_queue -- * * Copies to a given buffer or iovector from a VMCI Queue. Assumes that * offset + size does not wrap around in the queue. * * Results: * Zero on success, negative error code on failure. * * Side effects: * None. * *------------------------------------------------------------------------------ */ static int __vmci_memcpy_from_queue(void *dest, const struct vmci_queue *queue, uint64_t queue_offset, size_t size, bool is_iovec) { struct vmci_queue_kernel_if *kernel_if = queue->kernel_if; size_t bytes_copied = 0; while (bytes_copied < size) { const uint64_t page_index = (queue_offset + bytes_copied) / PAGE_SIZE; const size_t page_offset = (queue_offset + bytes_copied) & (PAGE_SIZE - 1); void *va; size_t to_copy; /* Skip header. */ va = (void *)kernel_if->dmas[page_index + 1].dma_vaddr; ASSERT(va); /* * Fill up the page if we have enough payload, or else * copy the remaining bytes. */ to_copy = MIN(PAGE_SIZE - page_offset, size - bytes_copied); if (is_iovec) { struct iovec *iov = (struct iovec *)dest; /* The iovec will track bytesCopied internally. */ vmci_memcpy_toiovec(iov, (uint8_t *)va + page_offset, to_copy); } else memcpy((uint8_t *)dest + bytes_copied, (uint8_t *)va + page_offset, to_copy); bytes_copied += to_copy; } return (VMCI_SUCCESS); } /* *------------------------------------------------------------------------------ * * vmci_memcpy_to_queue -- * * Copies from a given buffer to a VMCI Queue. * * Results: * Zero on success, negative error code on failure. * * Side effects: * None. * *------------------------------------------------------------------------------ */ int vmci_memcpy_to_queue(struct vmci_queue *queue, uint64_t queue_offset, const void *src, size_t src_offset, size_t size, int buf_type, bool can_block) { ASSERT(can_block); return (__vmci_memcpy_to_queue(queue, queue_offset, (uint8_t *)src + src_offset, size, false)); } /* *------------------------------------------------------------------------------ * * vmci_memcpy_from_queue -- * * Copies to a given buffer from a VMCI Queue. * * Results: * Zero on success, negative error code on failure. * * Side effects: * None. * *------------------------------------------------------------------------------ */ int vmci_memcpy_from_queue(void *dest, size_t dest_offset, const struct vmci_queue *queue, uint64_t queue_offset, size_t size, int buf_type, bool can_block) { ASSERT(can_block); return (__vmci_memcpy_from_queue((uint8_t *)dest + dest_offset, queue, queue_offset, size, false)); } /* *------------------------------------------------------------------------------ * * vmci_memcpy_to_queue_local -- * * Copies from a given buffer to a local VMCI queue. This is the * same as a regular copy. * * Results: * Zero on success, negative error code on failure. * * Side effects: * None. * *------------------------------------------------------------------------------ */ int vmci_memcpy_to_queue_local(struct vmci_queue *queue, uint64_t queue_offset, const void *src, size_t src_offset, size_t size, int buf_type, bool can_block) { ASSERT(can_block); return (__vmci_memcpy_to_queue(queue, queue_offset, (uint8_t *)src + src_offset, size, false)); } /* *------------------------------------------------------------------------------ * * vmci_memcpy_from_queue_local -- * * Copies to a given buffer from a VMCI Queue. * * Results: * Zero on success, negative error code on failure. * * Side effects: * None. * *------------------------------------------------------------------------------ */ int vmci_memcpy_from_queue_local(void *dest, size_t dest_offset, const struct vmci_queue *queue, uint64_t queue_offset, size_t size, int buf_type, bool can_block) { ASSERT(can_block); return (__vmci_memcpy_from_queue((uint8_t *)dest + dest_offset, queue, queue_offset, size, false)); } /*------------------------------------------------------------------------------ * * vmci_memcpy_to_queue_v -- * * Copies from a given iovec from a VMCI Queue. * * Results: * Zero on success, negative error code on failure. * * Side effects: * None. * *------------------------------------------------------------------------------ */ int vmci_memcpy_to_queue_v(struct vmci_queue *queue, uint64_t queue_offset, const void *src, size_t src_offset, size_t size, int buf_type, bool can_block) { ASSERT(can_block); /* * We ignore src_offset because src is really a struct iovec * and will * maintain offset internally. */ return (__vmci_memcpy_to_queue(queue, queue_offset, src, size, true)); } /* *------------------------------------------------------------------------------ * * vmci_memcpy_from_queue_v -- * * Copies to a given iovec from a VMCI Queue. * * Results: * Zero on success, negative error code on failure. * * Side effects: * None. * *------------------------------------------------------------------------------ */ int vmci_memcpy_from_queue_v(void *dest, size_t dest_offset, const struct vmci_queue *queue, uint64_t queue_offset, size_t size, int buf_type, bool can_block) { ASSERT(can_block); /* * We ignore dest_offset because dest is really a struct iovec * and * will maintain offset internally. */ return (__vmci_memcpy_from_queue(dest, queue, queue_offset, size, true)); } /* *------------------------------------------------------------------------------ * * vmci_read_port_bytes -- * * Copy memory from an I/O port to kernel memory. * * Results: * No results. * * Side effects: * None. * *------------------------------------------------------------------------------ */ void vmci_read_port_bytes(vmci_io_handle handle, vmci_io_port port, uint8_t *buffer, size_t buffer_length) { insb(port, buffer, buffer_length); }
