#include <linux/export.h>
#include <linux/kernel.h>
#include <asm/octeon/cvmx.h>
#include <asm/octeon/cvmx-spinlock.h>
#include <asm/octeon/cvmx-bootmem.h>
static struct cvmx_bootmem_desc *cvmx_bootmem_desc;
#define CVMX_BOOTMEM_NAMED_GET_FIELD(addr, field) \
__cvmx_bootmem_desc_get(addr, \
offsetof(struct cvmx_bootmem_named_block_desc, field), \
sizeof_field(struct cvmx_bootmem_named_block_desc, field))
static inline uint64_t __cvmx_bootmem_desc_get(uint64_t base, int offset,
int size)
{
base = (1ull << 63) | (base + offset);
switch (size) {
case 4:
return cvmx_read64_uint32(base);
case 8:
return cvmx_read64_uint64(base);
default:
return 0;
}
}
#define NEXT_OFFSET 0
#define SIZE_OFFSET 8
static void cvmx_bootmem_phy_set_size(uint64_t addr, uint64_t size)
{
cvmx_write64_uint64((addr + SIZE_OFFSET) | (1ull << 63), size);
}
static void cvmx_bootmem_phy_set_next(uint64_t addr, uint64_t next)
{
cvmx_write64_uint64((addr + NEXT_OFFSET) | (1ull << 63), next);
}
static uint64_t cvmx_bootmem_phy_get_size(uint64_t addr)
{
return cvmx_read64_uint64((addr + SIZE_OFFSET) | (1ull << 63));
}
static uint64_t cvmx_bootmem_phy_get_next(uint64_t addr)
{
return cvmx_read64_uint64((addr + NEXT_OFFSET) | (1ull << 63));
}
static void *cvmx_bootmem_alloc_range(uint64_t size, uint64_t alignment,
uint64_t min_addr, uint64_t max_addr)
{
int64_t address;
address =
cvmx_bootmem_phy_alloc(size, min_addr, max_addr, alignment, 0);
if (address > 0)
return cvmx_phys_to_ptr(address);
else
return NULL;
}
void *cvmx_bootmem_alloc_address(uint64_t size, uint64_t address,
uint64_t alignment)
{
return cvmx_bootmem_alloc_range(size, alignment, address,
address + size);
}
void *cvmx_bootmem_alloc_named_range(uint64_t size, uint64_t min_addr,
uint64_t max_addr, uint64_t align,
char *name)
{
int64_t addr;
addr = cvmx_bootmem_phy_named_block_alloc(size, min_addr, max_addr,
align, name, 0);
if (addr >= 0)
return cvmx_phys_to_ptr(addr);
else
return NULL;
}
void *cvmx_bootmem_alloc_named(uint64_t size, uint64_t alignment, char *name)
{
return cvmx_bootmem_alloc_named_range(size, 0, 0, alignment, name);
}
EXPORT_SYMBOL(cvmx_bootmem_alloc_named);
void cvmx_bootmem_lock(void)
{
cvmx_spinlock_lock((cvmx_spinlock_t *) &(cvmx_bootmem_desc->lock));
}
void cvmx_bootmem_unlock(void)
{
cvmx_spinlock_unlock((cvmx_spinlock_t *) &(cvmx_bootmem_desc->lock));
}
int cvmx_bootmem_init(void *mem_desc_ptr)
{
if (!cvmx_bootmem_desc) {
#if defined(CVMX_ABI_64)
cvmx_bootmem_desc = cvmx_phys_to_ptr(CAST64(mem_desc_ptr));
#else
cvmx_bootmem_desc = (struct cvmx_bootmem_desc *) mem_desc_ptr;
#endif
}
return 0;
}
int64_t cvmx_bootmem_phy_alloc(uint64_t req_size, uint64_t address_min,
uint64_t address_max, uint64_t alignment,
uint32_t flags)
{
uint64_t head_addr;
uint64_t ent_addr;
uint64_t prev_addr = 0;
uint64_t new_ent_addr = 0;
uint64_t desired_min_addr;
#ifdef DEBUG
cvmx_dprintf("cvmx_bootmem_phy_alloc: req_size: 0x%llx, "
"min_addr: 0x%llx, max_addr: 0x%llx, align: 0x%llx\n",
(unsigned long long)req_size,
(unsigned long long)address_min,
(unsigned long long)address_max,
(unsigned long long)alignment);
#endif
if (cvmx_bootmem_desc->major_version > 3) {
cvmx_dprintf("ERROR: Incompatible bootmem descriptor "
"version: %d.%d at addr: %p\n",
(int)cvmx_bootmem_desc->major_version,
(int)cvmx_bootmem_desc->minor_version,
cvmx_bootmem_desc);
goto error_out;
}
if (!req_size)
goto error_out;
req_size = (req_size + (CVMX_BOOTMEM_ALIGNMENT_SIZE - 1)) &
~(CVMX_BOOTMEM_ALIGNMENT_SIZE - 1);
if (address_min && !address_max)
address_max = address_min + req_size;
else if (!address_min && !address_max)
address_max = ~0ull;
if (alignment < CVMX_BOOTMEM_ALIGNMENT_SIZE)
alignment = CVMX_BOOTMEM_ALIGNMENT_SIZE;
if (alignment)
address_min = ALIGN(address_min, alignment);
if (req_size > address_max - address_min)
goto error_out;
if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
cvmx_bootmem_lock();
head_addr = cvmx_bootmem_desc->head_addr;
ent_addr = head_addr;
for (; ent_addr;
prev_addr = ent_addr,
ent_addr = cvmx_bootmem_phy_get_next(ent_addr)) {
uint64_t usable_base, usable_max;
uint64_t ent_size = cvmx_bootmem_phy_get_size(ent_addr);
if (cvmx_bootmem_phy_get_next(ent_addr)
&& ent_addr > cvmx_bootmem_phy_get_next(ent_addr)) {
cvmx_dprintf("Internal bootmem_alloc() error: ent: "
"0x%llx, next: 0x%llx\n",
(unsigned long long)ent_addr,
(unsigned long long)
cvmx_bootmem_phy_get_next(ent_addr));
goto error_out;
}
usable_base =
ALIGN(max(address_min, ent_addr), alignment);
usable_max = min(address_max, ent_addr + ent_size);
desired_min_addr = usable_base;
if (!((ent_addr + ent_size) > usable_base
&& ent_addr < address_max
&& req_size <= usable_max - usable_base))
continue;
if (flags & CVMX_BOOTMEM_FLAG_END_ALLOC) {
desired_min_addr = usable_max - req_size;
desired_min_addr &= ~(alignment - 1);
}
if (desired_min_addr == ent_addr) {
if (req_size < ent_size) {
new_ent_addr = ent_addr + req_size;
cvmx_bootmem_phy_set_next(new_ent_addr,
cvmx_bootmem_phy_get_next(ent_addr));
cvmx_bootmem_phy_set_size(new_ent_addr,
ent_size -
req_size);
cvmx_bootmem_phy_set_next(ent_addr,
new_ent_addr);
}
if (prev_addr)
cvmx_bootmem_phy_set_next(prev_addr,
cvmx_bootmem_phy_get_next(ent_addr));
else
cvmx_bootmem_desc->head_addr =
cvmx_bootmem_phy_get_next(ent_addr);
if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
cvmx_bootmem_unlock();
return desired_min_addr;
}
new_ent_addr = desired_min_addr;
cvmx_bootmem_phy_set_next(new_ent_addr,
cvmx_bootmem_phy_get_next
(ent_addr));
cvmx_bootmem_phy_set_size(new_ent_addr,
cvmx_bootmem_phy_get_size
(ent_addr) -
(desired_min_addr -
ent_addr));
cvmx_bootmem_phy_set_size(ent_addr,
desired_min_addr - ent_addr);
cvmx_bootmem_phy_set_next(ent_addr, new_ent_addr);
}
error_out:
if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
cvmx_bootmem_unlock();
return -1;
}
int __cvmx_bootmem_phy_free(uint64_t phy_addr, uint64_t size, uint32_t flags)
{
uint64_t cur_addr;
uint64_t prev_addr = 0;
int retval = 0;
#ifdef DEBUG
cvmx_dprintf("__cvmx_bootmem_phy_free addr: 0x%llx, size: 0x%llx\n",
(unsigned long long)phy_addr, (unsigned long long)size);
#endif
if (cvmx_bootmem_desc->major_version > 3) {
cvmx_dprintf("ERROR: Incompatible bootmem descriptor "
"version: %d.%d at addr: %p\n",
(int)cvmx_bootmem_desc->major_version,
(int)cvmx_bootmem_desc->minor_version,
cvmx_bootmem_desc);
return 0;
}
if (!size)
return 0;
if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
cvmx_bootmem_lock();
cur_addr = cvmx_bootmem_desc->head_addr;
if (cur_addr == 0 || phy_addr < cur_addr) {
if (cur_addr && phy_addr + size > cur_addr)
goto bootmem_free_done;
else if (phy_addr + size == cur_addr) {
cvmx_bootmem_phy_set_next(phy_addr,
cvmx_bootmem_phy_get_next
(cur_addr));
cvmx_bootmem_phy_set_size(phy_addr,
cvmx_bootmem_phy_get_size
(cur_addr) + size);
cvmx_bootmem_desc->head_addr = phy_addr;
} else {
cvmx_bootmem_phy_set_next(phy_addr, cur_addr);
cvmx_bootmem_phy_set_size(phy_addr, size);
cvmx_bootmem_desc->head_addr = phy_addr;
}
retval = 1;
goto bootmem_free_done;
}
while (cur_addr && phy_addr > cur_addr) {
prev_addr = cur_addr;
cur_addr = cvmx_bootmem_phy_get_next(cur_addr);
}
if (!cur_addr) {
if (prev_addr + cvmx_bootmem_phy_get_size(prev_addr) ==
phy_addr) {
cvmx_bootmem_phy_set_size(prev_addr,
cvmx_bootmem_phy_get_size
(prev_addr) + size);
} else {
cvmx_bootmem_phy_set_next(prev_addr, phy_addr);
cvmx_bootmem_phy_set_size(phy_addr, size);
cvmx_bootmem_phy_set_next(phy_addr, 0);
}
retval = 1;
goto bootmem_free_done;
} else {
if (prev_addr + cvmx_bootmem_phy_get_size(prev_addr) ==
phy_addr) {
cvmx_bootmem_phy_set_size(prev_addr,
cvmx_bootmem_phy_get_size
(prev_addr) + size);
if (phy_addr + size == cur_addr) {
cvmx_bootmem_phy_set_size(prev_addr,
cvmx_bootmem_phy_get_size(cur_addr) +
cvmx_bootmem_phy_get_size(prev_addr));
cvmx_bootmem_phy_set_next(prev_addr,
cvmx_bootmem_phy_get_next(cur_addr));
}
retval = 1;
goto bootmem_free_done;
} else if (phy_addr + size == cur_addr) {
cvmx_bootmem_phy_set_size(phy_addr,
cvmx_bootmem_phy_get_size
(cur_addr) + size);
cvmx_bootmem_phy_set_next(phy_addr,
cvmx_bootmem_phy_get_next
(cur_addr));
cvmx_bootmem_phy_set_next(prev_addr, phy_addr);
retval = 1;
goto bootmem_free_done;
}
cvmx_bootmem_phy_set_size(phy_addr, size);
cvmx_bootmem_phy_set_next(phy_addr, cur_addr);
cvmx_bootmem_phy_set_next(prev_addr, phy_addr);
}
retval = 1;
bootmem_free_done:
if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
cvmx_bootmem_unlock();
return retval;
}
static struct cvmx_bootmem_named_block_desc *
cvmx_bootmem_phy_named_block_find(char *name, uint32_t flags)
{
unsigned int i;
struct cvmx_bootmem_named_block_desc *named_block_array_ptr;
#ifdef DEBUG
cvmx_dprintf("cvmx_bootmem_phy_named_block_find: %s\n", name);
#endif
if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
cvmx_bootmem_lock();
named_block_array_ptr = (struct cvmx_bootmem_named_block_desc *)
cvmx_phys_to_ptr(cvmx_bootmem_desc->named_block_array_addr);
#ifdef DEBUG
cvmx_dprintf
("cvmx_bootmem_phy_named_block_find: named_block_array_ptr: %p\n",
named_block_array_ptr);
#endif
if (cvmx_bootmem_desc->major_version == 3) {
for (i = 0;
i < cvmx_bootmem_desc->named_block_num_blocks; i++) {
if ((name && named_block_array_ptr[i].size
&& !strncmp(name, named_block_array_ptr[i].name,
cvmx_bootmem_desc->named_block_name_len
- 1))
|| (!name && !named_block_array_ptr[i].size)) {
if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
cvmx_bootmem_unlock();
return &(named_block_array_ptr[i]);
}
}
} else {
cvmx_dprintf("ERROR: Incompatible bootmem descriptor "
"version: %d.%d at addr: %p\n",
(int)cvmx_bootmem_desc->major_version,
(int)cvmx_bootmem_desc->minor_version,
cvmx_bootmem_desc);
}
if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
cvmx_bootmem_unlock();
return NULL;
}
void *cvmx_bootmem_alloc_named_range_once(uint64_t size, uint64_t min_addr,
uint64_t max_addr, uint64_t align,
char *name,
void (*init) (void *))
{
int64_t addr;
void *ptr;
uint64_t named_block_desc_addr;
named_block_desc_addr = (uint64_t)
cvmx_bootmem_phy_named_block_find(name,
(uint32_t)CVMX_BOOTMEM_FLAG_NO_LOCKING);
if (named_block_desc_addr) {
addr = CVMX_BOOTMEM_NAMED_GET_FIELD(named_block_desc_addr,
base_addr);
return cvmx_phys_to_ptr(addr);
}
addr = cvmx_bootmem_phy_named_block_alloc(size, min_addr, max_addr,
align, name,
(uint32_t)CVMX_BOOTMEM_FLAG_NO_LOCKING);
if (addr < 0)
return NULL;
ptr = cvmx_phys_to_ptr(addr);
if (init)
init(ptr);
else
memset(ptr, 0, size);
return ptr;
}
EXPORT_SYMBOL(cvmx_bootmem_alloc_named_range_once);
struct cvmx_bootmem_named_block_desc *cvmx_bootmem_find_named_block(char *name)
{
return cvmx_bootmem_phy_named_block_find(name, 0);
}
EXPORT_SYMBOL(cvmx_bootmem_find_named_block);
static int cvmx_bootmem_phy_named_block_free(char *name, uint32_t flags)
{
struct cvmx_bootmem_named_block_desc *named_block_ptr;
if (cvmx_bootmem_desc->major_version != 3) {
cvmx_dprintf("ERROR: Incompatible bootmem descriptor version: "
"%d.%d at addr: %p\n",
(int)cvmx_bootmem_desc->major_version,
(int)cvmx_bootmem_desc->minor_version,
cvmx_bootmem_desc);
return 0;
}
#ifdef DEBUG
cvmx_dprintf("cvmx_bootmem_phy_named_block_free: %s\n", name);
#endif
cvmx_bootmem_lock();
named_block_ptr =
cvmx_bootmem_phy_named_block_find(name,
CVMX_BOOTMEM_FLAG_NO_LOCKING);
if (named_block_ptr) {
#ifdef DEBUG
cvmx_dprintf("cvmx_bootmem_phy_named_block_free: "
"%s, base: 0x%llx, size: 0x%llx\n",
name,
(unsigned long long)named_block_ptr->base_addr,
(unsigned long long)named_block_ptr->size);
#endif
__cvmx_bootmem_phy_free(named_block_ptr->base_addr,
named_block_ptr->size,
CVMX_BOOTMEM_FLAG_NO_LOCKING);
named_block_ptr->size = 0;
}
cvmx_bootmem_unlock();
return named_block_ptr != NULL;
}
int cvmx_bootmem_free_named(char *name)
{
return cvmx_bootmem_phy_named_block_free(name, 0);
}
int64_t cvmx_bootmem_phy_named_block_alloc(uint64_t size, uint64_t min_addr,
uint64_t max_addr,
uint64_t alignment,
char *name,
uint32_t flags)
{
int64_t addr_allocated;
struct cvmx_bootmem_named_block_desc *named_block_desc_ptr;
#ifdef DEBUG
cvmx_dprintf("cvmx_bootmem_phy_named_block_alloc: size: 0x%llx, min: "
"0x%llx, max: 0x%llx, align: 0x%llx, name: %s\n",
(unsigned long long)size,
(unsigned long long)min_addr,
(unsigned long long)max_addr,
(unsigned long long)alignment,
name);
#endif
if (cvmx_bootmem_desc->major_version != 3) {
cvmx_dprintf("ERROR: Incompatible bootmem descriptor version: "
"%d.%d at addr: %p\n",
(int)cvmx_bootmem_desc->major_version,
(int)cvmx_bootmem_desc->minor_version,
cvmx_bootmem_desc);
return -1;
}
if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
cvmx_spinlock_lock((cvmx_spinlock_t *)&(cvmx_bootmem_desc->lock));
named_block_desc_ptr =
cvmx_bootmem_phy_named_block_find(NULL,
flags | CVMX_BOOTMEM_FLAG_NO_LOCKING);
if (cvmx_bootmem_phy_named_block_find(name,
flags | CVMX_BOOTMEM_FLAG_NO_LOCKING) || !named_block_desc_ptr) {
if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
cvmx_spinlock_unlock((cvmx_spinlock_t *)&(cvmx_bootmem_desc->lock));
return -1;
}
size = ALIGN(size, CVMX_BOOTMEM_ALIGNMENT_SIZE);
addr_allocated = cvmx_bootmem_phy_alloc(size, min_addr, max_addr,
alignment,
flags | CVMX_BOOTMEM_FLAG_NO_LOCKING);
if (addr_allocated >= 0) {
named_block_desc_ptr->base_addr = addr_allocated;
named_block_desc_ptr->size = size;
strscpy(named_block_desc_ptr->name, name,
cvmx_bootmem_desc->named_block_name_len);
}
if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
cvmx_spinlock_unlock((cvmx_spinlock_t *)&(cvmx_bootmem_desc->lock));
return addr_allocated;
}
struct cvmx_bootmem_desc *cvmx_bootmem_get_desc(void)
{
return cvmx_bootmem_desc;
}