#include <sys/machsystm.h>
#include <sys/archsystm.h>
#include <sys/vm.h>
#include <sys/cpu.h>
#include <sys/atomic.h>
#include <sys/reboot.h>
#include <sys/kdi.h>
#include <sys/bootconf.h>
#include <sys/memlist_plat.h>
#include <sys/memlist_impl.h>
#include <sys/prom_plat.h>
#include <sys/prom_isa.h>
#include <sys/autoconf.h>
#include <sys/ivintr.h>
#include <sys/fpu/fpusystm.h>
#include <sys/iommutsb.h>
#include <vm/vm_dep.h>
#include <vm/seg_dev.h>
#include <vm/seg_kmem.h>
#include <vm/seg_kpm.h>
#include <vm/seg_map.h>
#include <vm/seg_kp.h>
#include <sys/sysconf.h>
#include <vm/hat_sfmmu.h>
#include <sys/kobj.h>
#include <sys/sun4asi.h>
#include <sys/clconf.h>
#include <sys/platform_module.h>
#include <sys/panic.h>
#include <sys/cpu_sgnblk_defs.h>
#include <sys/clock.h>
#include <sys/cmn_err.h>
#include <sys/dumphdr.h>
#include <sys/promif.h>
#include <sys/prom_debug.h>
#include <sys/traptrace.h>
#include <sys/memnode.h>
#include <sys/mem_cage.h>
#include <sys/mmu.h>
#include <sys/swap.h>
extern void setup_trap_table(void);
extern int cpu_intrq_setup(struct cpu *);
extern void cpu_intrq_register(struct cpu *);
extern void contig_mem_init(void);
extern caddr_t contig_mem_prealloc(caddr_t, pgcnt_t);
extern void mach_dump_buffer_init(void);
extern void mach_descrip_init(void);
extern void mach_descrip_startup_fini(void);
extern void mach_memscrub(void);
extern void mach_fpras(void);
extern void mach_cpu_halt_idle(void);
extern void mach_hw_copy_limit(void);
extern void load_mach_drivers(void);
extern void load_tod_module(void);
#pragma weak load_tod_module
extern int ndata_alloc_mmfsa(struct memlist *ndata);
#pragma weak ndata_alloc_mmfsa
extern void cif_init(void);
#pragma weak cif_init
extern void parse_idprom(void);
extern void add_vx_handler(char *, int, void (*)(cell_t *));
extern void mem_config_init(void);
extern void memseg_remap_init(void);
extern void mach_kpm_init(void);
extern void pcf_init();
extern int size_pse_array(pgcnt_t, int);
extern void pg_init();
extern int vac_size;
extern uint_t vac_mask;
extern uint_t vac_colors;
void *romp;
pgcnt_t physmem = 0;
pgcnt_t segkpsize =
btop(SEGKPDEFSIZE);
uint_t segmap_percent = 6;
int use_cache = 1;
int vac_copyback = 1;
char *cache_mode = NULL;
int use_mix = 1;
int prom_debug = 0;
caddr_t boot_tba;
uint_t tba_taken_over = 0;
caddr_t s_text;
caddr_t e_text;
caddr_t s_data;
caddr_t e_data;
caddr_t modtext;
size_t modtext_sz;
caddr_t moddata;
caddr_t e_moddata;
caddr_t econtig32;
caddr_t ncbase;
caddr_t ncend;
size_t ndata_remain_sz;
caddr_t nalloc_base;
caddr_t nalloc_end;
caddr_t valloc_base;
caddr_t kmem64_base;
caddr_t kmem64_end;
size_t kmem64_sz;
caddr_t kmem64_aligned_end;
int kmem64_szc;
uint64_t kmem64_pabase = (uint64_t)-1;
uintptr_t shm_alignment;
struct memlist *phys_install;
struct memlist *phys_avail;
struct memlist *virt_avail;
struct memlist *nopp_list;
struct memlist ndata;
int memexp_flag;
uint64_t ecache_flushaddr;
pgcnt_t obp_pages;
long page_hashsz;
unsigned int page_hashsz_shift;
struct page *pp_base;
size_t pp_sz;
struct page **page_hash;
pad_mutex_t *pse_mutex;
size_t pse_table_size;
int pse_shift;
struct seg ktextseg;
struct seg kvalloc;
struct seg kpseg;
struct seg ktexthole;
struct seg kmapseg;
struct seg kpmseg;
struct seg kdebugseg;
void *kpm_pp_base;
size_t kpm_pp_sz;
pgcnt_t kpm_npages;
struct seg *segkp = &kpseg;
struct seg *segkmap = &kmapseg;
struct seg *segkpm = &kpmseg;
int segzio_fromheap = 0;
caddr_t segzio_base;
pgcnt_t segziosize = 0;
int ppvm_enable = 0;
page_t *ppvm_base;
pgcnt_t ppvm_size = 0;
struct vnode kdebugvp;
const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE;
const size_t kdi_segdebugsize = SEGDEBUGSIZE;
struct seg kmem64;
struct memseg *memseg_free;
struct vnode unused_pages_vp;
size_t pagehash_sz;
uint64_t memlist_sz;
char tbr_wr_addr_inited = 0;
caddr_t mpo_heap32_buf = NULL;
size_t mpo_heap32_bufsz = 0;
static int ndata_alloc_memseg(struct memlist *, size_t);
static void memlist_new(uint64_t, uint64_t, struct memlist **);
static void memlist_add(uint64_t, uint64_t,
struct memlist **, struct memlist **);
static void kphysm_init(void);
static void kvm_init(void);
static void install_kmem64_tte(void);
static void startup_init(void);
static void startup_memlist(void);
static void startup_modules(void);
static void startup_bop_gone(void);
static void startup_vm(void);
static void startup_end(void);
static void setup_cage_params(void);
static void startup_create_io_node(void);
static pgcnt_t npages;
static struct memlist *memlist;
void *memlist_end;
static pgcnt_t bop_alloc_pages;
static caddr_t hblk_base;
uint_t hblk_alloc_dynamic = 0;
uint_t hblk1_min = H1MIN;
int thermal_powerdown_delay = 1200;
int page_relocate_ready = 0;
int kmem64_smchunks = 0;
#ifdef DEBUGGING_MEM
static int debugging_mem;
static void
printmemlist(char *title, struct memlist *list)
{
if (!debugging_mem)
return;
printf("%s\n", title);
while (list) {
prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n",
(uint32_t)(list->ml_address >> 32),
(uint32_t)list->ml_address,
(uint32_t)(list->ml_size >> 32),
(uint32_t)(list->ml_size));
list = list->ml_next;
}
}
void
printmemseg(struct memseg *memseg)
{
if (!debugging_mem)
return;
printf("memseg\n");
while (memseg) {
prom_printf("\tpage = 0x%p, epage = 0x%p, "
"pfn = 0x%x, epfn = 0x%x\n",
memseg->pages, memseg->epages,
memseg->pages_base, memseg->pages_end);
memseg = memseg->next;
}
}
#define debug_pause(str) halt((str))
#define MPRINTF(str) if (debugging_mem) prom_printf((str))
#define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a))
#define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b))
#define MPRINTF3(str, a, b, c) \
if (debugging_mem) prom_printf((str), (a), (b), (c))
#else
#define MPRINTF(str)
#define MPRINTF1(str, a)
#define MPRINTF2(str, a, b)
#define MPRINTF3(str, a, b, c)
#endif
extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base);
extern uint64_t ecache_flush_address(void);
#pragma weak load_platform_modules
#pragma weak plat_startup_memlist
#pragma weak ecache_init_scrub_flush_area
#pragma weak ecache_flush_address
int kernel_cage_enable = 1;
static void
setup_cage_params(void)
{
void (*func)(void);
func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0);
if (func != NULL) {
(*func)();
return;
}
if (kernel_cage_enable == 0) {
return;
}
kcage_range_init(phys_avail, KCAGE_DOWN, total_pages / 256);
if (kcage_on) {
cmn_err(CE_NOTE, "!Kernel Cage is ENABLED");
} else {
cmn_err(CE_NOTE, "!Kernel Cage is DISABLED");
}
}
void
startup(void)
{
startup_init();
if (&startup_platform)
startup_platform();
startup_memlist();
startup_modules();
setup_cage_params();
startup_bop_gone();
startup_vm();
startup_end();
}
struct regs sync_reg_buf;
uint64_t sync_tt;
void
sync_handler(void)
{
struct panic_trap_info ti;
int i;
for (i = 0; i < NCPU; i++) {
if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) {
cpu[i]->cpu_flags &= ~CPU_READY;
cpu[i]->cpu_flags |= CPU_QUIESCED;
CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id);
}
}
dump_plat_mincpu = 0;
if (!panicstr && (curthread->t_panic_trap == NULL)) {
ti.trap_type = sync_tt;
ti.trap_regs = &sync_reg_buf;
ti.trap_addr = NULL;
ti.trap_mmu_fsr = 0x0;
curthread->t_panic_trap = &ti;
}
if (panicstr)
CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1);
nopanicdebug = 1;
panic("sync initiated");
}
static void
startup_init(void)
{
char sync_str[] =
"warning @ warning off : sync "
"%%tl-c %%tstate h# %p x! "
"%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! "
"%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! "
"%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! "
"%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! "
"%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! "
"%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! "
"%%y h# %p l! %%tl-c %%tt h# %p x! "
"sync ; warning !";
char bp[sizeof (sync_str) + 16 * 20];
ptl1_init_cpu(&cpu0);
ppmapinit();
init_vx_handler();
(void) sprintf((char *)bp, sync_str,
(void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1,
(void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3,
(void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5,
(void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7,
(void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1,
(void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3,
(void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5,
(void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7,
(void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc,
(void *)&sync_reg_buf.r_y, (void *)&sync_tt);
prom_interpret(bp, 0, 0, 0, 0, 0);
add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler);
}
size_t
calc_pp_sz(pgcnt_t npages)
{
return (npages * sizeof (struct page));
}
size_t
calc_kpmpp_sz(pgcnt_t npages)
{
kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT;
kpm_pgsz = 1ull << kpm_pgshft;
kpm_pgoff = kpm_pgsz - 1;
kpmp2pshft = kpm_pgshft - PAGESHIFT;
kpmpnpgs = 1 << kpmp2pshft;
if (kpm_smallpages == 0) {
kpm_npages = ptokpmpr(physinstalled);
return (kpm_npages * sizeof (kpm_page_t));
} else {
kpm_npages = npages;
return (kpm_npages * sizeof (kpm_spage_t));
}
}
size_t
calc_pagehash_sz(pgcnt_t npages)
{
ASSERT(P2SAMEHIGHBIT((1 << PP_SHIFT), (sizeof (struct page))));
page_hashsz = npages / PAGE_HASHAVELEN;
page_hashsz_shift = MAX((AN_VPSHIFT + VNODE_ALIGN_LOG2 + 1),
highbit(page_hashsz));
page_hashsz = 1 << page_hashsz_shift;
return (page_hashsz * sizeof (struct page *));
}
int testkmem64_smchunks = 0;
int
alloc_kmem64(caddr_t base, caddr_t end)
{
int i;
caddr_t aligned_end = NULL;
if (testkmem64_smchunks)
return (1);
ASSERT(mmu_exported_pagesize_mask & (1 << TTE8K));
ASSERT(IS_P2ALIGNED(base, TTEBYTES(max_bootlp_tteszc)));
for (i = max_bootlp_tteszc; i >= TTE8K; i--) {
size_t alloc_size, alignsize;
#if !defined(C_OBP)
unsigned long long pa;
#endif
if ((mmu_exported_pagesize_mask & (1 << i)) == 0)
continue;
alignsize = TTEBYTES(i);
kmem64_szc = i;
if (mmu_btop(alignsize) > (npages >> 2))
continue;
aligned_end = (caddr_t)roundup((uintptr_t)end, alignsize);
alloc_size = aligned_end - base;
#if !defined(C_OBP)
if (prom_allocate_phys(alloc_size, alignsize, &pa) == 0) {
if (prom_claim_virt(alloc_size, base) != (caddr_t)-1) {
kmem64_pabase = pa;
kmem64_aligned_end = aligned_end;
install_kmem64_tte();
break;
} else {
prom_free_phys(alloc_size, pa);
}
}
#else
if (prom_alloc(base, alloc_size, alignsize) == base) {
kmem64_pabase = va_to_pa(kmem64_base);
kmem64_aligned_end = aligned_end;
break;
}
#endif
if (i == TTE8K) {
#ifdef sun4v
return (1);
#else
prom_panic("kmem64 allocation failure");
#endif
}
}
ASSERT(aligned_end != NULL);
return (0);
}
static prom_memlist_t *boot_physinstalled, *boot_physavail, *boot_virtavail;
static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len;
#if !defined(C_OBP)
char kmem64_obp_str[] =
"h# %lx constant kmem64-base "
"h# %lx constant kmem64-end "
"h# %lx constant kmem64-pagemask "
"h# %lx constant kmem64-template "
": kmem64-tte ( addr cnum -- false | tte-data true ) "
" if ( addr ) "
" drop false exit then ( false ) "
" dup kmem64-base kmem64-end within if ( addr ) "
" kmem64-pagemask and ( addr' ) "
" kmem64-base - ( addr' ) "
" kmem64-template + ( tte ) "
" true ( tte true ) "
" else ( addr ) "
" pgmap@ ( tte ) "
" dup 0< if true else drop false then ( tte true | false ) "
" then ( tte true | false ) "
"; "
"' kmem64-tte is va>tte-data "
;
static void
install_kmem64_tte()
{
char b[sizeof (kmem64_obp_str) + (4 * 16)];
tte_t tte;
PRM_DEBUG(kmem64_pabase);
PRM_DEBUG(kmem64_szc);
sfmmu_memtte(&tte, kmem64_pabase >> MMU_PAGESHIFT,
PROC_DATA | HAT_NOSYNC, kmem64_szc);
PRM_DEBUG(tte.ll);
(void) sprintf(b, kmem64_obp_str,
kmem64_base, kmem64_end, TTE_PAGEMASK(kmem64_szc), tte.ll);
ASSERT(strlen(b) < sizeof (b));
prom_interpret(b, 0, 0, 0, 0, 0);
}
#endif
#define MINMOVE_RAM_MB ((size_t)1900)
#define MB_TO_BYTES(mb) ((mb) * 1048576ul)
#define BYTES_TO_MB(b) ((b) / 1048576ul)
pgcnt_t tune_npages = (pgcnt_t)
(MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE);
#pragma weak page_set_colorequiv_arr_cpu
extern void page_set_colorequiv_arr_cpu(void);
extern void page_set_colorequiv_arr(void);
static pgcnt_t ramdisk_npages;
static struct memlist *old_phys_avail;
kcage_dir_t kcage_startup_dir = KCAGE_DOWN;
static void
startup_memlist(void)
{
size_t hmehash_sz, pagelist_sz, tt_sz;
size_t psetable_sz;
caddr_t alloc_base;
caddr_t memspace;
struct memlist *cur;
size_t syslimit = (size_t)SYSLIMIT;
size_t sysbase = (size_t)SYSBASE;
moddata = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE);
e_moddata = moddata + MODDATA;
nalloc_base = e_moddata;
nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M);
valloc_base = nalloc_base;
if (((uintptr_t)e_moddata & MMU_PAGEMASK4M) != (uintptr_t)s_data)
prom_panic("nucleus data overflow");
PRM_DEBUG(moddata);
PRM_DEBUG(nalloc_base);
PRM_DEBUG(nalloc_end);
PRM_DEBUG(e_text);
modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE);
if (((uintptr_t)e_text & MMU_PAGEMASK4M) != (uintptr_t)s_text)
prom_panic("nucleus text overflow");
modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) -
modtext;
PRM_DEBUG(modtext);
PRM_DEBUG(modtext_sz);
init_boot_memlists();
copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
&boot_physavail, &boot_physavail_len,
&boot_virtavail, &boot_virtavail_len);
installed_top_size_memlist_array(boot_physinstalled,
boot_physinstalled_len, &physmax, &physinstalled);
PRM_DEBUG(physinstalled);
PRM_DEBUG(physmax);
startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len);
npages = physinstalled - btop(MMU_PAGESIZE4M);
ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end);
if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0))
cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc");
if (ndata_alloc_tsbs(&ndata, npages) != 0)
cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc");
if (ndata_alloc_dmv(&ndata) != 0)
cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc");
if (ndata_alloc_page_mutexs(&ndata) != 0)
cmn_err(CE_PANIC,
"no more nucleus memory after page free lists alloc");
if (ndata_alloc_hat(&ndata) != 0)
cmn_err(CE_PANIC, "no more nucleus memory after hat alloc");
if (ndata_alloc_memseg(&ndata, boot_physavail_len) != 0)
cmn_err(CE_PANIC, "no more nucleus memory after memseg alloc");
alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE);
PRM_DEBUG(alloc_base);
alloc_base = sfmmu_ktsb_alloc(alloc_base);
alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
PRM_DEBUG(alloc_base);
alloc_base = iommu_tsb_init(alloc_base);
alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
PRM_DEBUG(alloc_base);
alloc_base = contig_mem_prealloc(alloc_base, npages);
if (&plat_startup_memlist) {
alloc_base = plat_startup_memlist(alloc_base);
alloc_base = (caddr_t)roundup((uintptr_t)alloc_base,
ecache_alignsize);
PRM_DEBUG(alloc_base);
}
econtig32 = alloc_base;
PRM_DEBUG(econtig32);
if (econtig32 > (caddr_t)KERNEL_LIMIT32)
cmn_err(CE_PANIC, "econtig32 too big");
pp_sz = calc_pp_sz(npages);
PRM_DEBUG(pp_sz);
if (kpm_enable) {
kpm_pp_sz = calc_kpmpp_sz(npages);
PRM_DEBUG(kpm_pp_sz);
}
hmehash_sz = calc_hmehash_sz(npages);
PRM_DEBUG(hmehash_sz);
pagehash_sz = calc_pagehash_sz(npages);
PRM_DEBUG(pagehash_sz);
pagelist_sz = calc_free_pagelist_sz();
PRM_DEBUG(pagelist_sz);
#ifdef TRAPTRACE
tt_sz = calc_traptrace_sz();
PRM_DEBUG(tt_sz);
#else
tt_sz = 0;
#endif
pse_shift = size_pse_array(npages, max_ncpus);
PRM_DEBUG(pse_shift);
pse_table_size = 1 << pse_shift;
PRM_DEBUG(pse_table_size);
psetable_sz = roundup(
pse_table_size * sizeof (pad_mutex_t), ecache_alignsize);
PRM_DEBUG(psetable_sz);
kmem64_sz = pp_sz + kpm_pp_sz + hmehash_sz + pagehash_sz +
pagelist_sz + tt_sz + psetable_sz;
kmem64_sz = roundup(kmem64_sz, PAGESIZE);
kmem64_base = (caddr_t)syslimit;
kmem64_end = kmem64_base + kmem64_sz;
if (alloc_kmem64(kmem64_base, kmem64_end)) {
size_t tmp_size;
caddr_t tmp_base;
pp_sz = roundup(pp_sz, PAGESIZE);
tmp_base = kmem64_base + pp_sz;
tmp_size = roundup(kpm_pp_sz + hmehash_sz + pagehash_sz +
pagelist_sz + tt_sz + psetable_sz, PAGESIZE);
if (prom_alloc(tmp_base, tmp_size, PAGESIZE) == 0)
prom_panic("kmem64 prom_alloc contig failed");
PRM_DEBUG(tmp_base);
PRM_DEBUG(tmp_size);
if (bop_alloc_chunk(kmem64_base, pp_sz, PAGESIZE) == 0)
prom_panic("kmem64 bop_alloc_chunk page_t failed");
PRM_DEBUG(kmem64_base);
PRM_DEBUG(pp_sz);
kmem64_aligned_end = kmem64_base + pp_sz + tmp_size;
ASSERT(kmem64_aligned_end >= kmem64_end);
kmem64_smchunks = 1;
} else {
if (kpm_smallpages == 0) {
npages -= kmem64_sz / (PAGESIZE + sizeof (struct page));
} else {
npages -= kmem64_sz / (PAGESIZE + sizeof (struct page) +
sizeof (kpm_spage_t));
}
pp_sz = npages * sizeof (struct page);
}
if (kmem64_aligned_end > (hole_start ? hole_start : kpm_vbase))
cmn_err(CE_PANIC, "not enough kmem64 space");
PRM_DEBUG(kmem64_base);
PRM_DEBUG(kmem64_end);
PRM_DEBUG(kmem64_aligned_end);
alloc_base = kmem64_base;
pp_base = (page_t *)alloc_base;
alloc_base += pp_sz;
alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
PRM_DEBUG(pp_base);
PRM_DEBUG(npages);
if (kpm_enable) {
kpm_pp_base = alloc_base;
if (kpm_smallpages == 0) {
kpm_pp_sz = kpm_npages * sizeof (kpm_page_t);
} else {
kpm_npages = ptokpmpr(npages);
kpm_pp_sz = kpm_npages * sizeof (kpm_spage_t);
}
alloc_base += kpm_pp_sz;
alloc_base =
(caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
PRM_DEBUG(kpm_pp_base);
}
alloc_base = alloc_hmehash(alloc_base);
alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
PRM_DEBUG(alloc_base);
page_hash = (page_t **)alloc_base;
alloc_base += pagehash_sz;
alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
PRM_DEBUG(page_hash);
alloc_base = alloc_page_freelists(alloc_base);
alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
PRM_DEBUG(alloc_base);
#ifdef TRAPTRACE
ttrace_buf = alloc_base;
alloc_base += tt_sz;
alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
PRM_DEBUG(alloc_base);
#endif
pse_mutex = (pad_mutex_t *)alloc_base;
alloc_base += psetable_sz;
alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
PRM_DEBUG(alloc_base);
if (kmem64_smchunks) {
kmem64_end = kmem64_aligned_end;
} else {
kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, PAGESIZE);
}
kmem64_sz = kmem64_end - kmem64_base;
if (&ecache_init_scrub_flush_area) {
alloc_base = ecache_init_scrub_flush_area(kmem64_aligned_end);
ASSERT(alloc_base <= (hole_start ? hole_start : kpm_vbase));
}
if (physmem == 0 || physmem > npages)
physmem = npages;
if (root_is_ramdisk == B_TRUE) {
ramdisk_npages = (ramdisk_size * 1024) / PAGESIZE;
physmem -= ramdisk_npages;
}
if (kpm_enable && (ndata_alloc_kpm(&ndata, kpm_npages) != 0))
cmn_err(CE_PANIC, "no more nucleus memory after kpm alloc");
memspace = prom_alloc((caddr_t)intr_vec_table, IVSIZE, MMU_PAGESIZE);
if (memspace != (caddr_t)intr_vec_table)
prom_panic("interrupt vector table allocation failure");
memlist_sz = sizeof (struct memlist) * (prom_phys_installed_len() +
prom_phys_avail_len() + prom_virt_avail_len());
memlist_sz *= 2;
memlist_sz = roundup(memlist_sz, PAGESIZE);
memspace = ndata_alloc(&ndata, memlist_sz, ecache_alignsize);
if (memspace == NULL)
cmn_err(CE_PANIC, "no more nucleus memory after memlist alloc");
memlist = (struct memlist *)memspace;
memlist_end = (char *)memspace + memlist_sz;
PRM_DEBUG(memlist);
PRM_DEBUG(memlist_end);
PRM_DEBUG(sysbase);
PRM_DEBUG(syslimit);
kernelheap_init((void *)sysbase, (void *)syslimit,
(caddr_t)sysbase + PAGESIZE, NULL, NULL);
copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
&boot_physavail, &boot_physavail_len,
&boot_virtavail, &boot_virtavail_len);
virt_avail = memlist;
copy_memlist(boot_virtavail, boot_virtavail_len, &memlist);
for (cur = virt_avail; cur->ml_next; cur = cur->ml_next) {
uint64_t range_base, range_size;
if ((range_base = cur->ml_address + cur->ml_size) <
(uint64_t)sysbase)
continue;
if (range_base >= (uint64_t)syslimit)
break;
range_size = MIN(cur->ml_next->ml_address,
(uint64_t)syslimit) - range_base;
(void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE,
0, 0, (void *)range_base, (void *)(range_base + range_size),
VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
}
phys_avail = memlist;
copy_memlist(boot_physavail, boot_physavail_len, &memlist);
nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE, nalloc_end);
if (nalloc_base == NULL)
nalloc_base = nalloc_end;
ndata_remain_sz = nalloc_end - nalloc_base;
phys_install = memlist;
copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist);
nopp_list = memlist;
memlist_new(va_to_pa(s_text), MMU_PAGESIZE4M, &memlist);
memlist_add(va_to_pa(s_data), MMU_PAGESIZE4M - ndata_remain_sz,
&memlist, &nopp_list);
if (!kmem64_smchunks)
memlist_add(kmem64_pabase, kmem64_sz, &memlist, &nopp_list);
if ((caddr_t)memlist > (memspace + memlist_sz))
prom_panic("memlist overflow");
pcf_init();
kphysm_init();
availrmem_initial = availrmem = freemem;
PRM_DEBUG(availrmem);
page_lock_init();
kmem_init();
if (&page_set_colorequiv_arr_cpu != NULL)
page_set_colorequiv_arr_cpu();
else
page_set_colorequiv_arr();
bp_init(shm_alignment, HAT_STRICTORDER);
if (mpo_heap32_bufsz > (size_t)0) {
(void) vmem_xalloc(heap32_arena, mpo_heap32_bufsz,
PAGESIZE, 0, 0, mpo_heap32_buf,
mpo_heap32_buf + mpo_heap32_bufsz,
VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
}
mem_config_init();
}
static void
startup_modules(void)
{
int nhblk1, nhblk8;
size_t nhblksz;
pgcnt_t pages_per_hblk;
size_t hme8blk_sz, hme1blk_sz;
if (&set_platform_defaults)
set_platform_defaults();
param_calc(0);
mod_setup();
param_init();
maxmem = physmem;
ncbase = kdi_segdebugbase;
ncend = kdi_segdebugbase;
hat_init();
seg_init();
create_va_to_tte();
forthdebug_init();
startup_create_io_node();
if (modloadonly("fs", "specfs") == -1)
halt("Can't load specfs");
if (modloadonly("fs", "devfs") == -1)
halt("Can't load devfs");
if (modloadonly("fs", "procfs") == -1)
halt("Can't load procfs");
if (modloadonly("misc", "swapgeneric") == -1)
halt("Can't load swapgeneric");
(void) modloadonly("sys", "lbl_edition");
dispinit();
parse_idprom();
clconf_init();
setup_ddi();
if (loadrootmodules() != 0)
debug_enter("Can't load the root filesystem");
if (&load_tod_module)
load_tod_module();
if (&load_platform_modules)
load_platform_modules();
setcpudelay();
copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
&boot_physavail, &boot_physavail_len,
&boot_virtavail, &boot_virtavail_len);
bop_alloc_pages = btopr(kmem64_end - kmem64_base);
pages_per_hblk = btop(HMEBLK_SPAN(TTE64K));
bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk);
nhblk1 = bop_alloc_pages / pages_per_hblk + hblk1_min;
bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len);
hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t));
hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t));
bop_alloc_pages += btopr(nhblk1 * hme1blk_sz);
pages_per_hblk = btop(HMEBLK_SPAN(TTE8K));
nhblk8 = 0;
while (bop_alloc_pages > 1) {
bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk);
nhblk8 += bop_alloc_pages /= pages_per_hblk;
bop_alloc_pages *= hme8blk_sz;
bop_alloc_pages = btopr(bop_alloc_pages);
}
nhblk8 += 2;
nhblk8 += boot_virtavail_len;
nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz;
nhblksz = roundup(nhblksz, MMU_PAGESIZE);
hblk_base = kmem_zalloc(nhblksz, KM_SLEEP);
sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1);
}
static void
startup_bop_gone(void)
{
mach_descrip_startup_fini();
bop_fini();
copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
&boot_physavail, &boot_physavail_len,
&boot_virtavail, &boot_virtavail_len);
if (&ecache_flush_address) {
ecache_flushaddr = ecache_flush_address();
if (ecache_flushaddr == (uint64_t)-1) {
cmn_err(CE_PANIC,
"startup: no memory to set ecache_flushaddr");
}
}
ASSERT(virt_avail != NULL);
memlist_free_list(virt_avail);
virt_avail = memlist;
copy_memlist(boot_virtavail, boot_virtavail_len, &memlist);
}
void
startup_fixup_physavail(void)
{
struct memlist *cur;
size_t kmem64_overmap_size = kmem64_aligned_end - kmem64_end;
PRM_DEBUG(kmem64_overmap_size);
copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
&boot_physavail, &boot_physavail_len,
&boot_virtavail, &boot_virtavail_len);
cur = memlist;
copy_memlist(boot_physavail, boot_physavail_len, &memlist);
if (kmem64_overmap_size) {
memlist_add(kmem64_pabase + (kmem64_end - kmem64_base),
kmem64_overmap_size, &memlist, &cur);
}
if (ndata_remain_sz >= MMU_PAGESIZE)
memlist_add(va_to_pa(nalloc_base),
(uint64_t)ndata_remain_sz, &memlist, &cur);
if ((caddr_t)memlist > (caddr_t)memlist_end)
cmn_err(CE_PANIC, "startup: memlist size exceeded");
sync_memlists(phys_avail, cur);
ASSERT(phys_avail != NULL);
old_phys_avail = phys_avail;
phys_avail = cur;
}
void
update_kcage_ranges(uint64_t addr, uint64_t len)
{
pfn_t base = btop(addr);
pgcnt_t num = btop(len);
int rv;
rv = kcage_range_add(base, num, kcage_startup_dir);
if (rv == ENOMEM) {
cmn_err(CE_WARN, "%ld megabytes not available to kernel cage",
(len == 0 ? 0 : BYTES_TO_MB(len)));
} else if (rv != 0) {
ASSERT(0);
cmn_err(CE_WARN, "unexpected kcage_range_add"
" return value %d", rv);
}
}
static void
startup_vm(void)
{
size_t i;
struct segmap_crargs a;
struct segkpm_crargs b;
uint64_t avmem;
caddr_t va;
pgcnt_t max_phys_segkp;
int mnode;
extern int use_brk_lpg, use_stk_lpg;
hat_kern_setup();
setup_trap_table();
install_va_to_tte();
tba_taken_over = 1;
mmu_init_kcontext();
CPUSET_ADD(cpu_ready_set, CPU->cpu_id);
CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS);
tbr_wr_addr_inited = 1;
kvm_init();
ASSERT(old_phys_avail != NULL && phys_avail != NULL);
if (kernel_cage_enable) {
diff_memlists(phys_avail, old_phys_avail, update_kcage_ranges);
}
memlist_free_list(old_phys_avail);
if (physmem + ramdisk_npages < npages) {
pgcnt_t diff, off;
struct page *pp;
struct seg kseg;
cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem);
off = 0;
diff = npages - (physmem + ramdisk_npages);
diff -= mmu_btopr(diff * sizeof (struct page));
kseg.s_as = &kas;
while (diff--) {
pp = page_create_va(&unused_pages_vp, (offset_t)off,
MMU_PAGESIZE, PG_WAIT | PG_EXCL,
&kseg, (caddr_t)off);
if (pp == NULL)
cmn_err(CE_PANIC, "limited physmem too much!");
page_io_unlock(pp);
page_downgrade(pp);
availrmem--;
off += MMU_PAGESIZE;
}
}
cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n",
(ulong_t)(physinstalled) << (PAGESHIFT - 10),
(ulong_t)(physinstalled) << (PAGESHIFT - 12));
avmem = (uint64_t)freemem << PAGESHIFT;
cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem);
if (physmem < privm_lpg_min_physmem) {
use_brk_lpg = 0;
use_stk_lpg = 0;
}
if (&plat_freelist_process) {
for (mnode = 0; mnode < max_mem_nodes; mnode++)
if (mem_node_config[mnode].exists)
plat_freelist_process(mnode);
}
va = (caddr_t)SEGKPBASE;
ASSERT(((uintptr_t)va & PAGEOFFSET) == 0);
max_phys_segkp = (physmem * 2);
if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) {
segkpsize = btop(SEGKPDEFSIZE);
cmn_err(CE_WARN, "Illegal value for segkpsize. "
"segkpsize has been reset to %ld pages", segkpsize);
}
i = ptob(MIN(segkpsize, max_phys_segkp));
rw_enter(&kas.a_lock, RW_WRITER);
if (seg_attach(&kas, va, i, segkp) < 0)
cmn_err(CE_PANIC, "startup: cannot attach segkp");
if (segkp_create(segkp) != 0)
cmn_err(CE_PANIC, "startup: segkp_create failed");
rw_exit(&kas.a_lock);
segmap_kpm = kpm_enable &&
segmap_kpm && PAGESIZE == MAXBSIZE;
if (kpm_enable) {
rw_enter(&kas.a_lock, RW_WRITER);
if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors,
segkpm) < 0)
cmn_err(CE_PANIC, "cannot attach segkpm");
b.prot = PROT_READ | PROT_WRITE;
b.nvcolors = shm_alignment >> MMU_PAGESHIFT;
if (segkpm_create(segkpm, (caddr_t)&b) != 0)
panic("segkpm_create segkpm");
rw_exit(&kas.a_lock);
mach_kpm_init();
}
va = kpm_vbase + (kpm_size * vac_colors);
if (!segzio_fromheap) {
size_t size;
size_t physmem_b = mmu_ptob(physmem);
if (segziosize == 0) {
size = physmem_b;
} else {
size = mmu_ptob(segziosize);
}
if (size < SEGZIOMINSIZE) {
size = SEGZIOMINSIZE;
} else if (size > SEGZIOMAXSIZE) {
size = SEGZIOMAXSIZE;
if (physmem_b > SEGZIOMAXSIZE) {
size += (physmem_b - SEGZIOMAXSIZE) / 2;
}
}
segziosize = mmu_btop(roundup(size, MMU_PAGESIZE));
segzio_base = va;
va += mmu_ptob(segziosize);
PRM_DEBUG(segziosize);
PRM_DEBUG(segzio_base);
rw_enter(&kas.a_lock, RW_WRITER);
(void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize),
&kzioseg);
(void) segkmem_create(&kzioseg);
segkmem_zio_init(segzio_base, mmu_ptob(segziosize));
rw_exit(&kas.a_lock);
}
if (ppvm_enable) {
caddr_t ppvm_max;
ppvm_size =
roundup(mmu_btop(kpm_size * vac_colors) * sizeof (page_t),
MMU_PAGESIZE);
ppvm_max = (caddr_t)(0ull - ppvm_size);
ppvm_base = (page_t *)va;
if ((caddr_t)ppvm_base <= hole_end) {
cmn_err(CE_WARN,
"Memory DR disabled: invalid DR map base: 0x%p\n",
(void *)ppvm_base);
ppvm_enable = 0;
} else if ((caddr_t)ppvm_base > ppvm_max) {
uint64_t diff = (caddr_t)ppvm_base - ppvm_max;
cmn_err(CE_WARN,
"Memory DR disabled: insufficient DR map size:"
" 0x%lx (needed 0x%lx)\n",
ppvm_size - diff, ppvm_size);
ppvm_enable = 0;
}
PRM_DEBUG(ppvm_size);
PRM_DEBUG(ppvm_base);
}
va = (caddr_t)SEGMAPBASE;
ASSERT(((uintptr_t)va & MAXBOFFSET) == 0);
i = mmu_ptob((freemem * segmap_percent) / 100);
if (i < MINMAPSIZE)
i = MINMAPSIZE;
if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem)))
i = MIN(SEGMAPSIZE, mmu_ptob(freemem));
i &= MAXBMASK;
rw_enter(&kas.a_lock, RW_WRITER);
if (seg_attach(&kas, va, i, segkmap) < 0)
cmn_err(CE_PANIC, "cannot attach segkmap");
a.prot = PROT_READ | PROT_WRITE;
a.shmsize = shm_alignment;
a.nfreelist = 0;
if (segmap_create(segkmap, (caddr_t)&a) != 0)
panic("segmap_create segkmap");
rw_exit(&kas.a_lock);
segdev_init();
}
static void
startup_end(void)
{
if ((caddr_t)memlist > (caddr_t)memlist_end)
panic("memlist overflow 2");
memlist_free_block((caddr_t)memlist,
((caddr_t)memlist_end - (caddr_t)memlist));
memlist = NULL;
page_relocate_ready = 1;
kern_setup1();
kcpc_hw_init();
contig_mem_init();
mach_descrip_init();
if (cpu_intrq_setup(CPU)) {
cmn_err(CE_PANIC, "cpu%d: setup failed", CPU->cpu_id);
}
cpu_intrq_register(CPU);
mach_htraptrace_setup(CPU->cpu_id);
mach_htraptrace_configure(CPU->cpu_id);
mach_dump_buffer_init();
cpu_intr_alloc(CPU, NINTR_THREADS);
(void) splzs();
error_init();
mach_hw_copy_limit();
(void) prom_set_preprom(kern_preprom);
(void) prom_set_postprom(kern_postprom);
CPU->cpu_m.mutex_ready = 1;
segnf_init();
configure();
mach_cpu_halt_idle();
}
void
post_startup(void)
{
#ifdef PTL1_PANIC_DEBUG
extern void init_ptl1_thread(void);
#endif
extern void abort_sequence_init(void);
bind_hwcap();
mach_memscrub();
abort_sequence_init();
(void) mod_sysctl(SYS_FORCELOAD, NULL);
(void) modload("fs", "procfs");
load_mach_drivers();
if (&load_platform_drivers)
load_platform_drivers();
if (!fpu_exists) {
if (modload("misc", "vis") == -1)
halt("Can't load vis");
}
mach_fpras();
maxmem = freemem;
pg_init();
#ifdef PTL1_PANIC_DEBUG
init_ptl1_thread();
#endif
}
#ifdef PTL1_PANIC_DEBUG
int ptl1_panic_test = 0;
int ptl1_panic_xc_one_test = 0;
int ptl1_panic_xc_all_test = 0;
int ptl1_panic_xt_one_test = 0;
int ptl1_panic_xt_all_test = 0;
kthread_id_t ptl1_thread_p = NULL;
kcondvar_t ptl1_cv;
kmutex_t ptl1_mutex;
int ptl1_recurse_count_threshold = 0x40;
int ptl1_recurse_trap_threshold = 0x3d;
extern void ptl1_recurse(int, int);
extern void ptl1_panic_xt(int, int);
static void
ptl1_wakeup(void *arg)
{
mutex_enter(&ptl1_mutex);
cv_signal(&ptl1_cv);
mutex_exit(&ptl1_mutex);
}
static void
ptl1_panic_xc(void)
{
ptl1_recurse(ptl1_recurse_count_threshold,
ptl1_recurse_trap_threshold);
}
static void
ptl1_thread(void)
{
mutex_enter(&ptl1_mutex);
while (ptl1_thread_p) {
cpuset_t other_cpus;
int cpu_id;
int my_cpu_id;
int target_cpu_id;
int target_found;
if (ptl1_panic_test) {
ptl1_recurse(ptl1_recurse_count_threshold,
ptl1_recurse_trap_threshold);
}
kpreempt_disable();
my_cpu_id = CPU->cpu_id;
other_cpus = cpu_ready_set;
CPUSET_DEL(other_cpus, CPU->cpu_id);
target_found = 0;
if (!CPUSET_ISNULL(other_cpus)) {
for (cpu_id = 0; cpu_id < NCPU; cpu_id++) {
if (cpu_id == my_cpu_id)
continue;
if (CPU_XCALL_READY(cpu_id)) {
target_cpu_id = cpu_id;
target_found = 1;
break;
}
}
ASSERT(target_found);
if (ptl1_panic_xc_one_test) {
xc_one(target_cpu_id,
(xcfunc_t *)ptl1_panic_xc, 0, 0);
}
if (ptl1_panic_xc_all_test) {
xc_some(other_cpus,
(xcfunc_t *)ptl1_panic_xc, 0, 0);
}
if (ptl1_panic_xt_one_test) {
xt_one(target_cpu_id,
(xcfunc_t *)ptl1_panic_xt, 0, 0);
}
if (ptl1_panic_xt_all_test) {
xt_some(other_cpus,
(xcfunc_t *)ptl1_panic_xt, 0, 0);
}
}
kpreempt_enable();
(void) timeout(ptl1_wakeup, NULL, hz);
(void) cv_wait(&ptl1_cv, &ptl1_mutex);
}
mutex_exit(&ptl1_mutex);
}
void
init_ptl1_thread(void)
{
ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0,
&p0, TS_RUN, 0);
}
#endif
static void
memlist_new(uint64_t start, uint64_t len, struct memlist **memlistp)
{
struct memlist *new;
new = *memlistp;
new->ml_address = start;
new->ml_size = len;
*memlistp = new + 1;
}
static void
memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp,
struct memlist **curmemlistp)
{
struct memlist *new = *memlistp;
memlist_new(start, len, memlistp);
memlist_insert(new, curmemlistp);
}
static int
ndata_alloc_memseg(struct memlist *ndata, size_t avail)
{
int nseg;
size_t memseg_sz;
struct memseg *msp;
memseg_sz = (avail + 10) * sizeof (struct memseg);
memseg_sz = roundup(memseg_sz, PAGESIZE);
nseg = memseg_sz / sizeof (struct memseg);
msp = ndata_alloc(ndata, memseg_sz, ecache_alignsize);
if (msp == NULL)
return (1);
PRM_DEBUG(memseg_free);
while (nseg--) {
msp->next = memseg_free;
memseg_free = msp;
msp++;
}
return (0);
}
static void
memseg_list_add(struct memseg *memsegp)
{
struct memseg **prev_memsegp;
pgcnt_t num;
num = MSEG_NPAGES(memsegp);
for (prev_memsegp = &memsegs; *prev_memsegp;
prev_memsegp = &((*prev_memsegp)->next)) {
if (num > MSEG_NPAGES(*prev_memsegp))
break;
}
memsegp->next = *prev_memsegp;
*prev_memsegp = memsegp;
if (kpm_enable) {
memsegp->nextpa = (memsegp->next) ?
va_to_pa(memsegp->next) : MSEG_NULLPTR_PA;
if (prev_memsegp != &memsegs) {
struct memseg *msp;
msp = (struct memseg *)((caddr_t)prev_memsegp -
offsetof(struct memseg, next));
msp->nextpa = va_to_pa(memsegp);
} else {
memsegspa = va_to_pa(memsegs);
}
}
}
#define PREFETCH_BYTES 64
void
add_physmem_cb(page_t *pp, pfn_t pnum)
{
extern void prefetch_page_w(void *);
pp->p_pagenum = pnum;
ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES);
prefetch_page_w((char *)pp);
}
static struct memseg *
memseg_find(pfn_t base, pfn_t *next)
{
struct memseg *seg;
if (next != NULL)
*next = LONG_MAX;
for (seg = memsegs; seg != NULL; seg = seg->next) {
if (base >= seg->pages_base && base < seg->pages_end)
return (seg);
if (next != NULL && seg->pages_base > base &&
seg->pages_base < *next)
*next = seg->pages_base;
}
return (NULL);
}
static void
kphysm_erase(uint64_t addr, uint64_t len)
{
struct page *pp;
struct memseg *seg;
pfn_t base = btop(addr), next;
pgcnt_t num = btop(len);
while (num != 0) {
pgcnt_t off, left;
seg = memseg_find(base, &next);
if (seg == NULL) {
if (next == LONG_MAX)
break;
left = MIN(next - base, num);
base += left, num -= left;
continue;
}
off = base - seg->pages_base;
pp = seg->pages + off;
left = num - MIN(num, (seg->pages_end - seg->pages_base) - off);
while (num != left) {
PP_SETNORELOC(pp);
add_physmem_cb(pp, base);
if (page_trylock(pp, SE_EXCL) == 0)
cmn_err(CE_PANIC, "prom page locked");
(void) page_hashin(pp, &promvp,
(offset_t)base, NULL);
(void) page_pp_lock(pp, 0, 1);
pp++, base++, num--;
}
}
}
static page_t *ppnext;
static pgcnt_t ppleft;
static void *kpm_ppnext;
static pgcnt_t kpm_ppleft;
static void
kphysm_memseg(uint64_t addr, uint64_t len)
{
pfn_t base = btop(addr);
pgcnt_t num = btop(len);
struct memseg *seg;
seg = memseg_free;
memseg_free = seg->next;
ASSERT(seg != NULL);
seg->pages = ppnext;
seg->epages = ppnext + num;
seg->pages_base = base;
seg->pages_end = base + num;
ppnext += num;
ppleft -= num;
if (kpm_enable) {
pgcnt_t kpnum = ptokpmpr(num);
if (kpnum > kpm_ppleft)
panic("kphysm_memseg: kpm_pp overflow");
seg->pagespa = va_to_pa(seg->pages);
seg->epagespa = va_to_pa(seg->epages);
seg->kpm_pbase = kpmptop(ptokpmp(base));
seg->kpm_nkpmpgs = kpnum;
if (kpm_smallpages) {
kpm_spage_t *kpm_pp = kpm_ppnext;
kpm_ppnext = kpm_pp + kpnum;
seg->kpm_spages = kpm_pp;
seg->kpm_pagespa = va_to_pa(seg->kpm_spages);
} else {
kpm_page_t *kpm_pp = kpm_ppnext;
kpm_ppnext = kpm_pp + kpnum;
seg->kpm_pages = kpm_pp;
seg->kpm_pagespa = va_to_pa(seg->kpm_pages);
ASSERT(
memseg_find(base - pmodkpmp(base), NULL) == NULL);
ASSERT(memseg_find(
roundup(base + num, kpmpnpgs) - 1, NULL) == NULL);
}
kpm_ppleft -= kpnum;
}
memseg_list_add(seg);
}
void
kphysm_add(uint64_t addr, uint64_t len, int reclaim)
{
struct page *pp;
struct memseg *seg;
pfn_t base = btop(addr);
pgcnt_t num = btop(len);
seg = memseg_find(base, NULL);
ASSERT(seg != NULL);
pp = seg->pages + (base - seg->pages_base);
if (reclaim) {
struct page *rpp = pp;
struct page *lpp = pp + num;
while (rpp < lpp) {
ASSERT(PAGE_EXCL(rpp) && rpp->p_vnode == &promvp);
ASSERT(PP_ISNORELOC(rpp));
PP_CLRNORELOC(rpp);
page_pp_unlock(rpp, 0, 1);
page_hashout(rpp, NULL);
page_unlock(rpp);
rpp++;
}
}
add_physmem(pp, num, base);
}
static void
kphysm_init(void)
{
struct memlist *pmem;
ASSERT(page_hash != NULL && page_hashsz != 0);
ppnext = pp_base;
ppleft = npages;
kpm_ppnext = kpm_pp_base;
kpm_ppleft = kpm_npages;
diff_memlists(phys_install, nopp_list, kphysm_memseg);
for (pmem = phys_avail; pmem != NULL; pmem = pmem->ml_next)
kphysm_add(pmem->ml_address, pmem->ml_size, 0);
diff_memlists(phys_install, phys_avail, kphysm_erase);
build_pfn_hash();
}
static void
kvm_init(void)
{
rw_enter(&kas.a_lock, RW_WRITER);
as_avlinit(&kas);
(void) seg_attach(&kas, (caddr_t)KERNELBASE,
(size_t)(e_moddata - KERNELBASE), &ktextseg);
(void) segkmem_create(&ktextseg);
(void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M),
(size_t)(MMU_PAGESIZE4M), &ktexthole);
(void) segkmem_create(&ktexthole);
(void) seg_attach(&kas, (caddr_t)valloc_base,
(size_t)(econtig32 - valloc_base), &kvalloc);
(void) segkmem_create(&kvalloc);
if (kmem64_base) {
(void) seg_attach(&kas, (caddr_t)kmem64_base,
(size_t)(kmem64_end - kmem64_base), &kmem64);
(void) segkmem_create(&kmem64);
}
(void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg);
(void) segkmem_create(&kvseg);
hblk_alloc_dynamic = 1;
memseg_remap_init();
segkmem_heap_lp_init();
(void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32,
&kvseg32);
(void) segkmem_create(&kvseg32);
(void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg);
(void) segkmem_create(&kdebugseg);
rw_exit(&kas.a_lock);
}
char obp_tte_str[] =
"h# %x constant MMU_PAGESHIFT "
"h# %x constant TTE8K "
"h# %x constant SFHME_SIZE "
"h# %x constant SFHME_TTE "
"h# %x constant HMEBLK_TAG "
"h# %x constant HMEBLK_NEXT "
"h# %x constant HMEBLK_MISC "
"h# %x constant HMEBLK_HME1 "
"h# %x constant NHMENTS "
"h# %x constant HBLK_SZMASK "
"h# %x constant HBLK_RANGE_SHIFT "
"h# %x constant HMEBP_HBLK "
"h# %x constant HMEBLK_ENDPA "
"h# %x constant HMEBUCKET_SIZE "
"h# %x constant HTAG_SFMMUPSZ "
"h# %x constant HTAG_BSPAGE_SHIFT "
"h# %x constant HTAG_REHASH_SHIFT "
"h# %x constant SFMMU_INVALID_SHMERID "
"h# %x constant mmu_hashcnt "
"h# %p constant uhme_hash "
"h# %p constant khme_hash "
"h# %x constant UHMEHASH_SZ "
"h# %x constant KHMEHASH_SZ "
"h# %p constant KCONTEXT "
"h# %p constant KHATID "
"h# %x constant ASI_MEM "
": PHYS-X@ ( phys -- data ) "
" ASI_MEM spacex@ "
"; "
": PHYS-W@ ( phys -- data ) "
" ASI_MEM spacew@ "
"; "
": PHYS-L@ ( phys -- data ) "
" ASI_MEM spaceL@ "
"; "
": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) "
" 3 * MMU_PAGESHIFT + "
"; "
": TTE_IS_VALID ( ttep -- flag ) "
" PHYS-X@ 0< "
"; "
": HME_HASH_SHIFT ( ttesz -- hmeshift ) "
" dup TTE8K = if "
" drop HBLK_RANGE_SHIFT "
" else "
" TTE_PAGE_SHIFT "
" then "
"; "
": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) "
" tuck >> swap MMU_PAGESHIFT - << "
"; "
": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) "
" >> over xor swap ( hash sfmmup ) "
" KHATID <> if ( hash ) "
" UHMEHASH_SZ and ( bucket ) "
" HMEBUCKET_SIZE * uhme_hash + ( hmebp ) "
" else ( hash ) "
" KHMEHASH_SZ and ( bucket ) "
" HMEBUCKET_SIZE * khme_hash + ( hmebp ) "
" then ( hmebp ) "
"; "
": HME_HASH_TABLE_SEARCH "
" ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) "
" >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) "
" dup HMEBLK_ENDPA <> if ( sfmmup hmeblkp ) ( r: hblktag ) "
" dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) "
" dup hmeblk_tag + 8 + phys-x@ 2 pick = if "
" true ( sfmmup hmeblkp true ) ( r: hblktag ) "
" else "
" hmeblk_next + phys-x@ false "
" ( sfmmup hmeblkp false ) ( r: hblktag ) "
" then "
" else "
" hmeblk_next + phys-x@ false "
" ( sfmmup hmeblkp false ) ( r: hblktag ) "
" then "
" else "
" drop 0 true "
" then "
" until r> drop "
"; "
": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) "
" over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) "
" HTAG_BSPAGE_SHIFT << ( sfmmup rehash htag-bspage )"
" swap HTAG_REHASH_SHIFT << or ( sfmmup htag-bspage-rehash )"
" SFMMU_INVALID_SHMERID or nip ( hblktag ) "
"; "
": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) "
" over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) "
" TTE8K = if ( hmeblkp addr ) "
" MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) "
" else ( hmeblkp addr ) "
" drop 0 ( hmeblkp 0 ) "
" then ( hmeblkp hme-index ) "
" SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) "
" SFHME_TTE + ( ttep ) "
"; "
": unix-tte ( addr cnum -- false | tte-data true ) "
" KCONTEXT = if ( addr ) "
" KHATID ( addr khatid ) "
" else ( addr ) "
" drop false exit ( false ) "
" then "
" ( addr khatid ) "
" mmu_hashcnt 1+ 1 do ( addr sfmmup ) "
" 2dup swap i HME_HASH_SHIFT "
"( addr sfmmup sfmmup addr hmeshift ) "
" HME_HASH_FUNCTION ( addr sfmmup hmebp ) "
" over i 4 pick "
"( addr sfmmup hmebp sfmmup rehash addr ) "
" HME_HASH_TAG ( addr sfmmup hmebp hblktag ) "
" HME_HASH_TABLE_SEARCH "
"( addr sfmmup { null | hmeblkp } ) "
" ?dup if ( addr sfmmup hmeblkp ) "
" nip swap HBLK_TO_TTEP ( ttep ) "
" dup TTE_IS_VALID if ( valid-ttep ) "
" PHYS-X@ true ( tte-data true ) "
" else ( invalid-tte ) "
" drop false ( false ) "
" then ( false | tte-data true ) "
" unloop exit ( false | tte-data true ) "
" then ( addr sfmmup ) "
" loop ( addr sfmmup ) "
" 2drop false ( false ) "
"; "
;
void
create_va_to_tte(void)
{
char *bp;
extern int khmehash_num, uhmehash_num;
extern struct hmehash_bucket *khme_hash, *uhme_hash;
#define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field))
bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP);
(void) sprintf(bp, obp_tte_str,
MMU_PAGESHIFT,
TTE8K,
sizeof (struct sf_hment),
OFFSET(struct sf_hment, hme_tte),
OFFSET(struct hme_blk, hblk_tag),
OFFSET(struct hme_blk, hblk_nextpa),
OFFSET(struct hme_blk, hblk_misc),
OFFSET(struct hme_blk, hblk_hme),
NHMENTS,
HBLK_SZMASK,
HBLK_RANGE_SHIFT,
OFFSET(struct hmehash_bucket, hmeh_nextpa),
HMEBLK_ENDPA,
sizeof (struct hmehash_bucket),
HTAG_SFMMUPSZ,
HTAG_BSPAGE_SHIFT,
HTAG_REHASH_SHIFT,
SFMMU_INVALID_SHMERID,
mmu_hashcnt,
(caddr_t)va_to_pa((caddr_t)uhme_hash),
(caddr_t)va_to_pa((caddr_t)khme_hash),
UHMEHASH_SZ,
KHMEHASH_SZ,
KCONTEXT,
KHATID,
ASI_MEM);
prom_interpret(bp, 0, 0, 0, 0, 0);
kobj_free(bp, MMU_PAGESIZE);
}
void
install_va_to_tte(void)
{
prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0);
}
static char *create_node =
"\" /\" find-device "
"new-device "
"\" os-io\" device-name "
"\" "OBP_DISPLAY_CONSOLE"\" device-type "
": cb-r/w ( adr,len method$ -- #read/#written ) "
" 2>r swap 2 2r> ['] $callback catch if "
" 2drop 3drop 0 "
" then "
"; "
": read ( adr,len -- #read ) "
" \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then "
" ( retN ... ret1 N ) "
" ?dup if "
" swap >r 1- 0 ?do drop loop r> "
" else "
" -2 "
" then "
"; "
": write ( adr,len -- #written ) "
" \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then "
" ( retN ... ret1 N ) "
" ?dup if "
" swap >r 1- 0 ?do drop loop r> "
" else "
" 0 "
" then "
"; "
": poll-tty ( -- ) ; "
": install-abort ( -- ) ['] poll-tty d# 10 alarm ; "
": remove-abort ( -- ) ['] poll-tty 0 alarm ; "
": cb-give/take ( $method -- ) "
" 0 -rot ['] $callback catch ?dup if "
" >r 2drop 2drop r> throw "
" else "
" 0 ?do drop loop "
" then "
"; "
": give ( -- ) \" exit-input\" cb-give/take ; "
": take ( -- ) \" enter-input\" cb-give/take ; "
": open ( -- ok? ) true ; "
": close ( -- ) ; "
"finish-device "
"device-end ";
static void
startup_create_io_node(void)
{
prom_interpret(create_node, 0, 0, 0, 0, 0);
}
extern caddr_t modtext;
extern size_t modtext_sz;
extern caddr_t moddata;
#define HEAPTEXT_ARENA(addr) \
((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \
(((uintptr_t)(addr) - HEAPTEXT_BASE) / \
(HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1))
#define HEAPTEXT_OVERSIZED(addr) \
((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE)
#define HEAPTEXT_IN_NUCLEUSDATA(addr) \
(((uintptr_t)(addr) >= KERNELBASE + 2 * MMU_PAGESIZE4M) && \
((uintptr_t)(addr) < KERNELBASE + 3 * MMU_PAGESIZE4M))
vmem_t *texthole_source[HEAPTEXT_NARENAS];
vmem_t *texthole_arena[HEAPTEXT_NARENAS];
kmutex_t texthole_lock;
char kern_bootargs[OBP_MAXPATHLEN];
char kern_bootfile[OBP_MAXPATHLEN];
void
kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena)
{
uintptr_t addr, limit;
addr = HEAPTEXT_BASE;
limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE;
for (; addr + HEAPTEXT_UNMAPPED <= limit;
addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) {
(void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE,
0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED),
VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
}
(void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0,
(void *)limit, (void *)(limit + PAGESIZE),
VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
*text_arena = vmem_create("module_text", modtext_sz ? modtext : NULL,
modtext_sz, sizeof (uintptr_t), segkmem_alloc, segkmem_free,
heaptext_arena, 0, VM_SLEEP);
*data_arena = vmem_create("module_data", moddata, MODDATA, 1,
segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP);
}
caddr_t
kobj_text_alloc(vmem_t *arena, size_t size)
{
caddr_t rval, better;
rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT);
if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval))
return (rval);
better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL,
(void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE),
VM_NOSLEEP | VM_BESTFIT);
if (better != NULL) {
vmem_free(arena, rval, size);
return (better);
}
return (rval);
}
caddr_t
kobj_texthole_alloc(caddr_t addr, size_t size)
{
int arena = HEAPTEXT_ARENA(addr);
char c[30];
uintptr_t base;
if (HEAPTEXT_OVERSIZED(addr) || HEAPTEXT_IN_NUCLEUSDATA(addr)) {
return (NULL);
}
mutex_enter(&texthole_lock);
if (texthole_arena[arena] == NULL) {
ASSERT(texthole_source[arena] == NULL);
if (arena == 0) {
texthole_source[0] = vmem_create("module_text_holesrc",
(void *)(KERNELBASE + MMU_PAGESIZE4M),
MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL,
0, VM_SLEEP);
} else {
base = HEAPTEXT_BASE +
(arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED);
(void) snprintf(c, sizeof (c),
"heaptext_holesrc_%d", arena);
texthole_source[arena] = vmem_create(c, (void *)base,
HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL,
0, VM_SLEEP);
}
(void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena);
texthole_arena[arena] = vmem_create(c, NULL, 0,
sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free,
texthole_source[arena], 0, VM_SLEEP);
}
mutex_exit(&texthole_lock);
ASSERT(texthole_arena[arena] != NULL);
ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS);
return (vmem_alloc(texthole_arena[arena], size,
VM_BESTFIT | VM_NOSLEEP));
}
void
kobj_texthole_free(caddr_t addr, size_t size)
{
int arena = HEAPTEXT_ARENA(addr);
ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS);
ASSERT(texthole_arena[arena] != NULL);
vmem_free(texthole_arena[arena], addr, size);
}
void
release_bootstrap(void)
{
if (&cif_init)
cif_init();
}
