/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2010 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Copyright 2019 Joyent, Inc. * Copyright 2019 Peter Tribble. */ #include <sys/sysmacros.h> #include <sys/prom_plat.h> #include <sys/prom_debug.h> #include <vm/hat_sfmmu.h> #include <vm/seg_kp.h> #include <vm/seg_kmem.h> #include <sys/machsystm.h> #include <sys/callb.h> #include <sys/cpu_module.h> #include <sys/pg.h> #include <sys/cmt.h> #include <sys/dtrace.h> #include <sys/reboot.h> #include <sys/kdi.h> #include <sys/traptrace.h> #ifdef TRAPTRACE #include <sys/bootconf.h> #endif /* TRAPTRACE */ #include <sys/cpu_sgnblk_defs.h> extern int cpu_intrq_setup(struct cpu *); extern void cpu_intrq_cleanup(struct cpu *); extern void cpu_intrq_register(struct cpu *); struct cpu *cpus; /* pointer to other cpus; dynamically allocate */ struct cpu *cpu[NCPU]; /* pointers to all CPUs */ uint64_t cpu_pa[NCPU]; /* pointers to all CPUs in PA */ cpu_core_t cpu_core[NCPU]; /* cpu_core structures */ #ifdef TRAPTRACE caddr_t ttrace_buf; /* kmem64 traptrace for all cpus except 0 */ #endif /* TRAPTRACE */ /* bit mask of cpus ready for x-calls, protected by cpu_lock */ cpuset_t cpu_ready_set; /* bit mask used to communicate with cpus during bringup */ static cpuset_t proxy_ready_set; static void slave_startup(void); /* * Defined in $KARCH/os/mach_mp_startup.c */ #pragma weak init_cpu_info /* * Amount of time (in milliseconds) we should wait before giving up on CPU * initialization and assuming that the CPU we're trying to wake up is dead * or out of control. */ #define CPU_WAKEUP_GRACE_MSEC 1000 #ifdef TRAPTRACE /* * This function sets traptrace buffers for all cpus * other than boot cpu. */ size_t calc_traptrace_sz(void) { return (TRAP_TSIZE * (max_ncpus - 1)); } #endif /* TRAPTRACE */ /* * common slave cpu initialization code */ void common_startup_init(cpu_t *cp, int cpuid) { kthread_id_t tp; sfmmu_t *sfmmup; caddr_t sp; /* * Allocate and initialize the startup thread for this CPU. */ tp = thread_create(NULL, 0, slave_startup, NULL, 0, &p0, TS_STOPPED, maxclsyspri); /* * Set state to TS_ONPROC since this thread will start running * as soon as the CPU comes online. * * All the other fields of the thread structure are setup by * thread_create(). */ THREAD_ONPROC(tp, cp); tp->t_preempt = 1; tp->t_bound_cpu = cp; tp->t_affinitycnt = 1; tp->t_cpu = cp; tp->t_disp_queue = cp->cpu_disp; sfmmup = astosfmmu(&kas); CPUSET_ADD(sfmmup->sfmmu_cpusran, cpuid); /* * Setup thread to start in slave_startup. */ sp = tp->t_stk; tp->t_pc = (uintptr_t)slave_startup - 8; tp->t_sp = (uintptr_t)((struct rwindow *)sp - 1) - STACK_BIAS; cp->cpu_id = cpuid; cp->cpu_self = cp; cp->cpu_thread = tp; cp->cpu_lwp = NULL; cp->cpu_dispthread = tp; cp->cpu_dispatch_pri = DISP_PRIO(tp); cp->cpu_startup_thread = tp; /* * The dispatcher may discover the CPU before it is in cpu_ready_set * and attempt to poke it. Before the CPU is in cpu_ready_set, any * cross calls to it will be dropped. We initialize * poke_cpu_outstanding to true so that poke_cpu will ignore any poke * requests for this CPU. Pokes that come in before the CPU is in * cpu_ready_set can be ignored because the CPU is about to come * online. */ cp->cpu_m.poke_cpu_outstanding = B_TRUE; } /* * parametric flag setting functions. these routines set the cpu * state just prior to releasing the slave cpu. */ void cold_flag_set(int cpuid) { cpu_t *cp; ASSERT(MUTEX_HELD(&cpu_lock)); cp = cpu[cpuid]; if (!(cp->cpu_flags & CPU_ENABLE)) ncpus_intr_enabled++; cp->cpu_flags |= CPU_RUNNING | CPU_ENABLE | CPU_EXISTS; cpu_add_active(cp); /* * Add CPU_READY after the cpu_add_active() call * to avoid pausing cp. */ cp->cpu_flags |= CPU_READY; /* ready */ cpu_set_state(cp); } static void warm_flag_set(int cpuid) { cpu_t *cp; ASSERT(MUTEX_HELD(&cpu_lock)); /* * warm start activates cpus into the OFFLINE state */ cp = cpu[cpuid]; cp->cpu_flags |= CPU_RUNNING | CPU_READY | CPU_EXISTS | CPU_OFFLINE | CPU_QUIESCED; cpu_set_state(cp); } /* * Internal cpu startup sequencer * The sequence is as follows: * * MASTER SLAVE * ------- ---------- * assume the kernel data is initialized * clear the proxy bit * start the slave cpu * wait for the slave cpu to set the proxy * * the slave runs slave_startup and then sets the proxy * the slave waits for the master to add slave to the ready set * * the master finishes the initialization and * adds the slave to the ready set * * the slave exits the startup thread and is running */ void start_cpu(int cpuid, void(*flag_func)(int)) { extern void cpu_startup(int); int timout; ASSERT(MUTEX_HELD(&cpu_lock)); /* * Before we begin the dance, tell DTrace that we're about to start * a CPU. */ if (dtrace_cpustart_init != NULL) (*dtrace_cpustart_init)(); /* start the slave cpu */ CPUSET_DEL(proxy_ready_set, cpuid); if (prom_test("SUNW,start-cpu-by-cpuid") == 0) { (void) prom_startcpu_bycpuid(cpuid, (caddr_t)&cpu_startup, cpuid); } else { /* "by-cpuid" interface didn't exist. Do it the old way */ pnode_t nodeid = cpunodes[cpuid].nodeid; ASSERT(nodeid != (pnode_t)0); (void) prom_startcpu(nodeid, (caddr_t)&cpu_startup, cpuid); } /* wait for the slave cpu to check in. */ for (timout = CPU_WAKEUP_GRACE_MSEC; timout; timout--) { if (CPU_IN_SET(proxy_ready_set, cpuid)) break; DELAY(1000); } if (timout == 0) { panic("cpu%d failed to start (2)", cpuid); } /* * The slave has started; we can tell DTrace that it's safe again. */ if (dtrace_cpustart_fini != NULL) (*dtrace_cpustart_fini)(); /* run the master side of stick synchronization for the slave cpu */ sticksync_master(); /* * deal with the cpu flags in a phase-specific manner * for various reasons, this needs to run after the slave * is checked in but before the slave is released. */ (*flag_func)(cpuid); /* release the slave */ CPUSET_ADD(cpu_ready_set, cpuid); } #ifdef TRAPTRACE int trap_tr0_inuse = 1; /* it is always used on the boot cpu */ int trap_trace_inuse[NCPU]; #endif /* TRAPTRACE */ #define cpu_next_free cpu_prev /* * Routine to set up a CPU to prepare for starting it up. */ int setup_cpu_common(int cpuid) { struct cpu *cp = NULL; kthread_id_t tp; #ifdef TRAPTRACE int tt_index; TRAP_TRACE_CTL *ctlp; caddr_t newbuf; #endif /* TRAPTRACE */ extern void idle(); int rval; ASSERT(MUTEX_HELD(&cpu_lock)); ASSERT(cpu[cpuid] == NULL); ASSERT(ncpus <= max_ncpus); #ifdef TRAPTRACE /* * allocate a traptrace buffer for this CPU. */ ctlp = &trap_trace_ctl[cpuid]; if (!trap_tr0_inuse) { trap_tr0_inuse = 1; newbuf = trap_tr0; tt_index = -1; } else { for (tt_index = 0; tt_index < (max_ncpus-1); tt_index++) if (!trap_trace_inuse[tt_index]) break; ASSERT(tt_index < max_ncpus - 1); trap_trace_inuse[tt_index] = 1; newbuf = (caddr_t)(ttrace_buf + (tt_index * TRAP_TSIZE)); } ctlp->d.vaddr_base = newbuf; ctlp->d.offset = ctlp->d.last_offset = 0; ctlp->d.limit = trap_trace_bufsize; ctlp->d.paddr_base = va_to_pa(newbuf); ASSERT(ctlp->d.paddr_base != (uint64_t)-1); #endif /* TRAPTRACE */ /* * initialize hv traptrace buffer for this CPU */ mach_htraptrace_setup(cpuid); /* * Obtain pointer to the appropriate cpu structure. */ if (cpu0.cpu_flags == 0) { cp = &cpu0; } else { /* * When dynamically allocating cpu structs, * cpus is used as a pointer to a list of freed * cpu structs. */ if (cpus) { /* grab the first cpu struct on the free list */ cp = cpus; if (cp->cpu_next_free) cpus = cp->cpu_next_free; else cpus = NULL; } } if (cp == NULL) cp = vmem_xalloc(static_alloc_arena, CPU_ALLOC_SIZE, CPU_ALLOC_SIZE, 0, 0, NULL, NULL, VM_SLEEP); bzero(cp, sizeof (*cp)); cp->cpu_id = cpuid; cp->cpu_self = cp; /* * Initialize ptl1_panic stack */ ptl1_init_cpu(cp); /* * Initialize the dispatcher for this CPU. */ disp_cpu_init(cp); /* * Bootstrap the CPU's PG data */ pg_cpu_bootstrap(cp); cpu_vm_data_init(cp); /* * Now, initialize per-CPU idle thread for this CPU. */ tp = thread_create(NULL, 0, idle, NULL, 0, &p0, TS_ONPROC, -1); cp->cpu_idle_thread = tp; tp->t_preempt = 1; tp->t_bound_cpu = cp; tp->t_affinitycnt = 1; tp->t_cpu = cp; tp->t_disp_queue = cp->cpu_disp; /* * Registering a thread in the callback table is usually * done in the initialization code of the thread. In this * case, we do it right after thread creation to avoid * blocking idle thread while registering itself. It also * avoids the possibility of reregistration in case a CPU * restarts its idle thread. */ CALLB_CPR_INIT_SAFE(tp, "idle"); init_cpu_info(cp); /* * Initialize the interrupt threads for this CPU */ cpu_intr_alloc(cp, NINTR_THREADS); /* * Add CPU to list of available CPUs. * It'll be on the active list after it is started. */ cpu_add_unit(cp); /* * Allocate and init cpu module private data structures, * including scrubber. */ cpu_init_private(cp); populate_idstr(cp); /* * Initialize the CPUs physical ID cache, and processor groups */ pghw_physid_create(cp); (void) pg_cpu_init(cp, B_FALSE); if ((rval = cpu_intrq_setup(cp)) != 0) { return (rval); } /* * Initialize MMU context domain information. */ sfmmu_cpu_init(cp); return (0); } /* * Routine to clean up a CPU after shutting it down. */ int cleanup_cpu_common(int cpuid) { struct cpu *cp; #ifdef TRAPTRACE int i; TRAP_TRACE_CTL *ctlp; caddr_t newbuf; #endif /* TRAPTRACE */ ASSERT(MUTEX_HELD(&cpu_lock)); ASSERT(cpu[cpuid] != NULL); cp = cpu[cpuid]; /* Free cpu module private data structures, including scrubber. */ cpu_uninit_private(cp); /* Free cpu ID string and brand string. */ if (cp->cpu_idstr) kmem_free(cp->cpu_idstr, strlen(cp->cpu_idstr) + 1); if (cp->cpu_brandstr) kmem_free(cp->cpu_brandstr, strlen(cp->cpu_brandstr) + 1); cpu_vm_data_destroy(cp); /* * Remove CPU from list of available CPUs. */ cpu_del_unit(cpuid); /* * Clean any machine specific interrupt states. */ cpu_intrq_cleanup(cp); /* * At this point, the only threads bound to this CPU should be * special per-cpu threads: it's idle thread, it's pause thread, * and it's interrupt threads. Clean these up. */ cpu_destroy_bound_threads(cp); /* * Free the interrupt stack. */ segkp_release(segkp, cp->cpu_intr_stack); /* * Free hv traptrace buffer for this CPU. */ mach_htraptrace_cleanup(cpuid); #ifdef TRAPTRACE /* * Free the traptrace buffer for this CPU. */ ctlp = &trap_trace_ctl[cpuid]; newbuf = ctlp->d.vaddr_base; i = (newbuf - ttrace_buf) / (TRAP_TSIZE); if (((newbuf - ttrace_buf) % (TRAP_TSIZE) == 0) && ((i >= 0) && (i < (max_ncpus-1)))) { /* * This CPU got it's trap trace buffer from the * boot-alloc'd bunch of them. */ trap_trace_inuse[i] = 0; bzero(newbuf, (TRAP_TSIZE)); } else if (newbuf == trap_tr0) { trap_tr0_inuse = 0; bzero(trap_tr0, (TRAP_TSIZE)); } else { cmn_err(CE_WARN, "failed to free trap trace buffer from cpu%d", cpuid); } bzero(ctlp, sizeof (*ctlp)); #endif /* TRAPTRACE */ /* * There is a race condition with mutex_vector_enter() which * caches a cpu pointer. The race is detected by checking cpu_next. */ disp_cpu_fini(cp); cpu_pa[cpuid] = 0; if (CPU_MMU_CTXP(cp)) sfmmu_cpu_cleanup(cp); bzero(cp, sizeof (*cp)); /* * Place the freed cpu structure on the list of freed cpus. */ if (cp != &cpu0) { if (cpus) { cp->cpu_next_free = cpus; cpus = cp; } else cpus = cp; } return (0); } /* * This routine is used to start a previously powered off processor. * Note that restarted cpus are initialized into the offline state. */ void restart_other_cpu(int cpuid) { struct cpu *cp; kthread_id_t tp; caddr_t sp; extern void idle(); ASSERT(MUTEX_HELD(&cpu_lock)); ASSERT(cpuid < NCPU && cpu[cpuid] != NULL); /* * Obtain pointer to the appropriate cpu structure. */ cp = cpu[cpuid]; common_startup_init(cp, cpuid); /* * idle thread t_lock is held when the idle thread is suspended. * Manually unlock the t_lock of idle loop so that we can resume * the suspended idle thread. * Also adjust the PC of idle thread for re-retry. */ cp->cpu_intr_actv = 0; /* clear the value from previous life */ cp->cpu_m.mutex_ready = 0; /* we are not ready yet */ lock_clear(&cp->cpu_idle_thread->t_lock); tp = cp->cpu_idle_thread; sp = tp->t_stk; tp->t_sp = (uintptr_t)((struct rwindow *)sp - 1) - STACK_BIAS; tp->t_pc = (uintptr_t)idle - 8; /* * restart the cpu now */ promsafe_pause_cpus(); start_cpu(cpuid, warm_flag_set); start_cpus(); /* call cmn_err outside pause_cpus/start_cpus to avoid deadlock */ cmn_err(CE_CONT, "!cpu%d initialization complete - restarted\n", cpuid); } /* * Startup function executed on 'other' CPUs. This is the first * C function after cpu_start sets up the cpu registers. */ static void slave_startup(void) { struct cpu *cp = CPU; ushort_t original_flags = cp->cpu_flags; mach_htraptrace_configure(cp->cpu_id); cpu_intrq_register(CPU); cp->cpu_m.mutex_ready = 1; /* acknowledge that we are done with initialization */ CPUSET_ADD(proxy_ready_set, cp->cpu_id); /* synchronize STICK */ sticksync_slave(); if (boothowto & RB_DEBUG) kdi_dvec_cpu_init(cp); /* * the slave will wait here forever -- assuming that the master * will get back to us. if it doesn't we've got bigger problems * than a master not replying to this slave. * the small delay improves the slave's responsiveness to the * master's ack and decreases the time window between master and * slave operations. */ while (!CPU_IN_SET(cpu_ready_set, cp->cpu_id)) DELAY(1); /* * The CPU is now in cpu_ready_set, safely able to take pokes. */ cp->cpu_m.poke_cpu_outstanding = B_FALSE; /* enable interrupts */ (void) spl0(); /* * Signature block update to indicate that this CPU is in OS now. * This needs to be done after the PIL is lowered since on * some platforms the update code may block. */ CPU_SIGNATURE(OS_SIG, SIGST_RUN, SIGSUBST_NULL, cp->cpu_id); /* * park the slave thread in a safe/quiet state and wait for the master * to finish configuring this CPU before proceeding to thread_exit(). */ while (((volatile ushort_t)cp->cpu_flags) & CPU_QUIESCED) DELAY(1); /* * Initialize CPC CPU state. */ kcpc_hw_startup_cpu(original_flags); /* * Notify the PG subsystem that the CPU has started */ pg_cmt_cpu_startup(CPU); /* * Now we are done with the startup thread, so free it up. */ thread_exit(); cmn_err(CE_PANIC, "slave_startup: cannot return"); /*NOTREACHED*/ } extern struct cpu *cpu[NCPU]; /* pointers to all CPUs */ /* * cpu_bringup_set is a tunable (via /etc/system, debugger, etc.) that * can be used during debugging to control which processors are brought * online at boot time. The variable represents a bitmap of the id's * of the processors that will be brought online. The initialization * of this variable depends on the type of cpuset_t, which varies * depending on the number of processors supported (see cpuvar.h). */ cpuset_t cpu_bringup_set; /* * Generic start-all cpus entry. Typically used during cold initialization. * Note that cold start cpus are initialized into the online state. */ /*ARGSUSED*/ void start_other_cpus(int flag) { int cpuid; extern void idlestop_init(void); int bootcpu; /* * Check if cpu_bringup_set has been explicitly set before * initializing it. */ if (CPUSET_ISNULL(cpu_bringup_set)) { CPUSET_ALL(cpu_bringup_set); } if (&cpu_feature_init) cpu_feature_init(); /* * Initialize CPC. */ kcpc_hw_init(); mutex_enter(&cpu_lock); /* * Initialize our own cpu_info. */ init_cpu_info(CPU); /* * Initialize CPU 0 cpu module private data area, including scrubber. */ cpu_init_private(CPU); populate_idstr(CPU); /* * perform such initialization as is needed * to be able to take CPUs on- and off-line. */ cpu_pause_init(); xc_init(); /* initialize processor crosscalls */ idlestop_init(); if (!use_mp) { mutex_exit(&cpu_lock); cmn_err(CE_CONT, "?***** Not in MP mode\n"); return; } /* * should we be initializing this cpu? */ bootcpu = getprocessorid(); /* * launch all the slave cpus now */ for (cpuid = 0; cpuid < NCPU; cpuid++) { pnode_t nodeid = cpunodes[cpuid].nodeid; if (nodeid == (pnode_t)0) continue; if (cpuid == bootcpu) { if (!CPU_IN_SET(cpu_bringup_set, cpuid)) { cmn_err(CE_WARN, "boot cpu not a member " "of cpu_bringup_set, adding it"); CPUSET_ADD(cpu_bringup_set, cpuid); } continue; } if (!CPU_IN_SET(cpu_bringup_set, cpuid)) continue; ASSERT(cpu[cpuid] == NULL); if (setup_cpu_common(cpuid)) { cmn_err(CE_PANIC, "cpu%d: setup failed", cpuid); } common_startup_init(cpu[cpuid], cpuid); start_cpu(cpuid, cold_flag_set); /* * Because slave_startup() gets fired off after init() * starts, we can't use the '?' trick to do 'boot -v' * printing - so we always direct the 'cpu .. online' * messages to the log. */ cmn_err(CE_CONT, "!cpu%d initialization complete - online\n", cpuid); cpu_state_change_notify(cpuid, CPU_SETUP); if (dtrace_cpu_init != NULL) (*dtrace_cpu_init)(cpuid); } /* * since all the cpus are online now, redistribute interrupts to them. */ intr_redist_all_cpus(); mutex_exit(&cpu_lock); /* * Start the Ecache scrubber. Must be done after all calls to * cpu_init_private for every cpu (including CPU 0). */ cpu_init_cache_scrub(); if (&cpu_mp_init) cpu_mp_init(); }
