#ifndef _X86_CPU_H_
#define _X86_CPU_H_
#if defined(_KERNEL) || defined(_STANDALONE)
#include <sys/types.h>
#else
#include <stdint.h>
#include <stdbool.h>
#endif
#if defined(_KERNEL) || defined(_KMEMUSER)
#if defined(_KERNEL_OPT)
#include "opt_xen.h"
#include "opt_svs.h"
#endif
#include <machine/frame.h>
#include <machine/pte.h>
#include <machine/segments.h>
#include <machine/tss.h>
#include <machine/intrdefs.h>
#include <x86/cacheinfo.h>
#include <sys/cpu_data.h>
#include <sys/evcnt.h>
#include <sys/device_if.h>
#ifdef SVS
#include <sys/mutex.h>
#endif
#ifdef XEN
#include <xen/include/public/xen.h>
#include <xen/include/public/event_channel.h>
#include <sys/mutex.h>
#endif
struct intrsource;
struct pmap;
struct kcpuset;
#ifdef __x86_64__
#define i386tss x86_64_tss
#endif
#define NIOPORTS 1024
#define IOMAPSIZE (NIOPORTS / 8)
struct cpu_tss {
#ifdef i386
struct i386tss dblflt_tss;
struct i386tss ddbipi_tss;
#endif
struct i386tss tss;
uint8_t iomap[IOMAPSIZE];
} __packed;
struct clockframe {
struct intrframe cf_if;
};
struct idt_vec {
void *iv_idt;
void *iv_idt_pentium;
char iv_allocmap[NIDT];
};
struct cpu_info {
struct cpu_data ci_data;
device_t ci_dev;
struct cpu_info *ci_self;
struct pmap *ci_pmap;
int ci_want_pmapload;
volatile int ci_tlbstate;
#define TLBSTATE_VALID 0
#define TLBSTATE_LAZY 1
#define TLBSTATE_STALE 2
int ci_curldt;
int ci_nintrhand;
uint64_t ci_scratch;
uintptr_t ci_pmap_data[128 / sizeof(uintptr_t)];
struct kcpuset *ci_tlb_cpuset;
struct idt_vec ci_idtvec;
int ci_kfpu_spl;
struct intrsource *ci_isources[MAX_INTR_SOURCES];
volatile int ci_mtx_count;
volatile int ci_mtx_oldspl;
union {
uint64_t ci_istate;
struct {
uint64_t ci_ipending:56;
uint64_t ci_ilevel:8;
};
} __aligned(8);
uint64_t ci_imasked;
int ci_idepth;
void * ci_intrstack;
uint64_t ci_imask[NIPL];
uint64_t ci_iunmask[NIPL];
uint32_t ci_signature;
uint32_t ci_vendor[4];
uint32_t ci_max_cpuid;
uint32_t ci_max_ext_cpuid;
volatile uint32_t ci_lapic_counter;
uint32_t ci_feat_val[8];
const struct cpu_functions *ci_func;
struct trapframe *ci_ddb_regs;
u_int ci_cflush_lsize;
struct x86_cache_info ci_cinfo[CAI_COUNT];
device_t ci_frequency;
device_t ci_padlock;
device_t ci_temperature;
device_t ci_vm;
union descriptor *ci_gdt;
struct cpu_tss *ci_tss;
int ci_tss_sel;
uintptr_t ci_suspend_gdt;
uint16_t ci_suspend_gdt_padding;
uintptr_t ci_suspend_idt;
uint16_t ci_suspend_idt_padding;
uint16_t ci_suspend_tr;
uint16_t ci_suspend_ldt;
uintptr_t ci_suspend_fs;
uintptr_t ci_suspend_gs;
uintptr_t ci_suspend_kgs;
uintptr_t ci_suspend_efer;
uintptr_t ci_suspend_reg[12];
uintptr_t ci_suspend_cr0;
uintptr_t ci_suspend_cr2;
uintptr_t ci_suspend_cr3;
uintptr_t ci_suspend_cr4;
uintptr_t ci_suspend_cr8;
int ci_want_resched __aligned(64);
uint32_t ci_ipis;
uint32_t ci_flags __aligned(64);
uint32_t ci_acpiid;
uint32_t ci_initapicid;
uint32_t ci_vcpuid;
cpuid_t ci_cpuid;
struct cpu_info *ci_next;
struct lwp *ci_curlwp __aligned(64);
struct lwp *ci_onproc;
int ci_padout __aligned(64);
#ifndef __HAVE_DIRECT_MAP
#define VPAGE_SRC 0
#define VPAGE_DST 1
#define VPAGE_ZER 2
#define VPAGE_PTP 3
#define VPAGE_MAX 4
vaddr_t vpage[VPAGE_MAX];
pt_entry_t *vpage_pte[VPAGE_MAX];
#endif
#ifdef PAE
uint32_t ci_pae_l3_pdirpa;
pd_entry_t * ci_pae_l3_pdir;
#endif
#ifdef SVS
pd_entry_t * ci_svs_updir;
paddr_t ci_svs_updirpa;
int ci_svs_ldt_sel;
kmutex_t ci_svs_mtx;
pd_entry_t * ci_svs_rsp0_pte;
vaddr_t ci_svs_rsp0;
vaddr_t ci_svs_ursp0;
vaddr_t ci_svs_krsp0;
vaddr_t ci_svs_utls;
#endif
#ifndef XENPV
struct evcnt ci_ipi_events[X86_NIPI];
#else
struct evcnt ci_ipi_events[XEN_NIPIS];
#endif
#ifdef XEN
volatile struct vcpu_info *ci_vcpu;
u_long ci_evtmask[NR_EVENT_CHANNELS];
evtchn_port_t ci_ipi_evtchn;
#if defined(XENPV)
#if defined(PAE) || defined(__x86_64__)
pd_entry_t * ci_kpm_pdir;
paddr_t ci_kpm_pdirpa;
kmutex_t ci_kpm_mtx;
#endif
#if defined(__x86_64__)
pd_entry_t * ci_normal_pdes[3];
paddr_t ci_xen_current_user_pgd;
#endif
size_t ci_xpq_idx;
#endif
uint64_t ci_xen_hardclock_systime_ns;
uint64_t ci_xen_last_systime_ns;
uint64_t ci_xen_systime_ns_skew;
vaddr_t ci_xen_clockf_pc;
bool ci_xen_clockf_usermode;
struct evcnt ci_xen_cpu_tsc_backwards_evcnt;
struct evcnt ci_xen_tsc_delta_negative_evcnt;
struct evcnt ci_xen_raw_systime_wraparound_evcnt;
struct evcnt ci_xen_raw_systime_backwards_evcnt;
struct evcnt ci_xen_systime_backwards_hardclock_evcnt;
struct evcnt ci_xen_missed_hardclock_evcnt;
struct evcnt ci_xen_timecounter_backwards_evcnt;
struct evcnt ci_xen_timecounter_jump_evcnt;
#endif
#if defined(GPROF) && defined(MULTIPROCESSOR)
struct gmonparam *ci_gmon;
#endif
};
#if defined(XEN) && !defined(XENPV)
__CTASSERT(XEN_NIPIS <= X86_NIPI);
#endif
#ifdef __x86_64__
#define X86_TF_RAX(tf) tf->tf_rax
#define X86_TF_RDX(tf) tf->tf_rdx
#define X86_TF_RSP(tf) tf->tf_rsp
#define X86_TF_RIP(tf) tf->tf_rip
#define X86_TF_RFLAGS(tf) tf->tf_rflags
#else
#define X86_TF_RAX(tf) tf->tf_eax
#define X86_TF_RDX(tf) tf->tf_edx
#define X86_TF_RSP(tf) tf->tf_esp
#define X86_TF_RIP(tf) tf->tf_eip
#define X86_TF_RFLAGS(tf) tf->tf_eflags
#endif
#define CPUF_BSP 0x0001
#define CPUF_AP 0x0002
#define CPUF_SP 0x0004
#define CPUF_PRIMARY 0x0008
#define CPUF_SYNCTSC 0x0800
#define CPUF_PRESENT 0x1000
#define CPUF_RUNNING 0x2000
#define CPUF_PAUSE 0x4000
#define CPUF_GO 0x8000
#endif
#ifdef _KERNEL
extern struct cpu_info cpu_info_primary;
extern struct cpu_info *cpu_info_list;
#define CPU_INFO_ITERATOR int __unused
#define CPU_INFO_FOREACH(cii, ci) ci = cpu_info_list; \
ci != NULL; ci = ci->ci_next
#define CPU_STARTUP(_ci, _target) ((_ci)->ci_func->start(_ci, _target))
#define CPU_STOP(_ci) ((_ci)->ci_func->stop(_ci))
#define CPU_START_CLEANUP(_ci) ((_ci)->ci_func->cleanup(_ci))
#if !defined(__GNUC__) || defined(_MODULE)
struct cpu_info *x86_curcpu(void);
# ifdef __GNUC__
lwp_t *x86_curlwp(void) __attribute__ ((const));
# else
lwp_t *x86_curlwp(void);
# endif
#endif
#define cpu_number() (cpu_index(curcpu()))
#define CPU_IS_PRIMARY(ci) ((ci)->ci_flags & CPUF_PRIMARY)
#define aston(l) ((l)->l_md.md_astpending = 1)
void cpu_boot_secondary_processors(void);
void cpu_init_idle_lwps(void);
void cpu_init_msrs(struct cpu_info *, bool);
void cpu_load_pmap(struct pmap *, struct pmap *);
void cpu_broadcast_halt(void);
void cpu_kick(struct cpu_info *);
void cpu_pcpuarea_init(struct cpu_info *);
void cpu_svs_init(struct cpu_info *);
void cpu_speculation_init(struct cpu_info *);
#define curcpu() x86_curcpu()
#define curlwp x86_curlwp()
#define curpcb ((struct pcb *)lwp_getpcb(curlwp))
extern void cpu_need_proftick(struct lwp *l);
extern void cpu_signotify(struct lwp *);
extern void (*delay_func)(unsigned int);
struct timeval;
#ifndef __HIDE_DELAY
#define DELAY(x) (*delay_func)(x)
#define delay(x) (*delay_func)(x)
#endif
extern int biosbasemem;
extern int biosextmem;
extern int cputype;
extern int cpuid_level;
extern int cpu_class;
extern char cpu_brand_string[];
extern int use_pae;
#ifdef __i386__
#define i386_fpu_present 1
int npx586bug1(int, int);
extern int i386_fpu_fdivbug;
extern int i386_use_fxsave;
extern int i386_has_sse;
extern int i386_has_sse2;
#else
#define i386_fpu_present 1
#define i386_fpu_fdivbug 0
#define i386_use_fxsave 1
#define i386_has_sse 1
#define i386_has_sse2 1
#endif
extern int x86_fpu_save;
#define FPU_SAVE_FSAVE 0
#define FPU_SAVE_FXSAVE 1
#define FPU_SAVE_XSAVE 2
#define FPU_SAVE_XSAVEOPT 3
extern unsigned int x86_fpu_save_size;
extern uint64_t x86_xsave_features;
extern size_t x86_xsave_offsets[];
extern size_t x86_xsave_sizes[];
extern uint32_t x86_fpu_mxcsr_mask;
bool x86_fpu_save_separate_p(void);
extern void (*x86_cpu_idle)(void);
#define cpu_idle() (*x86_cpu_idle)()
#ifdef i386
void cpu_set_tss_gates(struct cpu_info *);
#endif
void cpu_reset(void);
u_int tmx86_get_longrun_mode(void);
void tmx86_get_longrun_status(u_int *, u_int *, u_int *);
void tmx86_init_longrun(void);
void cpu_probe(struct cpu_info *);
void cpu_identify(struct cpu_info *);
void identify_hypervisor(void);
uint64_t cpu_tsc_freq_cpuid(struct cpu_info *);
void cpu_dcp_cacheinfo(struct cpu_info *, uint32_t);
typedef enum vm_guest {
VM_GUEST_NO = 0,
VM_GUEST_VM,
VM_GUEST_XENPV,
VM_GUEST_XENPVH,
VM_GUEST_XENHVM,
VM_GUEST_XENPVHVM,
VM_GUEST_HV,
VM_GUEST_VMWARE,
VM_GUEST_KVM,
VM_GUEST_VIRTUALBOX,
VM_GUEST_GENPVH,
VM_GUEST_NVMM,
VM_LAST
} vm_guest_t;
extern vm_guest_t vm_guest;
static __inline bool __unused
vm_guest_is_xenpv(void)
{
switch(vm_guest) {
case VM_GUEST_XENPV:
case VM_GUEST_XENPVH:
case VM_GUEST_XENPVHVM:
return true;
default:
return false;
}
}
static __inline bool __unused
vm_guest_is_xenpvh_or_pvhvm(void)
{
switch(vm_guest) {
case VM_GUEST_XENPVH:
case VM_GUEST_XENPVHVM:
return true;
default:
return false;
}
}
static __inline bool __unused
vm_guest_is_pvh(void)
{
switch(vm_guest) {
case VM_GUEST_XENPVH:
case VM_GUEST_GENPVH:
return true;
default:
return false;
}
}
void x86_cpu_topology(struct cpu_info *);
struct region_descriptor;
void lgdt(struct region_descriptor *);
#ifdef XENPV
void lgdt_finish(void);
#endif
struct pcb;
void savectx(struct pcb *);
void lwp_trampoline(void);
#ifdef XEN
void xen_startrtclock(void);
void xen_delay(unsigned int);
void xen_initclocks(void);
void xen_cpu_initclocks(void);
void xen_suspendclocks(struct cpu_info *);
void xen_resumeclocks(struct cpu_info *);
#endif
void initrtclock(u_long);
void startrtclock(void);
void i8254_delay(unsigned int);
void i8254_microtime(struct timeval *);
void i8254_initclocks(void);
unsigned int gettick(void);
extern void (*x86_delay)(unsigned int);
void cpu_probe_features(struct cpu_info *);
int x86_cpu_is_lcall(const void *);
void cpu_proc_fork(struct proc *, struct proc *);
paddr_t kvtop(void *);
void isa_defaultirq(void);
int isa_nmi(void);
void kgdb_port_init(void);
void x86_bus_space_init(void);
void x86_bus_space_mallocok(void);
#endif
#if defined(_KERNEL) || defined(_KMEMUSER)
#include <machine/psl.h>
#endif
#define CPU_CONSDEV 1
#define CPU_BIOSBASEMEM 2
#define CPU_BIOSEXTMEM 3
#define CPU_BOOTED_KERNEL 5
#define CPU_DISKINFO 6
#define CPU_FPU_PRESENT 7
#define CPU_OSFXSR 8
#define CPU_SSE 9
#define CPU_SSE2 10
#define CPU_TMLR_MODE 11
#define CPU_TMLR_FREQUENCY 12
#define CPU_TMLR_VOLTAGE 13
#define CPU_TMLR_PERCENTAGE 14
#define CPU_FPU_SAVE 15
#define CPU_FPU_SAVE_SIZE 16
#define CPU_XSAVE_FEATURES 17
#define MAX_BIOSDISKS 16
struct disklist {
int dl_nbiosdisks;
int dl_unused;
struct biosdisk_info {
int bi_dev;
int bi_cyl;
int bi_head;
int bi_sec;
uint64_t bi_lbasecs;
#define BIFLAG_INVALID 0x01
#define BIFLAG_EXTINT13 0x02
int bi_flags;
int bi_unused;
} dl_biosdisks[MAX_BIOSDISKS];
int dl_nnativedisks;
struct nativedisk_info {
char ni_devname[16];
int ni_nmatches;
int ni_biosmatches[MAX_BIOSDISKS];
} dl_nativedisks[1];
};
#endif