/*
 * Copyright 2018, Jérôme Duval, jerome.duval@gmail.com.
 * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de.
 * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org.
 * Copyright 2012, Alex Smith, alex@alex-smith.me.uk.
 * Distributed under the terms of the MIT License.
 *
 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
 * Distributed under the terms of the NewOS License.
 */


#include <cpu.h>

#include <string.h>
#include <stdlib.h>
#include <stdio.h>

#include <algorithm>

#include <ACPI.h>

#include <boot_device.h>
#include <commpage.h>
#include <debug.h>
#include <elf.h>
#include <safemode.h>
#include <smp.h>
#include <util/BitUtils.h>
#include <vm/vm.h>
#include <vm/vm_types.h>
#include <vm/VMAddressSpace.h>

#include <arch_system_info.h>
#include <arch/x86/apic.h>
#include <boot/kernel_args.h>

#include "paging/X86PagingStructures.h"
#include "paging/X86VMTranslationMap.h"


#define DUMP_FEATURE_STRING     1
#define DUMP_CPU_TOPOLOGY       1
#define DUMP_CPU_PATCHLEVEL_TYPE        1


/* cpu vendor info */
struct cpu_vendor_info {
        const char *vendor;
        const char *ident_string[2];
};

static const struct cpu_vendor_info vendor_info[VENDOR_NUM] = {
        { "Intel", { "GenuineIntel" } },
        { "AMD", { "AuthenticAMD" } },
        { "Cyrix", { "CyrixInstead" } },
        { "UMC", { "UMC UMC UMC" } },
        { "NexGen", { "NexGenDriven" } },
        { "Centaur", { "CentaurHauls" } },
        { "Rise", { "RiseRiseRise" } },
        { "Transmeta", { "GenuineTMx86", "TransmetaCPU" } },
        { "NSC", { "Geode by NSC" } },
        { "Hygon", { "HygonGenuine" } },
};

#define K8_SMIONCMPHALT                 (1ULL << 27)
#define K8_C1EONCMPHALT                 (1ULL << 28)

#define K8_CMPHALT                              (K8_SMIONCMPHALT | K8_C1EONCMPHALT)

struct set_mtrr_parameter {
        int32   index;
        uint64  base;
        uint64  length;
        uint8   type;
};

struct set_mtrrs_parameter {
        const x86_mtrr_info*    infos;
        uint32                                  count;
        uint8                                   defaultType;
};


#ifdef __x86_64__
extern addr_t _stac;
extern addr_t _clac;
extern addr_t _xsave;
extern addr_t _xsavec;
extern addr_t _xrstor;
uint64 gXsaveMask;
uint64 gFPUSaveLength = 512;
bool gHasXsave = false;
bool gHasXsavec = false;
#endif

extern "C" void x86_reboot(void);
        // from arch.S

void (*gCpuIdleFunc)(void);
#ifndef __x86_64__
void (*gX86SwapFPUFunc)(void* oldState, const void* newState) = x86_noop_swap;
bool gHasSSE = false;
#endif

static uint32 sCpuRendezvous;
static uint32 sCpuRendezvous2;
static uint32 sCpuRendezvous3;
static vint32 sTSCSyncRendezvous;

/* Some specials for the double fault handler */
static addr_t sDoubleFaultStacks = 0;
static const size_t kDoubleFaultStackSize = 4096;       // size per CPU

static x86_cpu_module_info* sCpuModule;


/* CPU topology information */
static uint32 (*sGetCPUTopologyID)(int currentCPU);
static uint32 sHierarchyMask[CPU_TOPOLOGY_LEVELS];
static uint32 sHierarchyShift[CPU_TOPOLOGY_LEVELS];

/* Cache topology information */
static uint32 sCacheSharingMask[CPU_MAX_CACHE_LEVEL];

static void* sUcodeData = NULL;
static size_t sUcodeDataSize = 0;
static void* sLoadedUcodeUpdate;
static spinlock sUcodeUpdateLock = B_SPINLOCK_INITIALIZER;

static bool sUsePAT = false;


static status_t
acpi_shutdown(bool rebootSystem)
{
        if (debug_debugger_running() || !are_interrupts_enabled())
                return B_ERROR;

        acpi_module_info* acpi;
        if (get_module(B_ACPI_MODULE_NAME, (module_info**)&acpi) != B_OK)
                return B_NOT_SUPPORTED;

        status_t status;
        if (rebootSystem) {
                status = acpi->reboot();
        } else {
                status = acpi->prepare_sleep_state(ACPI_POWER_STATE_OFF, NULL, 0);
                if (status == B_OK) {
                        //cpu_status state = disable_interrupts();
                        status = acpi->enter_sleep_state(ACPI_POWER_STATE_OFF);
                        //restore_interrupts(state);
                }
        }

        put_module(B_ACPI_MODULE_NAME);
        return status;
}


/*!     Disable CPU caches, and invalidate them. */
static void
disable_caches()
{
        x86_write_cr0((x86_read_cr0() | CR0_CACHE_DISABLE)
                & ~CR0_NOT_WRITE_THROUGH);
        wbinvd();
        arch_cpu_global_tlb_invalidate();
}


/*!     Invalidate CPU caches, and enable them. */
static void
enable_caches()
{
        wbinvd();
        arch_cpu_global_tlb_invalidate();
        x86_write_cr0(x86_read_cr0()
                & ~(CR0_CACHE_DISABLE | CR0_NOT_WRITE_THROUGH));
}


static void
set_mtrr(void* _parameter, int cpu)
{
        struct set_mtrr_parameter* parameter
                = (struct set_mtrr_parameter*)_parameter;

        // wait until all CPUs have arrived here
        smp_cpu_rendezvous(&sCpuRendezvous);

        // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU
        // that initiated the call_all_cpus() from doing that again and clearing
        // sCpuRendezvous2 before the last CPU has actually left the loop in
        // smp_cpu_rendezvous();
        if (cpu == 0)
                atomic_set((int32*)&sCpuRendezvous3, 0);

        disable_caches();

        sCpuModule->set_mtrr(parameter->index, parameter->base, parameter->length,
                parameter->type);

        enable_caches();

        // wait until all CPUs have arrived here
        smp_cpu_rendezvous(&sCpuRendezvous2);
        smp_cpu_rendezvous(&sCpuRendezvous3);
}


static void
set_mtrrs(void* _parameter, int cpu)
{
        set_mtrrs_parameter* parameter = (set_mtrrs_parameter*)_parameter;

        // wait until all CPUs have arrived here
        smp_cpu_rendezvous(&sCpuRendezvous);

        // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU
        // that initiated the call_all_cpus() from doing that again and clearing
        // sCpuRendezvous2 before the last CPU has actually left the loop in
        // smp_cpu_rendezvous();
        if (cpu == 0)
                atomic_set((int32*)&sCpuRendezvous3, 0);

        disable_caches();

        sCpuModule->set_mtrrs(parameter->defaultType, parameter->infos,
                parameter->count);

        enable_caches();

        // wait until all CPUs have arrived here
        smp_cpu_rendezvous(&sCpuRendezvous2);
        smp_cpu_rendezvous(&sCpuRendezvous3);
}


static void
init_mtrrs(void* _unused, int cpu)
{
        // wait until all CPUs have arrived here
        smp_cpu_rendezvous(&sCpuRendezvous);

        // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU
        // that initiated the call_all_cpus() from doing that again and clearing
        // sCpuRendezvous2 before the last CPU has actually left the loop in
        // smp_cpu_rendezvous();
        if (cpu == 0)
                atomic_set((int32*)&sCpuRendezvous3, 0);

        disable_caches();

        sCpuModule->init_mtrrs();

        enable_caches();

        // wait until all CPUs have arrived here
        smp_cpu_rendezvous(&sCpuRendezvous2);
        smp_cpu_rendezvous(&sCpuRendezvous3);
}


uint32
x86_count_mtrrs(void)
{
        if (sUsePAT) {
                // When PAT is supported, we completely ignore MTRRs and leave them as
                // initialized by firmware. This follows the suggestion in Intel SDM
                // that these don't usually need to be touched by anything after system
                // init. Using page attributes is the more flexible and modern approach
                // to memory type handling and they can override MTRRs in the critical
                // case of write-combining, usually used for framebuffers.
                dprintf("ignoring MTRRs due to PAT support\n");
                return 0;
        }

        if (sCpuModule == NULL)
                return 0;

        return sCpuModule->count_mtrrs();
}


void
x86_set_mtrr(uint32 index, uint64 base, uint64 length, uint8 type)
{
        struct set_mtrr_parameter parameter;
        parameter.index = index;
        parameter.base = base;
        parameter.length = length;
        parameter.type = type;

        sCpuRendezvous = sCpuRendezvous2 = 0;
        call_all_cpus(&set_mtrr, &parameter);
}


status_t
x86_get_mtrr(uint32 index, uint64* _base, uint64* _length, uint8* _type)
{
        // the MTRRs are identical on all CPUs, so it doesn't matter
        // on which CPU this runs
        return sCpuModule->get_mtrr(index, _base, _length, _type);
}


void
x86_set_mtrrs(uint8 defaultType, const x86_mtrr_info* infos, uint32 count)
{
        if (sCpuModule == NULL)
                return;

        struct set_mtrrs_parameter parameter;
        parameter.defaultType = defaultType;
        parameter.infos = infos;
        parameter.count = count;

        sCpuRendezvous = sCpuRendezvous2 = 0;
        call_all_cpus(&set_mtrrs, &parameter);
}


static void
init_pat(int cpu)
{
        disable_caches();

        uint64 value = x86_read_msr(IA32_MSR_PAT);
        dprintf("PAT MSR on CPU %d before init: %#" B_PRIx64 "\n", cpu, value);

        // Use PAT entry 4 for write-combining, leave the rest as is
        value &= ~(IA32_MSR_PAT_ENTRY_MASK << IA32_MSR_PAT_ENTRY_SHIFT(4));
        value |= IA32_MSR_PAT_TYPE_WRITE_COMBINING << IA32_MSR_PAT_ENTRY_SHIFT(4);

        dprintf("PAT MSR on CPU %d after init: %#" B_PRIx64 "\n", cpu, value);
        x86_write_msr(IA32_MSR_PAT, value);

        enable_caches();
}


void
x86_init_fpu(void)
{
        // All x86_64 CPUs support SSE, don't need to bother checking for it.
#ifndef __x86_64__
        if (!x86_check_feature(IA32_FEATURE_FPU, FEATURE_COMMON)) {
                // No FPU... time to install one in your 386?
                dprintf("%s: Warning: CPU has no reported FPU.\n", __func__);
                gX86SwapFPUFunc = x86_noop_swap;
                return;
        }

        if (!x86_check_feature(IA32_FEATURE_SSE, FEATURE_COMMON)
                || !x86_check_feature(IA32_FEATURE_FXSR, FEATURE_COMMON)) {
                dprintf("%s: CPU has no SSE... just enabling FPU.\n", __func__);
                // we don't have proper SSE support, just enable FPU
                x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU));
                gX86SwapFPUFunc = x86_fnsave_swap;
                return;
        }
#endif

        dprintf("%s: CPU has SSE... enabling FXSR and XMM.\n", __func__);
#ifndef __x86_64__
        // enable OS support for SSE
        x86_write_cr4(x86_read_cr4() | CR4_OS_FXSR | CR4_OS_XMM_EXCEPTION);
        x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU));

        gX86SwapFPUFunc = x86_fxsave_swap;
        gHasSSE = true;
#endif
}


#if DUMP_FEATURE_STRING
static void
dump_feature_string(int currentCPU, cpu_ent* cpu)
{
        char features[768];
        features[0] = 0;

        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FPU)
                strlcat(features, "fpu ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_VME)
                strlcat(features, "vme ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DE)
                strlcat(features, "de ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE)
                strlcat(features, "pse ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TSC)
                strlcat(features, "tsc ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MSR)
                strlcat(features, "msr ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAE)
                strlcat(features, "pae ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCE)
                strlcat(features, "mce ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CX8)
                strlcat(features, "cx8 ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_APIC)
                strlcat(features, "apic ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SEP)
                strlcat(features, "sep ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MTRR)
                strlcat(features, "mtrr ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PGE)
                strlcat(features, "pge ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCA)
                strlcat(features, "mca ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CMOV)
                strlcat(features, "cmov ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAT)
                strlcat(features, "pat ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE36)
                strlcat(features, "pse36 ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSN)
                strlcat(features, "psn ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CLFSH)
                strlcat(features, "clfsh ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DS)
                strlcat(features, "ds ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_ACPI)
                strlcat(features, "acpi ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MMX)
                strlcat(features, "mmx ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FXSR)
                strlcat(features, "fxsr ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE)
                strlcat(features, "sse ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE2)
                strlcat(features, "sse2 ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SS)
                strlcat(features, "ss ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_HTT)
                strlcat(features, "htt ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TM)
                strlcat(features, "tm ", sizeof(features));
        if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PBE)
                strlcat(features, "pbe ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE3)
                strlcat(features, "sse3 ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PCLMULQDQ)
                strlcat(features, "pclmulqdq ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DTES64)
                strlcat(features, "dtes64 ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MONITOR)
                strlcat(features, "monitor ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DSCPL)
                strlcat(features, "dscpl ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_VMX)
                strlcat(features, "vmx ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SMX)
                strlcat(features, "smx ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_EST)
                strlcat(features, "est ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TM2)
                strlcat(features, "tm2 ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSSE3)
                strlcat(features, "ssse3 ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CNXTID)
                strlcat(features, "cnxtid ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_FMA)
                strlcat(features, "fma ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CX16)
                strlcat(features, "cx16 ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_XTPR)
                strlcat(features, "xtpr ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PDCM)
                strlcat(features, "pdcm ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PCID)
                strlcat(features, "pcid ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DCA)
                strlcat(features, "dca ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE4_1)
                strlcat(features, "sse4_1 ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE4_2)
                strlcat(features, "sse4_2 ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_X2APIC)
                strlcat(features, "x2apic ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MOVBE)
                strlcat(features, "movbe ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_POPCNT)
                strlcat(features, "popcnt ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TSCDEADLINE)
                strlcat(features, "tscdeadline ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_AES)
                strlcat(features, "aes ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_XSAVE)
                strlcat(features, "xsave ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_OSXSAVE)
                strlcat(features, "osxsave ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_AVX)
                strlcat(features, "avx ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_F16C)
                strlcat(features, "f16c ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_RDRND)
                strlcat(features, "rdrnd ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_HYPERVISOR)
                strlcat(features, "hypervisor ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT_AMD_ECX] & IA32_FEATURE_AMD_EXT_MWAITX)
                strlcat(features, "mwaitx ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_SYSCALL)
                strlcat(features, "syscall ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_NX)
                strlcat(features, "nx ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_MMXEXT)
                strlcat(features, "mmxext ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_FFXSR)
                strlcat(features, "ffxsr ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_PDPE1GB)
                strlcat(features, "pdpe1gb ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_LONG)
                strlcat(features, "long ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOWEXT)
                strlcat(features, "3dnowext ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOW)
                strlcat(features, "3dnow ", sizeof(features));
        if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_DTS)
                strlcat(features, "dts ", sizeof(features));
        if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ITB)
                strlcat(features, "itb ", sizeof(features));
        if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ARAT)
                strlcat(features, "arat ", sizeof(features));
        if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_PLN)
                strlcat(features, "pln ", sizeof(features));
        if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ECMD)
                strlcat(features, "ecmd ", sizeof(features));
        if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_PTM)
                strlcat(features, "ptm ", sizeof(features));
        if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_HWP)
                strlcat(features, "hwp ", sizeof(features));
        if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_HWP_NOTIFY)
                strlcat(features, "hwp_notify ", sizeof(features));
        if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_HWP_ACTWIN)
                strlcat(features, "hwp_actwin ", sizeof(features));
        if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_HWP_EPP)
                strlcat(features, "hwp_epp ", sizeof(features));
        if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_HWP_PLR)
                strlcat(features, "hwp_plr ", sizeof(features));
        if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_HDC)
                strlcat(features, "hdc ", sizeof(features));
        if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_TBMT3)
                strlcat(features, "tbmt3 ", sizeof(features));
        if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_HWP_CAP)
                strlcat(features, "hwp_cap ", sizeof(features));
        if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_HWP_PECI)
                strlcat(features, "hwp_peci ", sizeof(features));
        if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_HWP_FLEX)
                strlcat(features, "hwp_flex ", sizeof(features));
        if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_HWP_FAST)
                strlcat(features, "hwp_fast ", sizeof(features));
        if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_HW_FEEDBACK)
                strlcat(features, "hw_feedback ", sizeof(features));
        if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_HWP_IGNIDL)
                strlcat(features, "hwp_ignidl ", sizeof(features));
        if (cpu->arch.feature[FEATURE_6_ECX] & IA32_FEATURE_APERFMPERF)
                strlcat(features, "aperfmperf ", sizeof(features));
        if (cpu->arch.feature[FEATURE_6_ECX] & IA32_FEATURE_EPB)
                strlcat(features, "epb ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_TSC_ADJUST)
                strlcat(features, "tsc_adjust ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_SGX)
                strlcat(features, "sgx ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_BMI1)
                strlcat(features, "bmi1 ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_HLE)
                strlcat(features, "hle ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX2)
                strlcat(features, "avx2 ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_SMEP)
                strlcat(features, "smep ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_BMI2)
                strlcat(features, "bmi2 ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_ERMS)
                strlcat(features, "erms ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_INVPCID)
                strlcat(features, "invpcid ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_RTM)
                strlcat(features, "rtm ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_CQM)
                strlcat(features, "cqm ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_MPX)
                strlcat(features, "mpx ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_RDT_A)
                strlcat(features, "rdt_a ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512F)
                strlcat(features, "avx512f ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512DQ)
                strlcat(features, "avx512dq ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_RDSEED)
                strlcat(features, "rdseed ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_ADX)
                strlcat(features, "adx ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_SMAP)
                strlcat(features, "smap ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512IFMA)
                strlcat(features, "avx512ifma ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_PCOMMIT)
                strlcat(features, "pcommit ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_CLFLUSHOPT)
                strlcat(features, "cflushopt ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_CLWB)
                strlcat(features, "clwb ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_INTEL_PT)
                strlcat(features, "intel_pt ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512PF)
                strlcat(features, "avx512pf ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512ER)
                strlcat(features, "avx512er ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512CD)
                strlcat(features, "avx512cd ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_SHA_NI)
                strlcat(features, "sha_ni ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512BW)
                strlcat(features, "avx512bw ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512VI)
                strlcat(features, "avx512vi ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_ECX] & IA32_FEATURE_AVX512VMBI)
                strlcat(features, "avx512vmbi ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_ECX] & IA32_FEATURE_UMIP)
                strlcat(features, "umip ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_ECX] & IA32_FEATURE_PKU)
                strlcat(features, "pku ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_ECX] & IA32_FEATURE_OSPKE)
                strlcat(features, "ospke ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_ECX] & IA32_FEATURE_WAITPKG)
                strlcat(features, "waitpkg ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_ECX] & IA32_FEATURE_AVX512VMBI2)
                strlcat(features, "avx512vmbi2 ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_ECX] & IA32_FEATURE_GFNI)
                strlcat(features, "gfni ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_ECX] & IA32_FEATURE_VAES)
                strlcat(features, "vaes ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_ECX] & IA32_FEATURE_VPCLMULQDQ)
                strlcat(features, "vpclmulqdq ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_ECX] & IA32_FEATURE_AVX512_VNNI)
                strlcat(features, "avx512vnni ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_ECX] & IA32_FEATURE_AVX512_BITALG)
                strlcat(features, "avx512bitalg ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_ECX] & IA32_FEATURE_AVX512_VPOPCNTDQ)
                strlcat(features, "avx512vpopcntdq ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_ECX] & IA32_FEATURE_LA57)
                strlcat(features, "la57 ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_ECX] & IA32_FEATURE_RDPID)
                strlcat(features, "rdpid ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_ECX] & IA32_FEATURE_SGX_LC)
                strlcat(features, "sgx_lc ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_HYBRID_CPU)
                strlcat(features, "hybrid ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_IBRS)
                strlcat(features, "ibrs ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_STIBP)
                strlcat(features, "stibp ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_L1D_FLUSH)
                strlcat(features, "l1d_flush ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_ARCH_CAPABILITIES)
                strlcat(features, "msr_arch ", sizeof(features));
        if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_SSBD)
                strlcat(features, "ssbd ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT_7_EDX] & IA32_FEATURE_AMD_HW_PSTATE)
                strlcat(features, "hwpstate ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT_7_EDX] & IA32_FEATURE_INVARIANT_TSC)
                strlcat(features, "constant_tsc ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT_7_EDX] & IA32_FEATURE_CPB)
                strlcat(features, "cpb ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT_7_EDX] & IA32_FEATURE_PROC_FEEDBACK)
                strlcat(features, "proc_feedback ", sizeof(features));
        if (cpu->arch.feature[FEATURE_D_1_EAX] & IA32_FEATURE_XSAVEOPT)
                strlcat(features, "xsaveopt ", sizeof(features));
        if (cpu->arch.feature[FEATURE_D_1_EAX] & IA32_FEATURE_XSAVEC)
                strlcat(features, "xsavec ", sizeof(features));
        if (cpu->arch.feature[FEATURE_D_1_EAX] & IA32_FEATURE_XGETBV1)
                strlcat(features, "xgetbv1 ", sizeof(features));
        if (cpu->arch.feature[FEATURE_D_1_EAX] & IA32_FEATURE_XSAVES)
                strlcat(features, "xsaves ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT_8_EBX] & IA32_FEATURE_CLZERO)
                strlcat(features, "clzero ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT_8_EBX] & IA32_FEATURE_IBPB)
                strlcat(features, "ibpb ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT_8_EBX] & IA32_FEATURE_AMD_SSBD)
                strlcat(features, "amd_ssbd ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT_8_EBX] & IA32_FEATURE_VIRT_SSBD)
                strlcat(features, "virt_ssbd ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT_8_EBX] & IA32_FEATURE_AMD_SSB_NO)
                strlcat(features, "amd_ssb_no ", sizeof(features));
        if (cpu->arch.feature[FEATURE_EXT_8_EBX] & IA32_FEATURE_CPPC)
                strlcat(features, "cppc ", sizeof(features));
        dprintf("CPU %d: features: %s\n", currentCPU, features);
}
#endif  // DUMP_FEATURE_STRING


static void
compute_cpu_hierarchy_masks(int maxLogicalID, int maxCoreID)
{
        ASSERT(maxLogicalID >= maxCoreID);
        const int kMaxSMTID = maxLogicalID / maxCoreID;

        sHierarchyMask[CPU_TOPOLOGY_SMT] = kMaxSMTID - 1;
        sHierarchyShift[CPU_TOPOLOGY_SMT] = 0;

        sHierarchyMask[CPU_TOPOLOGY_CORE] = (maxCoreID - 1) * kMaxSMTID;
        sHierarchyShift[CPU_TOPOLOGY_CORE]
                = count_set_bits(sHierarchyMask[CPU_TOPOLOGY_SMT]);

        const uint32 kSinglePackageMask = sHierarchyMask[CPU_TOPOLOGY_SMT]
                | sHierarchyMask[CPU_TOPOLOGY_CORE];
        sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~kSinglePackageMask;
        sHierarchyShift[CPU_TOPOLOGY_PACKAGE] = count_set_bits(kSinglePackageMask);
}


static uint32
get_cpu_legacy_initial_apic_id(int /* currentCPU */)
{
        cpuid_info cpuid;
        get_current_cpuid(&cpuid, 1, 0);
        return cpuid.regs.ebx >> 24;
}


static inline status_t
detect_amd_cpu_topology(uint32 maxBasicLeaf, uint32 maxExtendedLeaf)
{
        sGetCPUTopologyID = get_cpu_legacy_initial_apic_id;

        cpuid_info cpuid;
        get_current_cpuid(&cpuid, 1, 0);
        int maxLogicalID = next_power_of_2((cpuid.regs.ebx >> 16) & 0xff);

        int maxCoreID = 1;
        if (maxExtendedLeaf >= 0x80000008) {
                get_current_cpuid(&cpuid, 0x80000008, 0);
                maxCoreID = (cpuid.regs.ecx >> 12) & 0xf;
                if (maxCoreID != 0)
                        maxCoreID = 1 << maxCoreID;
                else
                        maxCoreID = next_power_of_2((cpuid.regs.edx & 0xf) + 1);
        }

        if (maxExtendedLeaf >= 0x80000001) {
                get_current_cpuid(&cpuid, 0x80000001, 0);
                if (x86_check_feature(IA32_FEATURE_AMD_EXT_CMPLEGACY,
                                FEATURE_EXT_AMD_ECX))
                        maxCoreID = maxLogicalID;
        }

        compute_cpu_hierarchy_masks(maxLogicalID, maxCoreID);

        return B_OK;
}


static void
detect_amd_cache_topology(uint32 maxExtendedLeaf)
{
        if (!x86_check_feature(IA32_FEATURE_AMD_EXT_TOPOLOGY, FEATURE_EXT_AMD_ECX))
                return;

        if (maxExtendedLeaf < 0x8000001d)
                return;

        uint8 hierarchyLevels[CPU_MAX_CACHE_LEVEL];
        int maxCacheLevel = 0;

        int currentLevel = 0;
        int cacheType;
        do {
                cpuid_info cpuid;
                get_current_cpuid(&cpuid, 0x8000001d, currentLevel);

                cacheType = cpuid.regs.eax & 0x1f;
                if (cacheType == 0)
                        break;

                int cacheLevel = (cpuid.regs.eax >> 5) & 0x7;
                int coresCount = next_power_of_2(((cpuid.regs.eax >> 14) & 0x3f) + 1);
                hierarchyLevels[cacheLevel - 1]
                        = coresCount * (sHierarchyMask[CPU_TOPOLOGY_SMT] + 1);
                maxCacheLevel = std::max(maxCacheLevel, cacheLevel);

                currentLevel++;
        } while (true);

        for (int i = 0; i < maxCacheLevel; i++)
                sCacheSharingMask[i] = ~uint32(hierarchyLevels[i] - 1);
        gCPUCacheLevelCount = maxCacheLevel;
}


static uint32
get_intel_cpu_initial_x2apic_id(int /* currentCPU */)
{
        cpuid_info cpuid;
        get_current_cpuid(&cpuid, 11, 0);
        return cpuid.regs.edx;
}


static inline status_t
detect_intel_cpu_topology_x2apic(uint32 maxBasicLeaf)
{

        uint32 leaf = 0;
        cpuid_info cpuid;
        if (maxBasicLeaf >= 0x1f) {
                get_current_cpuid(&cpuid, 0x1f, 0);
                if (cpuid.regs.ebx != 0)
                        leaf = 0x1f;
        }
        if (maxBasicLeaf >= 0xb && leaf == 0) {
                get_current_cpuid(&cpuid, 0xb, 0);
                if (cpuid.regs.ebx != 0)
                        leaf = 0xb;
        }
        if (leaf == 0)
                return B_UNSUPPORTED;

        uint8 hierarchyLevels[CPU_TOPOLOGY_LEVELS] = { 0 };

        int currentLevel = 0;
        unsigned int levelsSet = 0;
        do {
                cpuid_info cpuid;
                get_current_cpuid(&cpuid, leaf, currentLevel++);
                int levelType = (cpuid.regs.ecx >> 8) & 0xff;
                int levelValue = cpuid.regs.eax & 0x1f;

                if (levelType == 0)
                        break;

                switch (levelType) {
                        case 1: // SMT
                                hierarchyLevels[CPU_TOPOLOGY_SMT] = levelValue;
                                levelsSet |= 1;
                                break;
                        case 2: // core
                                hierarchyLevels[CPU_TOPOLOGY_CORE] = levelValue;
                                levelsSet |= 2;
                                break;
                }

        } while (levelsSet != 3);

        sGetCPUTopologyID = get_intel_cpu_initial_x2apic_id;

        for (int i = 1; i < CPU_TOPOLOGY_LEVELS; i++) {
                if ((levelsSet & (1u << i)) != 0)
                        continue;
                hierarchyLevels[i] = hierarchyLevels[i - 1];
        }

        for (int i = 0; i < CPU_TOPOLOGY_LEVELS; i++) {
                uint32 mask = ~uint32(0);
                if (i < CPU_TOPOLOGY_LEVELS - 1)
                        mask = (1u << hierarchyLevels[i]) - 1;
                if (i > 0)
                        mask &= ~sHierarchyMask[i - 1];
                sHierarchyMask[i] = mask;
                sHierarchyShift[i] = i > 0 ? hierarchyLevels[i - 1] : 0;
        }

        return B_OK;
}


static inline status_t
detect_intel_cpu_topology_legacy(uint32 maxBasicLeaf)
{
        sGetCPUTopologyID = get_cpu_legacy_initial_apic_id;

        cpuid_info cpuid;

        get_current_cpuid(&cpuid, 1, 0);
        int maxLogicalID = next_power_of_2((cpuid.regs.ebx >> 16) & 0xff);

        int maxCoreID = 1;
        if (maxBasicLeaf >= 4) {
                get_current_cpuid(&cpuid, 4, 0);
                maxCoreID = next_power_of_2((cpuid.regs.eax >> 26) + 1);
        }

        compute_cpu_hierarchy_masks(maxLogicalID, maxCoreID);

        return B_OK;
}


static void
detect_intel_cache_topology(uint32 maxBasicLeaf)
{
        if (maxBasicLeaf < 4)
                return;

        uint8 hierarchyLevels[CPU_MAX_CACHE_LEVEL];
        int maxCacheLevel = 0;

        int currentLevel = 0;
        int cacheType;
        do {
                cpuid_info cpuid;
                get_current_cpuid(&cpuid, 4, currentLevel);

                cacheType = cpuid.regs.eax & 0x1f;
                if (cacheType == 0)
                        break;

                int cacheLevel = (cpuid.regs.eax >> 5) & 0x7;
                hierarchyLevels[cacheLevel - 1]
                        = next_power_of_2(((cpuid.regs.eax >> 14) & 0x3f) + 1);
                maxCacheLevel = std::max(maxCacheLevel, cacheLevel);

                currentLevel++;
        } while (true);

        for (int i = 0; i < maxCacheLevel; i++)
                sCacheSharingMask[i] = ~uint32(hierarchyLevels[i] - 1);

        gCPUCacheLevelCount = maxCacheLevel;
}


static uint32
get_simple_cpu_topology_id(int currentCPU)
{
        return currentCPU;
}


static inline int
get_topology_level_id(uint32 id, cpu_topology_level level)
{
        ASSERT(level < CPU_TOPOLOGY_LEVELS);
        return (id & sHierarchyMask[level]) >> sHierarchyShift[level];
}


static void
detect_cpu_topology(int currentCPU, cpu_ent* cpu, uint32 maxBasicLeaf,
        uint32 maxExtendedLeaf)
{
        if (currentCPU == 0) {
                memset(sCacheSharingMask, 0xff, sizeof(sCacheSharingMask));

                status_t result = B_UNSUPPORTED;
                if (x86_check_feature(IA32_FEATURE_HTT, FEATURE_COMMON)) {
                        if (cpu->arch.vendor == VENDOR_AMD
                                || cpu->arch.vendor == VENDOR_HYGON) {
                                result = detect_amd_cpu_topology(maxBasicLeaf, maxExtendedLeaf);

                                if (result == B_OK)
                                        detect_amd_cache_topology(maxExtendedLeaf);
                        }

                        if (cpu->arch.vendor == VENDOR_INTEL) {
                                result = detect_intel_cpu_topology_x2apic(maxBasicLeaf);
                                if (result != B_OK)
                                        result = detect_intel_cpu_topology_legacy(maxBasicLeaf);

                                if (result == B_OK)
                                        detect_intel_cache_topology(maxBasicLeaf);
                        }
                }

                if (result != B_OK) {
                        dprintf("No CPU topology information available.\n");

                        sGetCPUTopologyID = get_simple_cpu_topology_id;

                        sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~uint32(0);
                }
        }

        ASSERT(sGetCPUTopologyID != NULL);
        int topologyID = sGetCPUTopologyID(currentCPU);
        cpu->topology_id[CPU_TOPOLOGY_SMT]
                = get_topology_level_id(topologyID, CPU_TOPOLOGY_SMT);
        cpu->topology_id[CPU_TOPOLOGY_CORE]
                = get_topology_level_id(topologyID, CPU_TOPOLOGY_CORE);
        cpu->topology_id[CPU_TOPOLOGY_PACKAGE]
                = get_topology_level_id(topologyID, CPU_TOPOLOGY_PACKAGE);

        unsigned int i;
        for (i = 0; i < gCPUCacheLevelCount; i++)
                cpu->cache_id[i] = topologyID & sCacheSharingMask[i];
        for (; i < CPU_MAX_CACHE_LEVEL; i++)
                cpu->cache_id[i] = -1;

#if DUMP_CPU_TOPOLOGY
        dprintf("CPU %d: apic id %d, package %d, core %d, smt %d\n", currentCPU,
                topologyID, cpu->topology_id[CPU_TOPOLOGY_PACKAGE],
                cpu->topology_id[CPU_TOPOLOGY_CORE],
                cpu->topology_id[CPU_TOPOLOGY_SMT]);

        if (gCPUCacheLevelCount > 0) {
                char cacheLevels[256];
                unsigned int offset = 0;
                for (i = 0; i < gCPUCacheLevelCount; i++) {
                        offset += snprintf(cacheLevels + offset,
                                        sizeof(cacheLevels) - offset,
                                        " L%d id %d%s", i + 1, cpu->cache_id[i],
                                        i < gCPUCacheLevelCount - 1 ? "," : "");

                        if (offset >= sizeof(cacheLevels))
                                break;
                }

                dprintf("CPU %d: cache sharing:%s\n", currentCPU, cacheLevels);
        }
#endif
}


static void
detect_intel_patch_level(cpu_ent* cpu)
{
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_HYPERVISOR) {
                cpu->arch.patch_level = 0;
                return;
        }

        x86_write_msr(IA32_MSR_UCODE_REV, 0);
        cpuid_info cpuid;
        get_current_cpuid(&cpuid, 1, 0);

        uint64 value = x86_read_msr(IA32_MSR_UCODE_REV);
        cpu->arch.patch_level = value >> 32;
}


static void
detect_amd_patch_level(cpu_ent* cpu)
{
        if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_HYPERVISOR) {
                cpu->arch.patch_level = 0;
                return;
        }

        uint64 value = x86_read_msr(IA32_MSR_UCODE_REV);
        cpu->arch.patch_level = (uint32)value;
}


static struct intel_microcode_header*
find_microcode_intel(addr_t data, size_t size, uint32 patchLevel)
{
        // 9.11.3 Processor Identification
        cpuid_info cpuid;
        get_current_cpuid(&cpuid, 1, 0);
        uint32 signature = cpuid.regs.eax;
        // 9.11.4 Platform Identification
        uint64 platformBits = (x86_read_msr(IA32_MSR_PLATFORM_ID) >> 50) & 0x7;
        uint64 mask = 1 << platformBits;

        while (size > 0) {
                if (size < sizeof(struct intel_microcode_header)) {
                        dprintf("find_microcode_intel update is too small for header\n");
                        break;
                }
                struct intel_microcode_header* header =
                        (struct intel_microcode_header*)data;

                uint32 totalSize = header->total_size;
                uint32 dataSize = header->data_size;
                if (dataSize == 0) {
                        dataSize = 2000;
                        totalSize = sizeof(struct intel_microcode_header)
                                + dataSize;
                }
                if (totalSize > size) {
                        dprintf("find_microcode_intel update is too small for data\n");
                        break;
                }

                uint32* dwords = (uint32*)data;
                // prepare the next update
                size -= totalSize;
                data += totalSize;

                if (header->loader_revision != 1) {
                        dprintf("find_microcode_intel incorrect loader version\n");
                        continue;
                }
                // 9.11.6 The microcode update data requires a 16-byte boundary
                // alignment.
                if (((addr_t)header % 16) != 0) {
                        dprintf("find_microcode_intel incorrect alignment\n");
                        continue;
                }
                uint32 sum = 0;
                for (uint32 i = 0; i < totalSize / 4; i++) {
                        sum += dwords[i];
                }
                if (sum != 0) {
                        dprintf("find_microcode_intel incorrect checksum\n");
                        continue;
                }
                if (patchLevel > header->update_revision) {
                        dprintf("find_microcode_intel update_revision is lower\n");
                        continue;
                }
                if (signature == header->processor_signature
                        && (mask & header->processor_flags) != 0) {
                        return header;
                }
                if (totalSize <= (sizeof(struct intel_microcode_header) + dataSize
                        + sizeof(struct intel_microcode_extended_signature_header))) {
                        continue;
                }
                struct intel_microcode_extended_signature_header* extSigHeader =
                        (struct intel_microcode_extended_signature_header*)((addr_t)header
                                + sizeof(struct intel_microcode_header) + dataSize);
                struct intel_microcode_extended_signature* extended_signature =
                        (struct intel_microcode_extended_signature*)((addr_t)extSigHeader
                                + sizeof(struct intel_microcode_extended_signature_header));
                for (uint32 i = 0; i < extSigHeader->extended_signature_count; i++) {
                        if (signature == extended_signature[i].processor_signature
                                && (mask & extended_signature[i].processor_flags) != 0)
                                return header;
                }
        }
        return NULL;
}


static void
load_microcode_intel(int currentCPU, cpu_ent* cpu)
{
        // serialize for HT cores
        if (currentCPU != 0)
                acquire_spinlock(&sUcodeUpdateLock);
        detect_intel_patch_level(cpu);
        uint32 revision = cpu->arch.patch_level;
        struct intel_microcode_header* update = (struct intel_microcode_header*)sLoadedUcodeUpdate;
        if (update == NULL) {
                update = find_microcode_intel((addr_t)sUcodeData, sUcodeDataSize,
                        revision);
        }
        if (update == NULL) {
                dprintf("CPU %d: no update found\n", currentCPU);
        } else if (update->update_revision != revision) {
                addr_t data = (addr_t)update + sizeof(struct intel_microcode_header);
                wbinvd();
                x86_write_msr(IA32_MSR_UCODE_WRITE, data);
                detect_intel_patch_level(cpu);
                if (revision == cpu->arch.patch_level) {
                        dprintf("CPU %d: update failed\n", currentCPU);
                } else {
                        if (sLoadedUcodeUpdate == NULL)
                                sLoadedUcodeUpdate = update;
                        dprintf("CPU %d: updated from revision 0x%" B_PRIx32 " to 0x%" B_PRIx32
                                "\n", currentCPU, revision, cpu->arch.patch_level);
                }
        }
        if (currentCPU != 0)
                release_spinlock(&sUcodeUpdateLock);
}


static struct amd_microcode_header*
find_microcode_amd(addr_t data, size_t size, uint32 patchLevel)
{
        // 9.11.3 Processor Identification
        cpuid_info cpuid;
        get_current_cpuid(&cpuid, 1, 0);
        uint32 signature = cpuid.regs.eax;

        if (size < sizeof(struct amd_container_header)) {
                dprintf("find_microcode_amd update is too small for header\n");
                return NULL;
        }
        struct amd_container_header* container = (struct amd_container_header*)data;
        if (container->magic != 0x414d44) {
                dprintf("find_microcode_amd update invalid magic\n");
                return NULL;
        }

        size -= sizeof(*container);
        data += sizeof(*container);

        struct amd_section_header* section =
                (struct amd_section_header*)data;
        if (section->type != 0 || section->size == 0) {
                dprintf("find_microcode_amd update first section invalid\n");
                return NULL;
        }

        size -= sizeof(*section);
        data += sizeof(*section);

        amd_equiv_cpu_entry* table = (amd_equiv_cpu_entry*)data;
        size -= section->size;
        data += section->size;

        uint16 equiv_id = 0;
        for (uint32 i = 0; table[i].installed_cpu != 0; i++) {
                if (signature == table[i].equiv_cpu) {
                        equiv_id = table[i].equiv_cpu;
                        dprintf("find_microcode_amd found equiv cpu: %x\n", equiv_id);
                        break;
                }
        }
        if (equiv_id == 0) {
                dprintf("find_microcode_amd update cpu not found in equiv table\n");
                return NULL;
        }

        while (size > sizeof(amd_section_header)) {
                struct amd_section_header* section = (struct amd_section_header*)data;
                size -= sizeof(*section);
                data += sizeof(*section);

                if (section->type != 1 || section->size > size
                        || section->size < sizeof(amd_microcode_header)) {
                        dprintf("find_microcode_amd update firmware section invalid\n");
                        return NULL;
                }
                struct amd_microcode_header* header = (struct amd_microcode_header*)data;
                size -= section->size;
                data += section->size;

                if (header->processor_rev_id != equiv_id) {
                        dprintf("find_microcode_amd update found rev_id %x\n", header->processor_rev_id);
                        continue;
                }
                if (patchLevel >= header->patch_id) {
                        dprintf("find_microcode_intel update_revision is lower\n");
                        continue;
                }
                if (header->nb_dev_id != 0 || header->sb_dev_id != 0) {
                        dprintf("find_microcode_amd update chipset specific firmware\n");
                        continue;
                }
                if (((addr_t)header % 16) != 0) {
                        dprintf("find_microcode_amd incorrect alignment\n");
                        continue;
                }

                return header;
        }
        dprintf("find_microcode_amd no fw update found for this cpu\n");
        return NULL;
}


static void
load_microcode_amd(int currentCPU, cpu_ent* cpu)
{
        // serialize for HT cores
        if (currentCPU != 0)
                acquire_spinlock(&sUcodeUpdateLock);
        detect_amd_patch_level(cpu);
        uint32 revision = cpu->arch.patch_level;
        struct amd_microcode_header* update = (struct amd_microcode_header*)sLoadedUcodeUpdate;
        if (update == NULL) {
                update = find_microcode_amd((addr_t)sUcodeData, sUcodeDataSize,
                        revision);
        }
        if (update != NULL) {
                addr_t data = (addr_t)update;
                wbinvd();

                x86_write_msr(MSR_K8_UCODE_UPDATE, data);

                detect_amd_patch_level(cpu);
                if (revision == cpu->arch.patch_level) {
                        dprintf("CPU %d: update failed\n", currentCPU);
                } else {
                        if (sLoadedUcodeUpdate == NULL)
                                sLoadedUcodeUpdate = update;
                        dprintf("CPU %d: updated from revision 0x%" B_PRIx32 " to 0x%" B_PRIx32
                                "\n", currentCPU, revision, cpu->arch.patch_level);
                }

        } else {
                dprintf("CPU %d: no update found\n", currentCPU);
        }

        if (currentCPU != 0)
                release_spinlock(&sUcodeUpdateLock);
}


static void
load_microcode(int currentCPU)
{
        if (sUcodeData == NULL)
                return;
        cpu_ent* cpu = get_cpu_struct();
        if ((cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_HYPERVISOR) != 0)
                return;
        if (cpu->arch.vendor == VENDOR_INTEL)
                load_microcode_intel(currentCPU, cpu);
        else if (cpu->arch.vendor == VENDOR_AMD)
                load_microcode_amd(currentCPU, cpu);
}


static uint8
get_hybrid_cpu_type()
{
        cpu_ent* cpu = get_cpu_struct();
        if ((cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_HYBRID_CPU) == 0)
                return 0;

#define X86_HYBRID_CPU_TYPE_ID_SHIFT       24
        cpuid_info cpuid;
        get_current_cpuid(&cpuid, 0x1a, 0);
        return cpuid.regs.eax >> X86_HYBRID_CPU_TYPE_ID_SHIFT;
}


static const char*
get_hybrid_cpu_type_string(uint8 type)
{
        switch (type) {
                case 0x20:
                        return "Atom";
                case 0x40:
                        return "Core";
                default:
                        return "";
        }
}


static void
detect_cpu(int currentCPU, bool full = true)
{
        cpu_ent* cpu = get_cpu_struct();
        char vendorString[17];
        cpuid_info cpuid;

        // clear out the cpu info data
        cpu->arch.vendor = VENDOR_UNKNOWN;
        cpu->arch.vendor_name = "UNKNOWN VENDOR";
        cpu->arch.feature[FEATURE_COMMON] = 0;
        cpu->arch.feature[FEATURE_EXT] = 0;
        cpu->arch.feature[FEATURE_EXT_AMD] = 0;
        cpu->arch.feature[FEATURE_7_EBX] = 0;
        cpu->arch.feature[FEATURE_7_ECX] = 0;
        cpu->arch.feature[FEATURE_7_EDX] = 0;
        cpu->arch.feature[FEATURE_D_1_EAX] = 0;
        cpu->arch.model_name[0] = 0;

        // print some fun data
        get_current_cpuid(&cpuid, 0, 0);
        uint32 maxBasicLeaf = cpuid.eax_0.max_eax;

        // build the vendor string
        memset(vendorString, 0, sizeof(vendorString));
        memcpy(vendorString, cpuid.eax_0.vendor_id, sizeof(cpuid.eax_0.vendor_id));

        // get the family, model, stepping
        get_current_cpuid(&cpuid, 1, 0);
        cpu->arch.type = cpuid.eax_1.type;
        cpu->arch.family = cpuid.eax_1.family;
        cpu->arch.extended_family = cpuid.eax_1.extended_family;
        cpu->arch.model = cpuid.eax_1.model;
        cpu->arch.extended_model = cpuid.eax_1.extended_model;
        cpu->arch.stepping = cpuid.eax_1.stepping;
        if (full) {
                dprintf("CPU %d: type %d family %d extended_family %d model %d "
                        "extended_model %d stepping %d, string '%s'\n",
                        currentCPU, cpu->arch.type, cpu->arch.family,
                        cpu->arch.extended_family, cpu->arch.model,
                        cpu->arch.extended_model, cpu->arch.stepping, vendorString);
        }

        // figure out what vendor we have here

        for (int32 i = 0; i < VENDOR_NUM; i++) {
                if (vendor_info[i].ident_string[0]
                        && !strcmp(vendorString, vendor_info[i].ident_string[0])) {
                        cpu->arch.vendor = (x86_vendors)i;
                        cpu->arch.vendor_name = vendor_info[i].vendor;
                        break;
                }
                if (vendor_info[i].ident_string[1]
                        && !strcmp(vendorString, vendor_info[i].ident_string[1])) {
                        cpu->arch.vendor = (x86_vendors)i;
                        cpu->arch.vendor_name = vendor_info[i].vendor;
                        break;
                }
        }

        // see if we can get the model name
        get_current_cpuid(&cpuid, 0x80000000, 0);
        uint32 maxExtendedLeaf = cpuid.eax_0.max_eax;
        if (maxExtendedLeaf >= 0x80000004) {
                // build the model string (need to swap ecx/edx data before copying)
                unsigned int temp;
                memset(cpu->arch.model_name, 0, sizeof(cpu->arch.model_name));

                get_current_cpuid(&cpuid, 0x80000002, 0);
                temp = cpuid.regs.edx;
                cpuid.regs.edx = cpuid.regs.ecx;
                cpuid.regs.ecx = temp;
                memcpy(cpu->arch.model_name, cpuid.as_chars, sizeof(cpuid.as_chars));

                get_current_cpuid(&cpuid, 0x80000003, 0);
                temp = cpuid.regs.edx;
                cpuid.regs.edx = cpuid.regs.ecx;
                cpuid.regs.ecx = temp;
                memcpy(cpu->arch.model_name + 16, cpuid.as_chars,
                        sizeof(cpuid.as_chars));

                get_current_cpuid(&cpuid, 0x80000004, 0);
                temp = cpuid.regs.edx;
                cpuid.regs.edx = cpuid.regs.ecx;
                cpuid.regs.ecx = temp;
                memcpy(cpu->arch.model_name + 32, cpuid.as_chars,
                        sizeof(cpuid.as_chars));

                // some cpus return a right-justified string
                int32 i = 0;
                while (cpu->arch.model_name[i] == ' ')
                        i++;
                if (i > 0) {
                        memmove(cpu->arch.model_name, &cpu->arch.model_name[i],
                                strlen(&cpu->arch.model_name[i]) + 1);
                }

                if (full) {
                        dprintf("CPU %d: vendor '%s' model name '%s'\n",
                                currentCPU, cpu->arch.vendor_name, cpu->arch.model_name);
                }
        } else {
                strlcpy(cpu->arch.model_name, "unknown", sizeof(cpu->arch.model_name));
        }

        // load feature bits
        get_current_cpuid(&cpuid, 1, 0);
        cpu->arch.feature[FEATURE_COMMON] = cpuid.eax_1.features; // edx
        cpu->arch.feature[FEATURE_EXT] = cpuid.eax_1.extended_features; // ecx

        if (!full)
                return;

        if (maxExtendedLeaf >= 0x80000001) {
                get_current_cpuid(&cpuid, 0x80000001, 0);
                if (cpu->arch.vendor == VENDOR_AMD)
                        cpu->arch.feature[FEATURE_EXT_AMD_ECX] = cpuid.regs.ecx; // ecx
                cpu->arch.feature[FEATURE_EXT_AMD] = cpuid.regs.edx; // edx
                if (cpu->arch.vendor != VENDOR_AMD)
                        cpu->arch.feature[FEATURE_EXT_AMD] &= IA32_FEATURES_INTEL_EXT;
        }

        if (maxBasicLeaf >= 6) {
                get_current_cpuid(&cpuid, 6, 0);
                cpu->arch.feature[FEATURE_6_EAX] = cpuid.regs.eax;
                cpu->arch.feature[FEATURE_6_ECX] = cpuid.regs.ecx;
        }

        if (maxBasicLeaf >= 7) {
                get_current_cpuid(&cpuid, 7, 0);
                cpu->arch.feature[FEATURE_7_EBX] = cpuid.regs.ebx;
                cpu->arch.feature[FEATURE_7_ECX] = cpuid.regs.ecx;
                cpu->arch.feature[FEATURE_7_EDX] = cpuid.regs.edx;
        }

        if (maxBasicLeaf >= 0xd) {
                get_current_cpuid(&cpuid, 0xd, 1);
                cpu->arch.feature[FEATURE_D_1_EAX] = cpuid.regs.eax;
        }

        if (maxExtendedLeaf >= 0x80000007) {
                get_current_cpuid(&cpuid, 0x80000007, 0);
                cpu->arch.feature[FEATURE_EXT_7_EDX] = cpuid.regs.edx;
        }

        if (maxExtendedLeaf >= 0x80000008) {
                get_current_cpuid(&cpuid, 0x80000008, 0);
                        cpu->arch.feature[FEATURE_EXT_8_EBX] = cpuid.regs.ebx;
        }

        detect_cpu_topology(currentCPU, cpu, maxBasicLeaf, maxExtendedLeaf);

        if (cpu->arch.vendor == VENDOR_INTEL)
                detect_intel_patch_level(cpu);
        else if (cpu->arch.vendor == VENDOR_AMD)
                detect_amd_patch_level(cpu);

        cpu->arch.hybrid_type = get_hybrid_cpu_type();

#if DUMP_FEATURE_STRING
        dump_feature_string(currentCPU, cpu);
#endif
#if DUMP_CPU_PATCHLEVEL_TYPE
        dprintf("CPU %d: patch_level 0x%" B_PRIx32 "%s%s\n", currentCPU,
                cpu->arch.patch_level,
                cpu->arch.hybrid_type != 0 ? ", hybrid type ": "",
                get_hybrid_cpu_type_string(cpu->arch.hybrid_type));
#endif
}


bool
x86_check_feature(uint32 feature, enum x86_feature_type type)
{
        cpu_ent* cpu = get_cpu_struct();

#if 0
        int i;
        dprintf("x86_check_feature: feature 0x%x, type %d\n", feature, type);
        for (i = 0; i < FEATURE_NUM; i++) {
                dprintf("features %d: 0x%x\n", i, cpu->arch.feature[i]);
        }
#endif

        return (cpu->arch.feature[type] & feature) != 0;
}


bool
x86_use_pat()
{
        return sUsePAT;
}


void*
x86_get_double_fault_stack(int32 cpu, size_t* _size)
{
        *_size = kDoubleFaultStackSize;
        return (void*)(sDoubleFaultStacks + kDoubleFaultStackSize * cpu);
}


/*!     If on a double fault stack, returns the index of the current CPU.
        Otherwise, returns -1.
*/
int32
x86_double_fault_get_cpu()
{
        addr_t stack = x86_get_stack_frame();
        int32 cpu = (stack - sDoubleFaultStacks) / kDoubleFaultStackSize;
        if (cpu < 0 || cpu >= smp_get_num_cpus())
                return -1;
        return cpu;
}


//      #pragma mark -


status_t
arch_cpu_preboot_init_percpu(kernel_args* args, int cpu)
{
        if (cpu == 0) {
                // We can't allocate pages at this stage in the boot process, only virtual addresses.
                sDoubleFaultStacks = vm_allocate_early(args,
                        kDoubleFaultStackSize * smp_get_num_cpus(), 0, 0, 0);
        }

        // On SMP system we want to synchronize the CPUs' TSCs, so system_time()
        // will return consistent values.
        if (smp_get_num_cpus() > 1) {
                // let the first CPU prepare the rendezvous point
                if (cpu == 0)
                        sTSCSyncRendezvous = smp_get_num_cpus() - 1;

                // One CPU after the other will drop out of this loop and be caught by
                // the loop below, until the last CPU (0) gets there. Save for +/- a few
                // cycles the CPUs should pass the second loop at the same time.
                while (sTSCSyncRendezvous != cpu) {
                }

                sTSCSyncRendezvous = cpu - 1;

                while (sTSCSyncRendezvous != -1) {
                }

                // reset TSC to 0
                x86_write_msr(IA32_MSR_TSC, 0);
        }

        x86_descriptors_preboot_init_percpu(args, cpu);

        return B_OK;
}


static void
halt_idle(void)
{
        asm("hlt");
}


static void
amdc1e_noarat_idle(void)
{
        uint64 msr = x86_read_msr(K8_MSR_IPM);
        if (msr & K8_CMPHALT)
                x86_write_msr(K8_MSR_IPM, msr & ~K8_CMPHALT);
        halt_idle();
}


static bool
detect_amdc1e_noarat()
{
        cpu_ent* cpu = get_cpu_struct();

        if (cpu->arch.vendor != VENDOR_AMD)
                return false;

        if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ARAT)
                return false;

        uint32 family = cpu->arch.family + cpu->arch.extended_family;
        uint32 model = (cpu->arch.extended_model << 4) | cpu->arch.model;
        if (family >= 0x12) {
                // Family 0x12 and higher processors support ARAT correctly,
                // but they don't declare it in the CPUID until 0x17 (Zen).
                cpu->arch.feature[FEATURE_6_EAX] |= IA32_FEATURE_ARAT;
                return false;
        }

        // Family lower than 0xf processors doesn't support C1E
        // Family 0xf with model <= 0x40 processors doesn't support C1E
        return (family > 0xf) || (family == 0xf && model > 0x40);
}


static void
init_tsc_with_cpuid(kernel_args* args, uint32* conversionFactor)
{
        cpu_ent* cpu = get_cpu_struct();
        if (cpu->arch.vendor != VENDOR_INTEL)
                return;

        uint32 model = (cpu->arch.extended_model << 4) | cpu->arch.model;
        cpuid_info cpuid;
        get_current_cpuid(&cpuid, 0, 0);
        uint32 maxBasicLeaf = cpuid.eax_0.max_eax;
        if (maxBasicLeaf < IA32_CPUID_LEAF_TSC)
                return;

        get_current_cpuid(&cpuid, IA32_CPUID_LEAF_TSC, 0);
        if (cpuid.regs.eax == 0 || cpuid.regs.ebx == 0)
                return;
        uint32 khz = cpuid.regs.ecx / 1000;
        uint32 denominator = cpuid.regs.eax;
        uint32 numerator = cpuid.regs.ebx;
        if (khz == 0 && model == 0x5f) {
                // CPUID_LEAF_FREQUENCY isn't supported, hardcoding
                khz = 25000;
        }

        if (khz == 0 && maxBasicLeaf >= IA32_CPUID_LEAF_FREQUENCY) {
                // for these CPUs the base frequency is also the tsc frequency
                get_current_cpuid(&cpuid, IA32_CPUID_LEAF_FREQUENCY, 0);
                khz = cpuid.regs.eax * 1000 * denominator / numerator;
        }
        if (khz == 0)
                return;

        dprintf("CPU: using TSC frequency from CPUID\n");
        // compute for microseconds as follows (1000000 << 32) / (tsc freq in Hz),
        // or (1000 << 32) / (tsc freq in kHz)
        *conversionFactor = (1000ULL << 32) / (khz * numerator / denominator);
        // overwrite the bootloader value
        args->arch_args.system_time_cv_factor = *conversionFactor;
}


static void
init_tsc_with_msr(kernel_args* args, uint32* conversionFactor)
{
        cpu_ent* cpuEnt = get_cpu_struct();
        if (cpuEnt->arch.vendor != VENDOR_AMD)
                return;

        uint32 family = cpuEnt->arch.family + cpuEnt->arch.extended_family;
        if (family < 0x10)
                return;
        uint64 value = x86_read_msr(MSR_F10H_HWCR);
        if ((value & HWCR_TSCFREQSEL) == 0)
                return;

        value = x86_read_msr(MSR_F10H_PSTATEDEF(0));
        if ((value & PSTATEDEF_EN) == 0)
                return;
        if (family != 0x17 && family != 0x19)
                return;

        uint64 khz = 200 * 1000;
        uint32 denominator = (value >> 8) & 0x3f;
        if (denominator < 0x8 || denominator > 0x2c)
                return;
        if (denominator > 0x1a && (denominator % 2) == 1)
                return;
        uint32 numerator = value & 0xff;
        if (numerator < 0x10)
                return;

        dprintf("CPU: using TSC frequency from MSR %" B_PRIu64 "\n", khz * numerator / denominator);
        // compute for microseconds as follows (1000000 << 32) / (tsc freq in Hz),
        // or (1000 << 32) / (tsc freq in kHz)
        *conversionFactor = (1000ULL << 32) / (khz * numerator / denominator);
        // overwrite the bootloader value
        args->arch_args.system_time_cv_factor = *conversionFactor;
}


static void
init_tsc(kernel_args* args)
{
        // init the TSC -> system_time() conversion factors

        // try to find the TSC frequency with CPUID
        uint32 conversionFactor = args->arch_args.system_time_cv_factor;
        init_tsc_with_cpuid(args, &conversionFactor);
        init_tsc_with_msr(args, &conversionFactor);
        uint64 conversionFactorNsecs = (uint64)conversionFactor * 1000;

#ifdef __x86_64__
        // The x86_64 system_time() implementation uses 64-bit multiplication and
        // therefore shifting is not necessary for low frequencies (it's also not
        // too likely that there'll be any x86_64 CPUs clocked under 1GHz).
        __x86_setup_system_time((uint64)conversionFactor << 32,
                conversionFactorNsecs);
#else
        if (conversionFactorNsecs >> 32 != 0) {
                // the TSC frequency is < 1 GHz, which forces us to shift the factor
                __x86_setup_system_time(conversionFactor, conversionFactorNsecs >> 16,
                        true);
        } else {
                // the TSC frequency is >= 1 GHz
                __x86_setup_system_time(conversionFactor, conversionFactorNsecs, false);
        }
#endif
}


status_t
arch_cpu_init_percpu(kernel_args* args, int cpu)
{
        detect_cpu(cpu, false);
        load_microcode(cpu);
        detect_cpu(cpu);

        if (cpu == 0) {
                init_tsc(args);

                if (detect_amdc1e_noarat())
                        gCpuIdleFunc = amdc1e_noarat_idle;
                else
                        gCpuIdleFunc = halt_idle;
        }

        if (x86_check_feature(IA32_FEATURE_MCE, FEATURE_COMMON))
                x86_write_cr4(x86_read_cr4() | IA32_CR4_MCE);

        cpu_ent* cpuEnt = get_cpu_struct();
        if (cpu == 0) {
                bool supportsPAT = x86_check_feature(IA32_FEATURE_PAT, FEATURE_COMMON);

                // Pentium II Errata A52 and Pentium III Errata E27 say the upper four
                // entries of the PAT are not useable as the PAT bit is ignored for 4K
                // PTEs. Pentium 4 Errata N46 says the PAT bit can be assumed 0 in some
                // specific cases. To avoid issues, disable PAT on such CPUs.
                bool brokenPAT = cpuEnt->arch.vendor == VENDOR_INTEL
                        && cpuEnt->arch.extended_family == 0
                        && cpuEnt->arch.extended_model == 0
                        && ((cpuEnt->arch.family == 6 && cpuEnt->arch.model <= 13)
                                || (cpuEnt->arch.family == 15 && cpuEnt->arch.model <= 6));

                sUsePAT = supportsPAT && !brokenPAT
                        && !get_safemode_boolean_early(args, B_SAFEMODE_DISABLE_PAT, false);

                if (sUsePAT) {
                        dprintf("using PAT for memory type configuration\n");
                } else {
                        dprintf("not using PAT for memory type configuration (%s)\n",
                                supportsPAT ? (brokenPAT ? "broken" : "disabled")
                                        : "unsupported");
                }
        }

        if (sUsePAT)
                init_pat(cpu);

#ifdef __x86_64__
        // if RDTSCP or RDPID are available write cpu number in TSC_AUX
        if (x86_check_feature(IA32_FEATURE_AMD_EXT_RDTSCP, FEATURE_EXT_AMD)
                || x86_check_feature(IA32_FEATURE_RDPID, FEATURE_7_ECX)) {
                x86_write_msr(IA32_MSR_TSC_AUX, cpu);
        }

        // make LFENCE a dispatch serializing instruction on AMD 64bit
        if (cpuEnt->arch.vendor == VENDOR_AMD) {
                uint32 family = cpuEnt->arch.family + cpuEnt->arch.extended_family;
                if (family >= 0x10 && family != 0x11) {
                        uint64 value = x86_read_msr(MSR_F10H_DE_CFG);
                        if ((value & DE_CFG_SERIALIZE_LFENCE) == 0)
                                x86_write_msr(MSR_F10H_DE_CFG, value | DE_CFG_SERIALIZE_LFENCE);
                }
        }
#endif

        if (x86_check_feature(IA32_FEATURE_APERFMPERF, FEATURE_6_ECX)) {
                gCPU[cpu].arch.mperf_prev = x86_read_msr(IA32_MSR_MPERF);
                gCPU[cpu].arch.aperf_prev = x86_read_msr(IA32_MSR_APERF);
                gCPU[cpu].arch.frequency = 0;
                gCPU[cpu].arch.perf_timestamp = 0;
        }
        return __x86_patch_errata_percpu(cpu);
}


status_t
arch_cpu_init(kernel_args* args)
{
        if (args->ucode_data != NULL
                && args->ucode_data_size > 0) {
                sUcodeData = args->ucode_data;
                sUcodeDataSize = args->ucode_data_size;
        } else {
                dprintf("CPU: no microcode provided\n");
        }

        // Initialize descriptor tables.
        x86_descriptors_init(args);

        return B_OK;
}


#ifdef __x86_64__
static void
enable_smap(void* dummy, int cpu)
{
        x86_write_cr4(x86_read_cr4() | IA32_CR4_SMAP);
}


static void
enable_smep(void* dummy, int cpu)
{
        x86_write_cr4(x86_read_cr4() | IA32_CR4_SMEP);
}


static void
enable_osxsave(void* dummy, int cpu)
{
        x86_write_cr4(x86_read_cr4() | IA32_CR4_OSXSAVE);
}


static void
enable_xsavemask(void* dummy, int cpu)
{
        xsetbv(0, gXsaveMask);
}
#endif


status_t
arch_cpu_init_post_vm(kernel_args* args)
{
        // allocate the area for the double fault stacks
        area_id stacks = create_area("double fault stacks",
                (void**)&sDoubleFaultStacks, B_EXACT_ADDRESS,
                kDoubleFaultStackSize * smp_get_num_cpus(),
                B_FULL_LOCK, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
        if (stacks < B_OK)
                panic("failed to create double fault stacks area: %" B_PRId32, stacks);

        X86PagingStructures* kernelPagingStructures
                = static_cast<X86VMTranslationMap*>(
                        VMAddressSpace::Kernel()->TranslationMap())->PagingStructures();

        // Set active translation map on each CPU.
        for (uint32 i = 0; i < args->num_cpus; i++) {
                gCPU[i].arch.active_paging_structures = kernelPagingStructures;
                kernelPagingStructures->AddReference();
        }

        if (!apic_available())
                x86_init_fpu();
        // else fpu gets set up in smp code

#ifdef __x86_64__
        // if available enable SMEP (Supervisor Memory Execution Protection)
        if (x86_check_feature(IA32_FEATURE_SMEP, FEATURE_7_EBX)) {
                if (!get_safemode_boolean(B_SAFEMODE_DISABLE_SMEP_SMAP, false)) {
                        dprintf("enable SMEP\n");
                        call_all_cpus_sync(&enable_smep, NULL);
                } else
                        dprintf("SMEP disabled per safemode setting\n");
        }

        // if available enable SMAP (Supervisor Memory Access Protection)
        if (x86_check_feature(IA32_FEATURE_SMAP, FEATURE_7_EBX)) {
                if (!get_safemode_boolean(B_SAFEMODE_DISABLE_SMEP_SMAP, false)) {
                        dprintf("enable SMAP\n");
                        call_all_cpus_sync(&enable_smap, NULL);

                        arch_altcodepatch_replace(ALTCODEPATCH_TAG_STAC, &_stac, 3);
                        arch_altcodepatch_replace(ALTCODEPATCH_TAG_CLAC, &_clac, 3);
                } else
                        dprintf("SMAP disabled per safemode setting\n");
        }

        // if available enable XSAVE (XSAVE and extended states)
        gHasXsave = x86_check_feature(IA32_FEATURE_EXT_XSAVE, FEATURE_EXT);
        if (gHasXsave) {
                gHasXsavec = x86_check_feature(IA32_FEATURE_XSAVEC,
                        FEATURE_D_1_EAX);

                call_all_cpus_sync(&enable_osxsave, NULL);
                gXsaveMask = IA32_XCR0_X87 | IA32_XCR0_SSE;
                cpuid_info cpuid;
                get_current_cpuid(&cpuid, IA32_CPUID_LEAF_XSTATE, 0);
                gXsaveMask |= (cpuid.regs.eax & IA32_XCR0_AVX);
                call_all_cpus_sync(&enable_xsavemask, NULL);
                get_current_cpuid(&cpuid, IA32_CPUID_LEAF_XSTATE, 0);
                gFPUSaveLength = cpuid.regs.ebx;
                if (gFPUSaveLength > sizeof(((struct arch_thread *)0)->user_fpu_state))
                        gFPUSaveLength = 832;

                arch_altcodepatch_replace(ALTCODEPATCH_TAG_XSAVE,
                        gHasXsavec ? &_xsavec : &_xsave, 4);
                arch_altcodepatch_replace(ALTCODEPATCH_TAG_XRSTOR,
                        &_xrstor, 4);

                dprintf("enable %s 0x%" B_PRIx64 " %" B_PRId64 "\n",
                        gHasXsavec ? "XSAVEC" : "XSAVE", gXsaveMask, gFPUSaveLength);
        }
#endif

        return B_OK;
}


status_t
arch_cpu_init_post_modules(kernel_args* args)
{
        // initialize CPU module

        void* cookie = open_module_list("cpu");

        while (true) {
                char name[B_FILE_NAME_LENGTH];
                size_t nameLength = sizeof(name);

                if (read_next_module_name(cookie, name, &nameLength) != B_OK
                        || get_module(name, (module_info**)&sCpuModule) == B_OK)
                        break;
        }

        close_module_list(cookie);

        // initialize MTRRs if available
        if (x86_count_mtrrs() > 0) {
                sCpuRendezvous = sCpuRendezvous2 = 0;
                call_all_cpus(&init_mtrrs, NULL);
        }

        size_t threadExitLen = (addr_t)x86_end_userspace_thread_exit
                - (addr_t)x86_userspace_thread_exit;
        addr_t threadExitPosition = fill_commpage_entry(
                COMMPAGE_ENTRY_X86_THREAD_EXIT, (const void*)x86_userspace_thread_exit,
                threadExitLen);

        // add the functions to the commpage image
        image_id image = get_commpage_image();

        elf_add_memory_image_symbol(image, "commpage_thread_exit",
                threadExitPosition, threadExitLen, B_SYMBOL_TYPE_TEXT);

        return B_OK;
}


void
arch_cpu_user_tlb_invalidate(intptr_t context)
{
        if (context != 0 && (intptr_t)x86_read_cr3() != context)
                return;

        x86_write_cr3(x86_read_cr3());
}


void
arch_cpu_global_tlb_invalidate()
{
        uint32 flags = x86_read_cr4();
        if ((flags & IA32_CR4_GLOBAL_PAGES) != 0) {
                // disable and reenable the global pages to flush all TLBs regardless
                // of the global page bit
                x86_write_cr4(flags & ~IA32_CR4_GLOBAL_PAGES);
                x86_write_cr4(flags | IA32_CR4_GLOBAL_PAGES);
        } else {
                arch_cpu_user_tlb_invalidate(0);
        }
}


void
arch_cpu_invalidate_tlb_range(intptr_t context, addr_t start, addr_t end)
{
        if (context != 0 && (intptr_t)x86_read_cr3() != context)
                return;

        while (start <= end) {
                invalidate_TLB(start);
                start += B_PAGE_SIZE;
        }
}


void
arch_cpu_invalidate_tlb_list(intptr_t context, addr_t pages[], int num_pages)
{
        if (context != 0 && (intptr_t)x86_read_cr3() != context)
                return;

        for (int i = 0; i < num_pages; i++)
                invalidate_TLB(pages[i]);
}


status_t
arch_cpu_shutdown(bool rebootSystem)
{
        if (acpi_shutdown(rebootSystem) == B_OK)
                return B_OK;

        if (!rebootSystem) {
#ifndef __x86_64__
                return apm_shutdown();
#else
                return B_NOT_SUPPORTED;
#endif
        }

        cpu_status state = disable_interrupts();

        // try to reset the system using the keyboard controller
        out8(0xfe, 0x64);

        // Give some time to the controller to do its job (0.5s)
        snooze(500000);

        // if that didn't help, try it this way
        x86_reboot();

        restore_interrupts(state);
        return B_ERROR;
}


void
arch_cpu_sync_icache(void* address, size_t length)
{
        // instruction cache is always consistent on x86
}