#include <linux/types.h>
#include <linux/acpi.h>
#include <linux/export.h>
#include <linux/bitfield.h>
#include <linux/cpumask.h>
#include <linux/sched/task_stack.h>
#include <linux/panic_notifier.h>
#include <linux/ptrace.h>
#include <linux/random.h>
#include <linux/efi.h>
#include <linux/kdebug.h>
#include <linux/kmsg_dump.h>
#include <linux/sizes.h>
#include <linux/slab.h>
#include <linux/dma-map-ops.h>
#include <linux/set_memory.h>
#include <hyperv/hvhdk.h>
#include <asm/mshyperv.h>
u64 hv_current_partition_id = HV_PARTITION_ID_SELF;
EXPORT_SYMBOL_GPL(hv_current_partition_id);
enum hv_partition_type hv_curr_partition_type;
EXPORT_SYMBOL_GPL(hv_curr_partition_type);
bool __weak hv_nested;
EXPORT_SYMBOL_GPL(hv_nested);
struct ms_hyperv_info __weak ms_hyperv;
EXPORT_SYMBOL_GPL(ms_hyperv);
u32 *hv_vp_index;
EXPORT_SYMBOL_GPL(hv_vp_index);
u32 hv_max_vp_index;
EXPORT_SYMBOL_GPL(hv_max_vp_index);
void * __percpu *hyperv_pcpu_input_arg;
EXPORT_SYMBOL_GPL(hyperv_pcpu_input_arg);
void * __percpu *hyperv_pcpu_output_arg;
EXPORT_SYMBOL_GPL(hyperv_pcpu_output_arg);
static void hv_kmsg_dump_unregister(void);
static struct ctl_table_header *hv_ctl_table_hdr;
u8 * __percpu *hv_synic_eventring_tail;
EXPORT_SYMBOL_GPL(hv_synic_eventring_tail);
void __init hv_common_free(void)
{
unregister_sysctl_table(hv_ctl_table_hdr);
hv_ctl_table_hdr = NULL;
if (ms_hyperv.misc_features & HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE)
hv_kmsg_dump_unregister();
kfree(hv_vp_index);
hv_vp_index = NULL;
free_percpu(hyperv_pcpu_output_arg);
hyperv_pcpu_output_arg = NULL;
free_percpu(hyperv_pcpu_input_arg);
hyperv_pcpu_input_arg = NULL;
free_percpu(hv_synic_eventring_tail);
hv_synic_eventring_tail = NULL;
}
static void *hv_panic_page;
static int sysctl_record_panic_msg = 1;
static const struct ctl_table hv_ctl_table[] = {
{
.procname = "hyperv_record_panic_msg",
.data = &sysctl_record_panic_msg,
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_dointvec_minmax,
.extra1 = SYSCTL_ZERO,
.extra2 = SYSCTL_ONE
},
};
static int hv_die_panic_notify_crash(struct notifier_block *self,
unsigned long val, void *args);
static struct notifier_block hyperv_die_report_block = {
.notifier_call = hv_die_panic_notify_crash,
};
static struct notifier_block hyperv_panic_report_block = {
.notifier_call = hv_die_panic_notify_crash,
};
static int hv_die_panic_notify_crash(struct notifier_block *self,
unsigned long val, void *args)
{
struct pt_regs *regs;
bool is_die;
if (self == &hyperv_panic_report_block) {
is_die = false;
regs = current_pt_regs();
} else {
if (val != DIE_OOPS)
return NOTIFY_DONE;
is_die = true;
regs = ((struct die_args *)args)->regs;
}
if (!sysctl_record_panic_msg || !hv_panic_page)
hyperv_report_panic(regs, val, is_die);
return NOTIFY_DONE;
}
static void hv_kmsg_dump(struct kmsg_dumper *dumper,
struct kmsg_dump_detail *detail)
{
struct kmsg_dump_iter iter;
size_t bytes_written;
if (detail->reason != KMSG_DUMP_PANIC || !sysctl_record_panic_msg)
return;
kmsg_dump_rewind(&iter);
bytes_written = 0;
(void)kmsg_dump_get_buffer(&iter, false, hv_panic_page, HV_HYP_PAGE_SIZE,
&bytes_written);
hv_set_msr(HV_MSR_CRASH_P0, 0);
hv_set_msr(HV_MSR_CRASH_P1, 0);
hv_set_msr(HV_MSR_CRASH_P2, 0);
hv_set_msr(HV_MSR_CRASH_P3, bytes_written ? virt_to_phys(hv_panic_page) : 0);
hv_set_msr(HV_MSR_CRASH_P4, bytes_written);
hv_set_msr(HV_MSR_CRASH_CTL,
(HV_CRASH_CTL_CRASH_NOTIFY |
HV_CRASH_CTL_CRASH_NOTIFY_MSG));
}
static struct kmsg_dumper hv_kmsg_dumper = {
.dump = hv_kmsg_dump,
};
static void hv_kmsg_dump_unregister(void)
{
kmsg_dump_unregister(&hv_kmsg_dumper);
unregister_die_notifier(&hyperv_die_report_block);
atomic_notifier_chain_unregister(&panic_notifier_list,
&hyperv_panic_report_block);
kfree(hv_panic_page);
hv_panic_page = NULL;
}
static void hv_kmsg_dump_register(void)
{
int ret;
hv_panic_page = kzalloc(HV_HYP_PAGE_SIZE, GFP_KERNEL);
if (!hv_panic_page) {
pr_err("Hyper-V: panic message page memory allocation failed\n");
return;
}
ret = kmsg_dump_register(&hv_kmsg_dumper);
if (ret) {
pr_err("Hyper-V: kmsg dump register error 0x%x\n", ret);
kfree(hv_panic_page);
hv_panic_page = NULL;
}
}
static inline bool hv_output_page_exists(void)
{
return hv_parent_partition() || IS_ENABLED(CONFIG_HYPERV_VTL_MODE);
}
void __init hv_get_partition_id(void)
{
struct hv_output_get_partition_id *output;
unsigned long flags;
u64 status, pt_id;
local_irq_save(flags);
output = *this_cpu_ptr(hyperv_pcpu_input_arg);
status = hv_do_hypercall(HVCALL_GET_PARTITION_ID, NULL, output);
pt_id = output->partition_id;
local_irq_restore(flags);
if (hv_result_success(status))
hv_current_partition_id = pt_id;
else
pr_err("Hyper-V: failed to get partition ID: %#x\n",
hv_result(status));
}
#if IS_ENABLED(CONFIG_HYPERV_VTL_MODE)
u8 __init get_vtl(void)
{
u64 control = HV_HYPERCALL_REP_COMP_1 | HVCALL_GET_VP_REGISTERS;
struct hv_input_get_vp_registers *input;
struct hv_output_get_vp_registers *output;
unsigned long flags;
u64 ret;
local_irq_save(flags);
input = *this_cpu_ptr(hyperv_pcpu_input_arg);
output = *this_cpu_ptr(hyperv_pcpu_output_arg);
memset(input, 0, struct_size(input, names, 1));
input->partition_id = HV_PARTITION_ID_SELF;
input->vp_index = HV_VP_INDEX_SELF;
input->input_vtl.as_uint8 = 0;
input->names[0] = HV_REGISTER_VSM_VP_STATUS;
ret = hv_do_hypercall(control, input, output);
if (hv_result_success(ret)) {
ret = output->values[0].reg8 & HV_VTL_MASK;
} else {
pr_err("Failed to get VTL(error: %lld) exiting...\n", ret);
BUG();
}
local_irq_restore(flags);
return ret;
}
#endif
int __init hv_common_init(void)
{
int i;
union hv_hypervisor_version_info version;
if (!hv_get_hypervisor_version(&version))
pr_info("Hyper-V: Hypervisor Build %d.%d.%d.%d-%d-%d\n",
version.major_version, version.minor_version,
version.build_number, version.service_number,
version.service_pack, version.service_branch);
if (hv_is_isolation_supported())
sysctl_record_panic_msg = 0;
if (ms_hyperv.misc_features & HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE) {
u64 hyperv_crash_ctl;
crash_kexec_post_notifiers = true;
pr_info("Hyper-V: enabling crash_kexec_post_notifiers\n");
hv_ctl_table_hdr = register_sysctl("kernel", hv_ctl_table);
if (!hv_ctl_table_hdr)
pr_err("Hyper-V: sysctl table register error");
hyperv_crash_ctl = hv_get_msr(HV_MSR_CRASH_CTL);
if (hyperv_crash_ctl & HV_CRASH_CTL_CRASH_NOTIFY_MSG)
hv_kmsg_dump_register();
register_die_notifier(&hyperv_die_report_block);
atomic_notifier_chain_register(&panic_notifier_list,
&hyperv_panic_report_block);
}
hyperv_pcpu_input_arg = alloc_percpu(void *);
BUG_ON(!hyperv_pcpu_input_arg);
if (hv_output_page_exists()) {
hyperv_pcpu_output_arg = alloc_percpu(void *);
BUG_ON(!hyperv_pcpu_output_arg);
}
if (hv_parent_partition()) {
hv_synic_eventring_tail = alloc_percpu(u8 *);
BUG_ON(!hv_synic_eventring_tail);
}
hv_vp_index = kmalloc_array(nr_cpu_ids, sizeof(*hv_vp_index),
GFP_KERNEL);
if (!hv_vp_index) {
hv_common_free();
return -ENOMEM;
}
for (i = 0; i < nr_cpu_ids; i++)
hv_vp_index[i] = VP_INVAL;
return 0;
}
void __init ms_hyperv_late_init(void)
{
struct acpi_table_header *header;
acpi_status status;
u8 *randomdata;
u32 length, i;
if (!IS_ENABLED(CONFIG_ACPI))
return;
status = acpi_get_table("OEM0", 0, &header);
if (ACPI_FAILURE(status) || !header)
return;
if (strncmp(header->oem_table_id, "MICROSFT", 8))
goto error;
if (header->length < sizeof(*header) + 8 ||
header->length > sizeof(*header) + SZ_4K)
goto error;
length = header->length - sizeof(*header);
randomdata = (u8 *)(header + 1);
pr_debug("Hyper-V: Seeding rng with %d random bytes from ACPI table OEM0\n",
length);
add_bootloader_randomness(randomdata, length);
for (i = 0; i < length; i++) {
header->checksum += randomdata[i];
randomdata[i] = 0;
}
for (i = 0; i < sizeof(header->length); i++)
header->checksum += ((u8 *)&header->length)[i];
header->length = sizeof(*header);
for (i = 0; i < sizeof(header->length); i++)
header->checksum -= ((u8 *)&header->length)[i];
error:
acpi_put_table(header);
}
int hv_common_cpu_init(unsigned int cpu)
{
void **inputarg, **outputarg;
u8 **synic_eventring_tail;
u64 msr_vp_index;
gfp_t flags;
const int pgcount = hv_output_page_exists() ? 2 : 1;
void *mem;
int ret = 0;
flags = irqs_disabled() ? GFP_ATOMIC : GFP_KERNEL;
inputarg = (void **)this_cpu_ptr(hyperv_pcpu_input_arg);
if (!*inputarg) {
mem = kmalloc_array(pgcount, HV_HYP_PAGE_SIZE, flags);
if (!mem)
return -ENOMEM;
if (hv_output_page_exists()) {
outputarg = (void **)this_cpu_ptr(hyperv_pcpu_output_arg);
*outputarg = (char *)mem + HV_HYP_PAGE_SIZE;
}
if (!ms_hyperv.paravisor_present &&
(hv_isolation_type_snp() || hv_isolation_type_tdx())) {
ret = set_memory_decrypted((unsigned long)mem, pgcount);
if (ret) {
return ret;
}
memset(mem, 0x00, pgcount * HV_HYP_PAGE_SIZE);
}
*inputarg = mem;
}
msr_vp_index = hv_get_msr(HV_MSR_VP_INDEX);
hv_vp_index[cpu] = msr_vp_index;
if (msr_vp_index > hv_max_vp_index)
hv_max_vp_index = msr_vp_index;
if (hv_parent_partition()) {
synic_eventring_tail = (u8 **)this_cpu_ptr(hv_synic_eventring_tail);
*synic_eventring_tail = kcalloc(HV_SYNIC_SINT_COUNT,
sizeof(u8), flags);
if (unlikely(!*synic_eventring_tail))
ret = -ENOMEM;
}
return ret;
}
int hv_common_cpu_die(unsigned int cpu)
{
u8 **synic_eventring_tail;
if (hv_parent_partition()) {
synic_eventring_tail = this_cpu_ptr(hv_synic_eventring_tail);
kfree(*synic_eventring_tail);
*synic_eventring_tail = NULL;
}
return 0;
}
bool hv_query_ext_cap(u64 cap_query)
{
static u64 hv_extended_cap __aligned(8);
static bool hv_extended_cap_queried;
u64 status;
if (!(ms_hyperv.priv_high & HV_ENABLE_EXTENDED_HYPERCALLS))
return false;
if (hv_extended_cap_queried)
return hv_extended_cap & cap_query;
status = hv_do_hypercall(HV_EXT_CALL_QUERY_CAPABILITIES, NULL,
&hv_extended_cap);
hv_extended_cap_queried = true;
if (!hv_result_success(status)) {
pr_err("Hyper-V: Extended query capabilities hypercall failed 0x%llx\n",
status);
return false;
}
return hv_extended_cap & cap_query;
}
EXPORT_SYMBOL_GPL(hv_query_ext_cap);
void hv_setup_dma_ops(struct device *dev, bool coherent)
{
arch_setup_dma_ops(dev, coherent);
}
EXPORT_SYMBOL_GPL(hv_setup_dma_ops);
bool hv_is_hibernation_supported(void)
{
return !hv_root_partition() && acpi_sleep_state_supported(ACPI_STATE_S4);
}
EXPORT_SYMBOL_GPL(hv_is_hibernation_supported);
static u64 __hv_read_ref_counter(void)
{
return hv_get_msr(HV_MSR_TIME_REF_COUNT);
}
u64 (*hv_read_reference_counter)(void) = __hv_read_ref_counter;
EXPORT_SYMBOL_GPL(hv_read_reference_counter);
bool __weak hv_is_isolation_supported(void)
{
return false;
}
EXPORT_SYMBOL_GPL(hv_is_isolation_supported);
bool __weak hv_isolation_type_snp(void)
{
return false;
}
EXPORT_SYMBOL_GPL(hv_isolation_type_snp);
bool __weak hv_isolation_type_tdx(void)
{
return false;
}
EXPORT_SYMBOL_GPL(hv_isolation_type_tdx);
void __weak hv_setup_vmbus_handler(void (*handler)(void))
{
}
EXPORT_SYMBOL_GPL(hv_setup_vmbus_handler);
void __weak hv_remove_vmbus_handler(void)
{
}
EXPORT_SYMBOL_GPL(hv_remove_vmbus_handler);
void __weak hv_setup_mshv_handler(void (*handler)(void))
{
}
EXPORT_SYMBOL_GPL(hv_setup_mshv_handler);
void __weak hv_setup_kexec_handler(void (*handler)(void))
{
}
EXPORT_SYMBOL_GPL(hv_setup_kexec_handler);
void __weak hv_remove_kexec_handler(void)
{
}
EXPORT_SYMBOL_GPL(hv_remove_kexec_handler);
void __weak hv_setup_crash_handler(void (*handler)(struct pt_regs *regs))
{
}
EXPORT_SYMBOL_GPL(hv_setup_crash_handler);
void __weak hv_remove_crash_handler(void)
{
}
EXPORT_SYMBOL_GPL(hv_remove_crash_handler);
void __weak hyperv_cleanup(void)
{
}
EXPORT_SYMBOL_GPL(hyperv_cleanup);
u64 __weak hv_ghcb_hypercall(u64 control, void *input, void *output, u32 input_size)
{
return HV_STATUS_INVALID_PARAMETER;
}
EXPORT_SYMBOL_GPL(hv_ghcb_hypercall);
u64 __weak hv_tdx_hypercall(u64 control, u64 param1, u64 param2)
{
return HV_STATUS_INVALID_PARAMETER;
}
EXPORT_SYMBOL_GPL(hv_tdx_hypercall);
void __weak hv_enable_coco_interrupt(unsigned int cpu, unsigned int vector, bool set)
{
}
EXPORT_SYMBOL_GPL(hv_enable_coco_interrupt);
void __weak hv_para_set_sint_proxy(bool enable)
{
}
EXPORT_SYMBOL_GPL(hv_para_set_sint_proxy);
u64 __weak hv_para_get_synic_register(unsigned int reg)
{
return ~0ULL;
}
EXPORT_SYMBOL_GPL(hv_para_get_synic_register);
void __weak hv_para_set_synic_register(unsigned int reg, u64 val)
{
}
EXPORT_SYMBOL_GPL(hv_para_set_synic_register);
void hv_identify_partition_type(void)
{
hv_curr_partition_type = HV_PARTITION_TYPE_GUEST;
if ((ms_hyperv.priv_high & HV_CREATE_PARTITIONS) &&
!(ms_hyperv.priv_high & HV_ISOLATION)) {
if (!IS_ENABLED(CONFIG_MSHV_ROOT)) {
pr_crit("Hyper-V: CONFIG_MSHV_ROOT not enabled!\n");
} else if (ms_hyperv.priv_high & HV_CPU_MANAGEMENT) {
pr_info("Hyper-V: running as root partition\n");
hv_curr_partition_type = HV_PARTITION_TYPE_ROOT;
} else {
pr_info("Hyper-V: running as L1VH partition\n");
hv_curr_partition_type = HV_PARTITION_TYPE_L1VH;
}
}
}
struct hv_status_info {
char *string;
int errno;
u16 code;
};
static const struct hv_status_info hv_status_infos[] = {
#define _STATUS_INFO(status, errno) { #status, (errno), (status) }
_STATUS_INFO(HV_STATUS_SUCCESS, 0),
_STATUS_INFO(HV_STATUS_INVALID_HYPERCALL_CODE, -EINVAL),
_STATUS_INFO(HV_STATUS_INVALID_HYPERCALL_INPUT, -EINVAL),
_STATUS_INFO(HV_STATUS_INVALID_ALIGNMENT, -EIO),
_STATUS_INFO(HV_STATUS_INVALID_PARAMETER, -EINVAL),
_STATUS_INFO(HV_STATUS_ACCESS_DENIED, -EIO),
_STATUS_INFO(HV_STATUS_INVALID_PARTITION_STATE, -EIO),
_STATUS_INFO(HV_STATUS_OPERATION_DENIED, -EIO),
_STATUS_INFO(HV_STATUS_UNKNOWN_PROPERTY, -EIO),
_STATUS_INFO(HV_STATUS_PROPERTY_VALUE_OUT_OF_RANGE, -EIO),
_STATUS_INFO(HV_STATUS_INSUFFICIENT_MEMORY, -ENOMEM),
_STATUS_INFO(HV_STATUS_INSUFFICIENT_CONTIGUOUS_MEMORY, -ENOMEM),
_STATUS_INFO(HV_STATUS_INSUFFICIENT_ROOT_MEMORY, -ENOMEM),
_STATUS_INFO(HV_STATUS_INSUFFICIENT_CONTIGUOUS_ROOT_MEMORY, -ENOMEM),
_STATUS_INFO(HV_STATUS_INVALID_PARTITION_ID, -EINVAL),
_STATUS_INFO(HV_STATUS_INVALID_VP_INDEX, -EINVAL),
_STATUS_INFO(HV_STATUS_NOT_FOUND, -EIO),
_STATUS_INFO(HV_STATUS_INVALID_PORT_ID, -EINVAL),
_STATUS_INFO(HV_STATUS_INVALID_CONNECTION_ID, -EINVAL),
_STATUS_INFO(HV_STATUS_INSUFFICIENT_BUFFERS, -EIO),
_STATUS_INFO(HV_STATUS_NOT_ACKNOWLEDGED, -EIO),
_STATUS_INFO(HV_STATUS_INVALID_VP_STATE, -EIO),
_STATUS_INFO(HV_STATUS_NO_RESOURCES, -EIO),
_STATUS_INFO(HV_STATUS_PROCESSOR_FEATURE_NOT_SUPPORTED, -EIO),
_STATUS_INFO(HV_STATUS_INVALID_LP_INDEX, -EINVAL),
_STATUS_INFO(HV_STATUS_INVALID_REGISTER_VALUE, -EINVAL),
_STATUS_INFO(HV_STATUS_INVALID_LP_INDEX, -EIO),
_STATUS_INFO(HV_STATUS_INVALID_REGISTER_VALUE, -EIO),
_STATUS_INFO(HV_STATUS_OPERATION_FAILED, -EIO),
_STATUS_INFO(HV_STATUS_TIME_OUT, -EIO),
_STATUS_INFO(HV_STATUS_CALL_PENDING, -EIO),
_STATUS_INFO(HV_STATUS_VTL_ALREADY_ENABLED, -EIO),
#undef _STATUS_INFO
};
static inline const struct hv_status_info *find_hv_status_info(u64 hv_status)
{
int i;
u16 code = hv_result(hv_status);
for (i = 0; i < ARRAY_SIZE(hv_status_infos); ++i) {
const struct hv_status_info *info = &hv_status_infos[i];
if (info->code == code)
return info;
}
return NULL;
}
int hv_result_to_errno(u64 status)
{
const struct hv_status_info *info;
if (unlikely(status == U64_MAX))
return -EOPNOTSUPP;
info = find_hv_status_info(status);
if (info)
return info->errno;
return -EIO;
}
EXPORT_SYMBOL_GPL(hv_result_to_errno);
const char *hv_result_to_string(u64 status)
{
const struct hv_status_info *info;
if (unlikely(status == U64_MAX))
return "Hypercall page missing!";
info = find_hv_status_info(status);
if (info)
return info->string;
return "Unknown";
}
EXPORT_SYMBOL_GPL(hv_result_to_string);