// SPDX-License-Identifier: GPL-2.0 /* * Intel Transactional Synchronization Extensions (TSX) control. * * Copyright (C) 2019-2021 Intel Corporation * * Author: * Pawan Gupta <pawan.kumar.gupta@linux.intel.com> */ #include <linux/cpufeature.h> #include <asm/cmdline.h> #include <asm/cpu.h> #include <asm/msr.h> #include "cpu.h" #undef pr_fmt #define pr_fmt(fmt) "tsx: " fmt enum tsx_ctrl_states { TSX_CTRL_AUTO, TSX_CTRL_ENABLE, TSX_CTRL_DISABLE, TSX_CTRL_RTM_ALWAYS_ABORT, TSX_CTRL_NOT_SUPPORTED, }; static enum tsx_ctrl_states tsx_ctrl_state __ro_after_init = IS_ENABLED(CONFIG_X86_INTEL_TSX_MODE_AUTO) ? TSX_CTRL_AUTO : IS_ENABLED(CONFIG_X86_INTEL_TSX_MODE_OFF) ? TSX_CTRL_DISABLE : TSX_CTRL_ENABLE; static void tsx_disable(void) { u64 tsx; rdmsrq(MSR_IA32_TSX_CTRL, tsx); /* Force all transactions to immediately abort */ tsx |= TSX_CTRL_RTM_DISABLE; /* * Ensure TSX support is not enumerated in CPUID. * This is visible to userspace and will ensure they * do not waste resources trying TSX transactions that * will always abort. */ tsx |= TSX_CTRL_CPUID_CLEAR; wrmsrq(MSR_IA32_TSX_CTRL, tsx); } static void tsx_enable(void) { u64 tsx; rdmsrq(MSR_IA32_TSX_CTRL, tsx); /* Enable the RTM feature in the cpu */ tsx &= ~TSX_CTRL_RTM_DISABLE; /* * Ensure TSX support is enumerated in CPUID. * This is visible to userspace and will ensure they * can enumerate and use the TSX feature. */ tsx &= ~TSX_CTRL_CPUID_CLEAR; wrmsrq(MSR_IA32_TSX_CTRL, tsx); } static enum tsx_ctrl_states x86_get_tsx_auto_mode(void) { if (boot_cpu_has_bug(X86_BUG_TAA)) return TSX_CTRL_DISABLE; return TSX_CTRL_ENABLE; } /* * Disabling TSX is not a trivial business. * * First of all, there's a CPUID bit: X86_FEATURE_RTM_ALWAYS_ABORT * which says that TSX is practically disabled (all transactions are * aborted by default). When that bit is set, the kernel unconditionally * disables TSX. * * In order to do that, however, it needs to dance a bit: * * 1. The first method to disable it is through MSR_TSX_FORCE_ABORT and * the MSR is present only when *two* CPUID bits are set: * * - X86_FEATURE_RTM_ALWAYS_ABORT * - X86_FEATURE_TSX_FORCE_ABORT * * 2. The second method is for CPUs which do not have the above-mentioned * MSR: those use a different MSR - MSR_IA32_TSX_CTRL and disable TSX * through that one. Those CPUs can also have the initially mentioned * CPUID bit X86_FEATURE_RTM_ALWAYS_ABORT set and for those the same strategy * applies: TSX gets disabled unconditionally. * * When either of the two methods are present, the kernel disables TSX and * clears the respective RTM and HLE feature flags. * * An additional twist in the whole thing presents late microcode loading * which, when done, may cause for the X86_FEATURE_RTM_ALWAYS_ABORT CPUID * bit to be set after the update. * * A subsequent hotplug operation on any logical CPU except the BSP will * cause for the supported CPUID feature bits to get re-detected and, if * RTM and HLE get cleared all of a sudden, but, userspace did consult * them before the update, then funny explosions will happen. Long story * short: the kernel doesn't modify CPUID feature bits after booting. * * That's why, this function's call in init_intel() doesn't clear the * feature flags. */ static void tsx_clear_cpuid(void) { u64 msr; /* * MSR_TFA_TSX_CPUID_CLEAR bit is only present when both CPUID * bits RTM_ALWAYS_ABORT and TSX_FORCE_ABORT are present. */ if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT) && boot_cpu_has(X86_FEATURE_TSX_FORCE_ABORT)) { rdmsrq(MSR_TSX_FORCE_ABORT, msr); msr |= MSR_TFA_TSX_CPUID_CLEAR; wrmsrq(MSR_TSX_FORCE_ABORT, msr); } else if (cpu_feature_enabled(X86_FEATURE_MSR_TSX_CTRL)) { rdmsrq(MSR_IA32_TSX_CTRL, msr); msr |= TSX_CTRL_CPUID_CLEAR; wrmsrq(MSR_IA32_TSX_CTRL, msr); } } /* * Disable TSX development mode * * When the microcode released in Feb 2022 is applied, TSX will be disabled by * default on some processors. MSR 0x122 (TSX_CTRL) and MSR 0x123 * (IA32_MCU_OPT_CTRL) can be used to re-enable TSX for development, doing so is * not recommended for production deployments. In particular, applying MD_CLEAR * flows for mitigation of the Intel TSX Asynchronous Abort (TAA) transient * execution attack may not be effective on these processors when Intel TSX is * enabled with updated microcode. */ static void tsx_dev_mode_disable(void) { u64 mcu_opt_ctrl; /* Check if RTM_ALLOW exists */ if (!boot_cpu_has_bug(X86_BUG_TAA) || !cpu_feature_enabled(X86_FEATURE_MSR_TSX_CTRL) || !cpu_feature_enabled(X86_FEATURE_SRBDS_CTRL)) return; rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_opt_ctrl); if (mcu_opt_ctrl & RTM_ALLOW) { mcu_opt_ctrl &= ~RTM_ALLOW; wrmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_opt_ctrl); setup_force_cpu_cap(X86_FEATURE_RTM_ALWAYS_ABORT); } } static int __init tsx_parse_cmdline(char *str) { if (!str) return -EINVAL; if (!strcmp(str, "on")) { tsx_ctrl_state = TSX_CTRL_ENABLE; } else if (!strcmp(str, "off")) { tsx_ctrl_state = TSX_CTRL_DISABLE; } else if (!strcmp(str, "auto")) { tsx_ctrl_state = TSX_CTRL_AUTO; } else { tsx_ctrl_state = TSX_CTRL_DISABLE; pr_err("invalid option, defaulting to off\n"); } return 0; } early_param("tsx", tsx_parse_cmdline); void __init tsx_init(void) { tsx_dev_mode_disable(); /* * Hardware will always abort a TSX transaction when the CPUID bit * RTM_ALWAYS_ABORT is set. In this case, it is better not to enumerate * CPUID.RTM and CPUID.HLE bits. Clear them here. */ if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT)) { tsx_ctrl_state = TSX_CTRL_RTM_ALWAYS_ABORT; tsx_clear_cpuid(); setup_clear_cpu_cap(X86_FEATURE_RTM); setup_clear_cpu_cap(X86_FEATURE_HLE); return; } /* * TSX is controlled via MSR_IA32_TSX_CTRL. However, support for this * MSR is enumerated by ARCH_CAP_TSX_MSR bit in MSR_IA32_ARCH_CAPABILITIES. * * TSX control (aka MSR_IA32_TSX_CTRL) is only available after a * microcode update on CPUs that have their MSR_IA32_ARCH_CAPABILITIES * bit MDS_NO=1. CPUs with MDS_NO=0 are not planned to get * MSR_IA32_TSX_CTRL support even after a microcode update. Thus, * tsx= cmdline requests will do nothing on CPUs without * MSR_IA32_TSX_CTRL support. */ if (x86_read_arch_cap_msr() & ARCH_CAP_TSX_CTRL_MSR) { setup_force_cpu_cap(X86_FEATURE_MSR_TSX_CTRL); } else { tsx_ctrl_state = TSX_CTRL_NOT_SUPPORTED; return; } if (tsx_ctrl_state == TSX_CTRL_AUTO) tsx_ctrl_state = x86_get_tsx_auto_mode(); if (tsx_ctrl_state == TSX_CTRL_DISABLE) { tsx_disable(); /* * tsx_disable() will change the state of the RTM and HLE CPUID * bits. Clear them here since they are now expected to be not * set. */ setup_clear_cpu_cap(X86_FEATURE_RTM); setup_clear_cpu_cap(X86_FEATURE_HLE); } else if (tsx_ctrl_state == TSX_CTRL_ENABLE) { /* * HW defaults TSX to be enabled at bootup. * We may still need the TSX enable support * during init for special cases like * kexec after TSX is disabled. */ tsx_enable(); /* * tsx_enable() will change the state of the RTM and HLE CPUID * bits. Force them here since they are now expected to be set. */ setup_force_cpu_cap(X86_FEATURE_RTM); setup_force_cpu_cap(X86_FEATURE_HLE); } } void tsx_ap_init(void) { tsx_dev_mode_disable(); if (tsx_ctrl_state == TSX_CTRL_ENABLE) tsx_enable(); else if (tsx_ctrl_state == TSX_CTRL_DISABLE) tsx_disable(); else if (tsx_ctrl_state == TSX_CTRL_RTM_ALWAYS_ABORT) /* See comment over that function for more details. */ tsx_clear_cpuid(); }
