// SPDX-License-Identifier: GPL-2.0
/*
 * Test for x86 KVM_SET_PMU_EVENT_FILTER.
 *
 * Copyright (C) 2022, Google LLC.
 *
 * This work is licensed under the terms of the GNU GPL, version 2.
 *
 * Verifies the expected behavior of allow lists and deny lists for
 * virtual PMU events.
 */
#include "kvm_util.h"
#include "pmu.h"
#include "processor.h"
#include "test_util.h"

#define NUM_BRANCHES 42
#define MAX_TEST_EVENTS         10

#define PMU_EVENT_FILTER_INVALID_ACTION         (KVM_PMU_EVENT_DENY + 1)
#define PMU_EVENT_FILTER_INVALID_FLAGS                  (KVM_PMU_EVENT_FLAGS_VALID_MASK << 1)
#define PMU_EVENT_FILTER_INVALID_NEVENTS                (KVM_PMU_EVENT_FILTER_MAX_EVENTS + 1)

struct __kvm_pmu_event_filter {
        __u32 action;
        __u32 nevents;
        __u32 fixed_counter_bitmap;
        __u32 flags;
        __u32 pad[4];
        __u64 events[KVM_PMU_EVENT_FILTER_MAX_EVENTS];
};

/*
 * This event list comprises Intel's known architectural events, plus AMD's
 * Branch Instructions Retired for Zen CPUs.  Note, AMD and Intel use the
 * same encoding for Instructions Retired.
 */
kvm_static_assert(INTEL_ARCH_INSTRUCTIONS_RETIRED == AMD_ZEN_INSTRUCTIONS_RETIRED);

static const struct __kvm_pmu_event_filter base_event_filter = {
        .nevents = ARRAY_SIZE(base_event_filter.events),
        .events = {
                INTEL_ARCH_CPU_CYCLES,
                INTEL_ARCH_INSTRUCTIONS_RETIRED,
                INTEL_ARCH_REFERENCE_CYCLES,
                INTEL_ARCH_LLC_REFERENCES,
                INTEL_ARCH_LLC_MISSES,
                INTEL_ARCH_BRANCHES_RETIRED,
                INTEL_ARCH_BRANCHES_MISPREDICTED,
                INTEL_ARCH_TOPDOWN_SLOTS,
                AMD_ZEN_BRANCHES_RETIRED,
        },
};

struct {
        u64 loads;
        u64 stores;
        u64 loads_stores;
        u64 branches_retired;
        u64 instructions_retired;
} pmc_results;

/*
 * If we encounter a #GP during the guest PMU sanity check, then the guest
 * PMU is not functional. Inform the hypervisor via GUEST_SYNC(0).
 */
static void guest_gp_handler(struct ex_regs *regs)
{
        GUEST_SYNC(-EFAULT);
}

/*
 * Check that we can write a new value to the given MSR and read it back.
 * The caller should provide a non-empty set of bits that are safe to flip.
 *
 * Return on success. GUEST_SYNC(0) on error.
 */
static void check_msr(u32 msr, u64 bits_to_flip)
{
        u64 v = rdmsr(msr) ^ bits_to_flip;

        wrmsr(msr, v);
        if (rdmsr(msr) != v)
                GUEST_SYNC(-EIO);

        v ^= bits_to_flip;
        wrmsr(msr, v);
        if (rdmsr(msr) != v)
                GUEST_SYNC(-EIO);
}

static void run_and_measure_loop(u32 msr_base)
{
        const u64 branches_retired = rdmsr(msr_base + 0);
        const u64 insn_retired = rdmsr(msr_base + 1);

        __asm__ __volatile__("loop ." : "+c"((int){NUM_BRANCHES}));

        pmc_results.branches_retired = rdmsr(msr_base + 0) - branches_retired;
        pmc_results.instructions_retired = rdmsr(msr_base + 1) - insn_retired;
}

static void intel_guest_code(void)
{
        check_msr(MSR_CORE_PERF_GLOBAL_CTRL, 1);
        check_msr(MSR_P6_EVNTSEL0, 0xffff);
        check_msr(MSR_IA32_PMC0, 0xffff);
        GUEST_SYNC(0);

        for (;;) {
                wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0);
                wrmsr(MSR_P6_EVNTSEL0, ARCH_PERFMON_EVENTSEL_ENABLE |
                      ARCH_PERFMON_EVENTSEL_OS | INTEL_ARCH_BRANCHES_RETIRED);
                wrmsr(MSR_P6_EVNTSEL1, ARCH_PERFMON_EVENTSEL_ENABLE |
                      ARCH_PERFMON_EVENTSEL_OS | INTEL_ARCH_INSTRUCTIONS_RETIRED);
                wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0x3);

                run_and_measure_loop(MSR_IA32_PMC0);
                GUEST_SYNC(0);
        }
}

/*
 * To avoid needing a check for CPUID.80000001:ECX.PerfCtrExtCore[bit 23],
 * this code uses the always-available, legacy K7 PMU MSRs, which alias to
 * the first four of the six extended core PMU MSRs.
 */
static void amd_guest_code(void)
{
        check_msr(MSR_K7_EVNTSEL0, 0xffff);
        check_msr(MSR_K7_PERFCTR0, 0xffff);
        GUEST_SYNC(0);

        for (;;) {
                wrmsr(MSR_K7_EVNTSEL0, 0);
                wrmsr(MSR_K7_EVNTSEL0, ARCH_PERFMON_EVENTSEL_ENABLE |
                      ARCH_PERFMON_EVENTSEL_OS | AMD_ZEN_BRANCHES_RETIRED);
                wrmsr(MSR_K7_EVNTSEL1, ARCH_PERFMON_EVENTSEL_ENABLE |
                      ARCH_PERFMON_EVENTSEL_OS | AMD_ZEN_INSTRUCTIONS_RETIRED);

                run_and_measure_loop(MSR_K7_PERFCTR0);
                GUEST_SYNC(0);
        }
}

/*
 * Run the VM to the next GUEST_SYNC(value), and return the value passed
 * to the sync. Any other exit from the guest is fatal.
 */
static u64 run_vcpu_to_sync(struct kvm_vcpu *vcpu)
{
        struct ucall uc;

        vcpu_run(vcpu);
        TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO);
        get_ucall(vcpu, &uc);
        TEST_ASSERT(uc.cmd == UCALL_SYNC,
                    "Received ucall other than UCALL_SYNC: %lu", uc.cmd);
        return uc.args[1];
}

static void run_vcpu_and_sync_pmc_results(struct kvm_vcpu *vcpu)
{
        u64 r;

        memset(&pmc_results, 0, sizeof(pmc_results));
        sync_global_to_guest(vcpu->vm, pmc_results);

        r = run_vcpu_to_sync(vcpu);
        TEST_ASSERT(!r, "Unexpected sync value: 0x%lx", r);

        sync_global_from_guest(vcpu->vm, pmc_results);
}

/*
 * In a nested environment or if the vPMU is disabled, the guest PMU
 * might not work as architected (accessing the PMU MSRs may raise
 * #GP, or writes could simply be discarded). In those situations,
 * there is no point in running these tests. The guest code will perform
 * a sanity check and then GUEST_SYNC(success). In the case of failure,
 * the behavior of the guest on resumption is undefined.
 */
static bool sanity_check_pmu(struct kvm_vcpu *vcpu)
{
        u64 r;

        vm_install_exception_handler(vcpu->vm, GP_VECTOR, guest_gp_handler);
        r = run_vcpu_to_sync(vcpu);
        vm_install_exception_handler(vcpu->vm, GP_VECTOR, NULL);

        return !r;
}

/*
 * Remove the first occurrence of 'event' (if any) from the filter's
 * event list.
 */
static void remove_event(struct __kvm_pmu_event_filter *f, u64 event)
{
        bool found = false;
        int i;

        for (i = 0; i < f->nevents; i++) {
                if (found)
                        f->events[i - 1] = f->events[i];
                else
                        found = f->events[i] == event;
        }
        if (found)
                f->nevents--;
}

#define ASSERT_PMC_COUNTING_INSTRUCTIONS()                                              \
do {                                                                                    \
        u64 br = pmc_results.branches_retired;                                  \
        u64 ir = pmc_results.instructions_retired;                                      \
        bool br_matched = this_pmu_has_errata(BRANCHES_RETIRED_OVERCOUNT) ?             \
                          br >= NUM_BRANCHES : br == NUM_BRANCHES;                      \
                                                                                        \
        if (br && !br_matched)                                                          \
                pr_info("%s: Branch instructions retired = %lu (expected %u)\n",        \
                        __func__, br, NUM_BRANCHES);                                    \
        TEST_ASSERT(br, "%s: Branch instructions retired = %lu (expected > 0)",         \
                    __func__, br);                                                      \
        TEST_ASSERT(ir, "%s: Instructions retired = %lu (expected > 0)",                \
                    __func__, ir);                                                      \
} while (0)

#define ASSERT_PMC_NOT_COUNTING_INSTRUCTIONS()                                          \
do {                                                                                    \
        u64 br = pmc_results.branches_retired;                                  \
        u64 ir = pmc_results.instructions_retired;                                      \
                                                                                        \
        TEST_ASSERT(!br, "%s: Branch instructions retired = %lu (expected 0)",          \
                    __func__, br);                                                      \
        TEST_ASSERT(!ir, "%s: Instructions retired = %lu (expected 0)",                 \
                    __func__, ir);                                                      \
} while (0)

static void test_without_filter(struct kvm_vcpu *vcpu)
{
        run_vcpu_and_sync_pmc_results(vcpu);

        ASSERT_PMC_COUNTING_INSTRUCTIONS();
}

static void test_with_filter(struct kvm_vcpu *vcpu,
                             struct __kvm_pmu_event_filter *__f)
{
        struct kvm_pmu_event_filter *f = (void *)__f;

        vm_ioctl(vcpu->vm, KVM_SET_PMU_EVENT_FILTER, f);
        run_vcpu_and_sync_pmc_results(vcpu);
}

static void test_amd_deny_list(struct kvm_vcpu *vcpu)
{
        struct __kvm_pmu_event_filter f = {
                .action = KVM_PMU_EVENT_DENY,
                .nevents = 1,
                .events = {
                        RAW_EVENT(0x1C2, 0),
                },
        };

        test_with_filter(vcpu, &f);

        ASSERT_PMC_COUNTING_INSTRUCTIONS();
}

static void test_member_deny_list(struct kvm_vcpu *vcpu)
{
        struct __kvm_pmu_event_filter f = base_event_filter;

        f.action = KVM_PMU_EVENT_DENY;
        test_with_filter(vcpu, &f);

        ASSERT_PMC_NOT_COUNTING_INSTRUCTIONS();
}

static void test_member_allow_list(struct kvm_vcpu *vcpu)
{
        struct __kvm_pmu_event_filter f = base_event_filter;

        f.action = KVM_PMU_EVENT_ALLOW;
        test_with_filter(vcpu, &f);

        ASSERT_PMC_COUNTING_INSTRUCTIONS();
}

static void test_not_member_deny_list(struct kvm_vcpu *vcpu)
{
        struct __kvm_pmu_event_filter f = base_event_filter;

        f.action = KVM_PMU_EVENT_DENY;

        remove_event(&f, INTEL_ARCH_INSTRUCTIONS_RETIRED);
        remove_event(&f, INTEL_ARCH_BRANCHES_RETIRED);
        remove_event(&f, AMD_ZEN_BRANCHES_RETIRED);
        test_with_filter(vcpu, &f);

        ASSERT_PMC_COUNTING_INSTRUCTIONS();
}

static void test_not_member_allow_list(struct kvm_vcpu *vcpu)
{
        struct __kvm_pmu_event_filter f = base_event_filter;

        f.action = KVM_PMU_EVENT_ALLOW;

        remove_event(&f, INTEL_ARCH_INSTRUCTIONS_RETIRED);
        remove_event(&f, INTEL_ARCH_BRANCHES_RETIRED);
        remove_event(&f, AMD_ZEN_BRANCHES_RETIRED);
        test_with_filter(vcpu, &f);

        ASSERT_PMC_NOT_COUNTING_INSTRUCTIONS();
}

/*
 * Verify that setting KVM_PMU_CAP_DISABLE prevents the use of the PMU.
 *
 * Note that KVM_CAP_PMU_CAPABILITY must be invoked prior to creating VCPUs.
 */
static void test_pmu_config_disable(void (*guest_code)(void))
{
        struct kvm_vcpu *vcpu;
        int r;
        struct kvm_vm *vm;

        r = kvm_check_cap(KVM_CAP_PMU_CAPABILITY);
        if (!(r & KVM_PMU_CAP_DISABLE))
                return;

        vm = vm_create(1);

        vm_enable_cap(vm, KVM_CAP_PMU_CAPABILITY, KVM_PMU_CAP_DISABLE);

        vcpu = vm_vcpu_add(vm, 0, guest_code);
        TEST_ASSERT(!sanity_check_pmu(vcpu),
                    "Guest should not be able to use disabled PMU.");

        kvm_vm_free(vm);
}

/*
 * On Intel, check for a non-zero PMU version, at least one general-purpose
 * counter per logical processor, and support for counting the number of branch
 * instructions retired.
 */
static bool use_intel_pmu(void)
{
        return host_cpu_is_intel &&
               kvm_cpu_property(X86_PROPERTY_PMU_VERSION) &&
               kvm_cpu_property(X86_PROPERTY_PMU_NR_GP_COUNTERS) &&
               kvm_pmu_has(X86_PMU_FEATURE_BRANCH_INSNS_RETIRED);
}

/*
 * On AMD, all Family 17h+ CPUs (Zen and its successors) use event encoding
 * 0xc2,0 for Branch Instructions Retired.
 */
static bool use_amd_pmu(void)
{
        return (host_cpu_is_amd && kvm_cpu_family() >= 0x17) ||
               host_cpu_is_hygon;
}

/*
 * "MEM_INST_RETIRED.ALL_LOADS", "MEM_INST_RETIRED.ALL_STORES", and
 * "MEM_INST_RETIRED.ANY" from https://perfmon-events.intel.com/
 * supported on Intel Xeon processors:
 *  - Sapphire Rapids, Ice Lake, Cascade Lake, Skylake.
 */
#define MEM_INST_RETIRED                0xD0
#define MEM_INST_RETIRED_LOAD           RAW_EVENT(MEM_INST_RETIRED, 0x81)
#define MEM_INST_RETIRED_STORE          RAW_EVENT(MEM_INST_RETIRED, 0x82)
#define MEM_INST_RETIRED_LOAD_STORE     RAW_EVENT(MEM_INST_RETIRED, 0x83)

static bool supports_event_mem_inst_retired(void)
{
        u32 eax, ebx, ecx, edx;

        cpuid(1, &eax, &ebx, &ecx, &edx);
        if (x86_family(eax) == 0x6) {
                switch (x86_model(eax)) {
                /* Sapphire Rapids */
                case 0x8F:
                /* Ice Lake */
                case 0x6A:
                /* Skylake */
                /* Cascade Lake */
                case 0x55:
                        return true;
                }
        }

        return false;
}

/*
 * "LS Dispatch", from Processor Programming Reference
 * (PPR) for AMD Family 17h Model 01h, Revision B1 Processors,
 * Preliminary Processor Programming Reference (PPR) for AMD Family
 * 17h Model 31h, Revision B0 Processors, and Preliminary Processor
 * Programming Reference (PPR) for AMD Family 19h Model 01h, Revision
 * B1 Processors Volume 1 of 2.
 */
#define LS_DISPATCH             0x29
#define LS_DISPATCH_LOAD        RAW_EVENT(LS_DISPATCH, BIT(0))
#define LS_DISPATCH_STORE       RAW_EVENT(LS_DISPATCH, BIT(1))
#define LS_DISPATCH_LOAD_STORE  RAW_EVENT(LS_DISPATCH, BIT(2))

#define INCLUDE_MASKED_ENTRY(event_select, mask, match) \
        KVM_PMU_ENCODE_MASKED_ENTRY(event_select, mask, match, false)
#define EXCLUDE_MASKED_ENTRY(event_select, mask, match) \
        KVM_PMU_ENCODE_MASKED_ENTRY(event_select, mask, match, true)

static void masked_events_guest_test(u32 msr_base)
{
        /*
         * The actual value of the counters don't determine the outcome of
         * the test.  Only that they are zero or non-zero.
         */
        const u64 loads = rdmsr(msr_base + 0);
        const u64 stores = rdmsr(msr_base + 1);
        const u64 loads_stores = rdmsr(msr_base + 2);
        int val;


        __asm__ __volatile__("movl $0, %[v];"
                             "movl %[v], %%eax;"
                             "incl %[v];"
                             : [v]"+m"(val) :: "eax");

        pmc_results.loads = rdmsr(msr_base + 0) - loads;
        pmc_results.stores = rdmsr(msr_base + 1) - stores;
        pmc_results.loads_stores = rdmsr(msr_base + 2) - loads_stores;
}

static void intel_masked_events_guest_code(void)
{
        for (;;) {
                wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0);

                wrmsr(MSR_P6_EVNTSEL0 + 0, ARCH_PERFMON_EVENTSEL_ENABLE |
                      ARCH_PERFMON_EVENTSEL_OS | MEM_INST_RETIRED_LOAD);
                wrmsr(MSR_P6_EVNTSEL0 + 1, ARCH_PERFMON_EVENTSEL_ENABLE |
                      ARCH_PERFMON_EVENTSEL_OS | MEM_INST_RETIRED_STORE);
                wrmsr(MSR_P6_EVNTSEL0 + 2, ARCH_PERFMON_EVENTSEL_ENABLE |
                      ARCH_PERFMON_EVENTSEL_OS | MEM_INST_RETIRED_LOAD_STORE);

                wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0x7);

                masked_events_guest_test(MSR_IA32_PMC0);
                GUEST_SYNC(0);
        }
}

static void amd_masked_events_guest_code(void)
{
        for (;;) {
                wrmsr(MSR_K7_EVNTSEL0, 0);
                wrmsr(MSR_K7_EVNTSEL1, 0);
                wrmsr(MSR_K7_EVNTSEL2, 0);

                wrmsr(MSR_K7_EVNTSEL0, ARCH_PERFMON_EVENTSEL_ENABLE |
                      ARCH_PERFMON_EVENTSEL_OS | LS_DISPATCH_LOAD);
                wrmsr(MSR_K7_EVNTSEL1, ARCH_PERFMON_EVENTSEL_ENABLE |
                      ARCH_PERFMON_EVENTSEL_OS | LS_DISPATCH_STORE);
                wrmsr(MSR_K7_EVNTSEL2, ARCH_PERFMON_EVENTSEL_ENABLE |
                      ARCH_PERFMON_EVENTSEL_OS | LS_DISPATCH_LOAD_STORE);

                masked_events_guest_test(MSR_K7_PERFCTR0);
                GUEST_SYNC(0);
        }
}

static void run_masked_events_test(struct kvm_vcpu *vcpu,
                                   const u64 masked_events[],
                                   const int nmasked_events)
{
        struct __kvm_pmu_event_filter f = {
                .nevents = nmasked_events,
                .action = KVM_PMU_EVENT_ALLOW,
                .flags = KVM_PMU_EVENT_FLAG_MASKED_EVENTS,
        };

        memcpy(f.events, masked_events, sizeof(u64) * nmasked_events);
        test_with_filter(vcpu, &f);
}

#define ALLOW_LOADS             BIT(0)
#define ALLOW_STORES            BIT(1)
#define ALLOW_LOADS_STORES      BIT(2)

struct masked_events_test {
        u64 intel_events[MAX_TEST_EVENTS];
        u64 intel_event_end;
        u64 amd_events[MAX_TEST_EVENTS];
        u64 amd_event_end;
        const char *msg;
        u32 flags;
};

/*
 * These are the test cases for the masked events tests.
 *
 * For each test, the guest enables 3 PMU counters (loads, stores,
 * loads + stores).  The filter is then set in KVM with the masked events
 * provided.  The test then verifies that the counters agree with which
 * ones should be counting and which ones should be filtered.
 */
const struct masked_events_test test_cases[] = {
        {
                .intel_events = {
                        INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFF, 0x81),
                },
                .amd_events = {
                        INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xFF, BIT(0)),
                },
                .msg = "Only allow loads.",
                .flags = ALLOW_LOADS,
        }, {
                .intel_events = {
                        INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFF, 0x82),
                },
                .amd_events = {
                        INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xFF, BIT(1)),
                },
                .msg = "Only allow stores.",
                .flags = ALLOW_STORES,
        }, {
                .intel_events = {
                        INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFF, 0x83),
                },
                .amd_events = {
                        INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xFF, BIT(2)),
                },
                .msg = "Only allow loads + stores.",
                .flags = ALLOW_LOADS_STORES,
        }, {
                .intel_events = {
                        INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0x7C, 0),
                        EXCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFF, 0x83),
                },
                .amd_events = {
                        INCLUDE_MASKED_ENTRY(LS_DISPATCH, ~(BIT(0) | BIT(1)), 0),
                },
                .msg = "Only allow loads and stores.",
                .flags = ALLOW_LOADS | ALLOW_STORES,
        }, {
                .intel_events = {
                        INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0x7C, 0),
                        EXCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFF, 0x82),
                },
                .amd_events = {
                        INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xF8, 0),
                        EXCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xFF, BIT(1)),
                },
                .msg = "Only allow loads and loads + stores.",
                .flags = ALLOW_LOADS | ALLOW_LOADS_STORES
        }, {
                .intel_events = {
                        INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFE, 0x82),
                },
                .amd_events = {
                        INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xF8, 0),
                        EXCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xFF, BIT(0)),
                },
                .msg = "Only allow stores and loads + stores.",
                .flags = ALLOW_STORES | ALLOW_LOADS_STORES
        }, {
                .intel_events = {
                        INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0x7C, 0),
                },
                .amd_events = {
                        INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xF8, 0),
                },
                .msg = "Only allow loads, stores, and loads + stores.",
                .flags = ALLOW_LOADS | ALLOW_STORES | ALLOW_LOADS_STORES
        },
};

static int append_test_events(const struct masked_events_test *test,
                              u64 *events, int nevents)
{
        const u64 *evts;
        int i;

        evts = use_intel_pmu() ? test->intel_events : test->amd_events;
        for (i = 0; i < MAX_TEST_EVENTS; i++) {
                if (evts[i] == 0)
                        break;

                events[nevents + i] = evts[i];
        }

        return nevents + i;
}

static bool bool_eq(bool a, bool b)
{
        return a == b;
}

static void run_masked_events_tests(struct kvm_vcpu *vcpu, u64 *events,
                                    int nevents)
{
        int ntests = ARRAY_SIZE(test_cases);
        int i, n;

        for (i = 0; i < ntests; i++) {
                const struct masked_events_test *test = &test_cases[i];

                /* Do any test case events overflow MAX_TEST_EVENTS? */
                assert(test->intel_event_end == 0);
                assert(test->amd_event_end == 0);

                n = append_test_events(test, events, nevents);

                run_masked_events_test(vcpu, events, n);

                TEST_ASSERT(bool_eq(pmc_results.loads, test->flags & ALLOW_LOADS) &&
                            bool_eq(pmc_results.stores, test->flags & ALLOW_STORES) &&
                            bool_eq(pmc_results.loads_stores,
                                    test->flags & ALLOW_LOADS_STORES),
                            "%s  loads: %lu, stores: %lu, loads + stores: %lu",
                            test->msg, pmc_results.loads, pmc_results.stores,
                            pmc_results.loads_stores);
        }
}

static void add_dummy_events(u64 *events, int nevents)
{
        int i;

        for (i = 0; i < nevents; i++) {
                int event_select = i % 0xFF;
                bool exclude = ((i % 4) == 0);

                if (event_select == MEM_INST_RETIRED ||
                    event_select == LS_DISPATCH)
                        event_select++;

                events[i] = KVM_PMU_ENCODE_MASKED_ENTRY(event_select, 0,
                                                        0, exclude);
        }
}

static void test_masked_events(struct kvm_vcpu *vcpu)
{
        int nevents = KVM_PMU_EVENT_FILTER_MAX_EVENTS - MAX_TEST_EVENTS;
        u64 events[KVM_PMU_EVENT_FILTER_MAX_EVENTS];

        /* Run the test cases against a sparse PMU event filter. */
        run_masked_events_tests(vcpu, events, 0);

        /* Run the test cases against a dense PMU event filter. */
        add_dummy_events(events, KVM_PMU_EVENT_FILTER_MAX_EVENTS);
        run_masked_events_tests(vcpu, events, nevents);
}

static int set_pmu_event_filter(struct kvm_vcpu *vcpu,
                                struct __kvm_pmu_event_filter *__f)
{
        struct kvm_pmu_event_filter *f = (void *)__f;

        return __vm_ioctl(vcpu->vm, KVM_SET_PMU_EVENT_FILTER, f);
}

static int set_pmu_single_event_filter(struct kvm_vcpu *vcpu, u64 event,
                                       u32 flags, u32 action)
{
        struct __kvm_pmu_event_filter f = {
                .nevents = 1,
                .flags = flags,
                .action = action,
                .events = {
                        event,
                },
        };

        return set_pmu_event_filter(vcpu, &f);
}

static void test_filter_ioctl(struct kvm_vcpu *vcpu)
{
        u8 nr_fixed_counters = kvm_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS);
        struct __kvm_pmu_event_filter f;
        u64 e = ~0ul;
        int r;

        /*
         * Unfortunately having invalid bits set in event data is expected to
         * pass when flags == 0 (bits other than eventsel+umask).
         */
        r = set_pmu_single_event_filter(vcpu, e, 0, KVM_PMU_EVENT_ALLOW);
        TEST_ASSERT(r == 0, "Valid PMU Event Filter is failing");

        r = set_pmu_single_event_filter(vcpu, e,
                                        KVM_PMU_EVENT_FLAG_MASKED_EVENTS,
                                        KVM_PMU_EVENT_ALLOW);
        TEST_ASSERT(r != 0, "Invalid PMU Event Filter is expected to fail");

        e = KVM_PMU_ENCODE_MASKED_ENTRY(0xff, 0xff, 0xff, 0xf);
        r = set_pmu_single_event_filter(vcpu, e,
                                        KVM_PMU_EVENT_FLAG_MASKED_EVENTS,
                                        KVM_PMU_EVENT_ALLOW);
        TEST_ASSERT(r == 0, "Valid PMU Event Filter is failing");

        f = base_event_filter;
        f.action = PMU_EVENT_FILTER_INVALID_ACTION;
        r = set_pmu_event_filter(vcpu, &f);
        TEST_ASSERT(r, "Set invalid action is expected to fail");

        f = base_event_filter;
        f.flags = PMU_EVENT_FILTER_INVALID_FLAGS;
        r = set_pmu_event_filter(vcpu, &f);
        TEST_ASSERT(r, "Set invalid flags is expected to fail");

        f = base_event_filter;
        f.nevents = PMU_EVENT_FILTER_INVALID_NEVENTS;
        r = set_pmu_event_filter(vcpu, &f);
        TEST_ASSERT(r, "Exceeding the max number of filter events should fail");

        f = base_event_filter;
        f.fixed_counter_bitmap = ~GENMASK_ULL(nr_fixed_counters, 0);
        r = set_pmu_event_filter(vcpu, &f);
        TEST_ASSERT(!r, "Masking non-existent fixed counters should be allowed");
}

static void intel_run_fixed_counter_guest_code(u8 idx)
{
        u8 nr_fixed_counters = this_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS);

        for (;;) {
                wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0);
                wrmsr(MSR_CORE_PERF_FIXED_CTR0 + idx, 0);

                /* Only OS_EN bit is enabled for fixed counter[idx]. */
                wrmsr(MSR_CORE_PERF_FIXED_CTR_CTRL, FIXED_PMC_CTRL(idx, FIXED_PMC_KERNEL));
                wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, FIXED_PMC_GLOBAL_CTRL_ENABLE(idx));
                if (nr_fixed_counters > 1)
                        wrmsr(MSR_CORE_PERF_FIXED_CTR_CTRL,
                              FIXED_PMC_CTRL(idx, FIXED_PMC_KERNEL) |
                              FIXED_PMC_CTRL((idx + 1) % nr_fixed_counters, FIXED_PMC_KERNEL));
                __asm__ __volatile__("loop ." : "+c"((int){NUM_BRANCHES}));
                wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0);

                GUEST_SYNC(rdmsr(MSR_CORE_PERF_FIXED_CTR0 + idx));
        }
}

static u64 test_with_fixed_counter_filter(struct kvm_vcpu *vcpu,
                                          u32 action, u32 bitmap)
{
        struct __kvm_pmu_event_filter f = {
                .action = action,
                .fixed_counter_bitmap = bitmap,
        };
        set_pmu_event_filter(vcpu, &f);

        return run_vcpu_to_sync(vcpu);
}

static u64 test_set_gp_and_fixed_event_filter(struct kvm_vcpu *vcpu,
                                              u32 action,
                                              u32 bitmap)
{
        struct __kvm_pmu_event_filter f = base_event_filter;

        f.action = action;
        f.fixed_counter_bitmap = bitmap;
        set_pmu_event_filter(vcpu, &f);

        return run_vcpu_to_sync(vcpu);
}

static void __test_fixed_counter_bitmap(struct kvm_vcpu *vcpu, u8 idx,
                                        u8 nr_fixed_counters)
{
        unsigned int i;
        u32 bitmap;
        u64 count;

        TEST_ASSERT(nr_fixed_counters < sizeof(bitmap) * 8,
                    "Invalid nr_fixed_counters");

        /*
         * Check the fixed performance counter can count normally when KVM
         * userspace doesn't set any pmu filter.
         */
        count = run_vcpu_to_sync(vcpu);
        TEST_ASSERT(count, "Unexpected count value: %ld", count);

        for (i = 0; i < BIT(nr_fixed_counters); i++) {
                bitmap = BIT(i);
                count = test_with_fixed_counter_filter(vcpu, KVM_PMU_EVENT_ALLOW,
                                                       bitmap);
                TEST_ASSERT_EQ(!!count, !!(bitmap & BIT(idx)));

                count = test_with_fixed_counter_filter(vcpu, KVM_PMU_EVENT_DENY,
                                                       bitmap);
                TEST_ASSERT_EQ(!!count, !(bitmap & BIT(idx)));

                /*
                 * Check that fixed_counter_bitmap has higher priority than
                 * events[] when both are set.
                 */
                count = test_set_gp_and_fixed_event_filter(vcpu,
                                                           KVM_PMU_EVENT_ALLOW,
                                                           bitmap);
                TEST_ASSERT_EQ(!!count, !!(bitmap & BIT(idx)));

                count = test_set_gp_and_fixed_event_filter(vcpu,
                                                           KVM_PMU_EVENT_DENY,
                                                           bitmap);
                TEST_ASSERT_EQ(!!count, !(bitmap & BIT(idx)));
        }
}

static void test_fixed_counter_bitmap(void)
{
        u8 nr_fixed_counters = kvm_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS);
        struct kvm_vm *vm;
        struct kvm_vcpu *vcpu;
        u8 idx;

        /*
         * Check that pmu_event_filter works as expected when it's applied to
         * fixed performance counters.
         */
        for (idx = 0; idx < nr_fixed_counters; idx++) {
                vm = vm_create_with_one_vcpu(&vcpu,
                                             intel_run_fixed_counter_guest_code);
                vcpu_args_set(vcpu, 1, idx);
                __test_fixed_counter_bitmap(vcpu, idx, nr_fixed_counters);
                kvm_vm_free(vm);
        }
}

int main(int argc, char *argv[])
{
        void (*guest_code)(void);
        struct kvm_vcpu *vcpu, *vcpu2 = NULL;
        struct kvm_vm *vm;

        TEST_REQUIRE(kvm_is_pmu_enabled());
        TEST_REQUIRE(kvm_has_cap(KVM_CAP_PMU_EVENT_FILTER));
        TEST_REQUIRE(kvm_has_cap(KVM_CAP_PMU_EVENT_MASKED_EVENTS));

        TEST_REQUIRE(use_intel_pmu() || use_amd_pmu());
        guest_code = use_intel_pmu() ? intel_guest_code : amd_guest_code;

        vm = vm_create_with_one_vcpu(&vcpu, guest_code);

        TEST_REQUIRE(sanity_check_pmu(vcpu));

        if (use_amd_pmu())
                test_amd_deny_list(vcpu);

        test_without_filter(vcpu);
        test_member_deny_list(vcpu);
        test_member_allow_list(vcpu);
        test_not_member_deny_list(vcpu);
        test_not_member_allow_list(vcpu);

        if (use_intel_pmu() &&
            supports_event_mem_inst_retired() &&
            kvm_cpu_property(X86_PROPERTY_PMU_NR_GP_COUNTERS) >= 3)
                vcpu2 = vm_vcpu_add(vm, 2, intel_masked_events_guest_code);
        else if (use_amd_pmu())
                vcpu2 = vm_vcpu_add(vm, 2, amd_masked_events_guest_code);

        if (vcpu2)
                test_masked_events(vcpu2);
        test_filter_ioctl(vcpu);

        kvm_vm_free(vm);

        test_pmu_config_disable(guest_code);
        test_fixed_counter_bitmap();

        return 0;
}