// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright IBM Corp. 1999, 2023
 */

#include <linux/cpuhotplug.h>
#include <linux/sched/task.h>
#include <linux/errno.h>
#include <linux/init.h>
#include <linux/irq.h>
#include <asm/asm-extable.h>
#include <asm/asm-offsets.h>
#include <asm/pfault.h>
#include <asm/diag.h>

#define __SUBCODE_MASK 0x0600
#define __PF_RES_FIELD 0x8000000000000000UL

/*
 * 'pfault' pseudo page faults routines.
 */
static int pfault_disable;

static int __init nopfault(char *str)
{
        pfault_disable = 1;
        return 1;
}
early_param("nopfault", nopfault);

struct pfault_refbk {
        u16 refdiagc;
        u16 reffcode;
        u16 refdwlen;
        u16 refversn;
        u64 refgaddr;
        u64 refselmk;
        u64 refcmpmk;
        u64 reserved;
};

static struct pfault_refbk pfault_init_refbk = {
        .refdiagc = 0x258,
        .reffcode = 0,
        .refdwlen = 5,
        .refversn = 2,
        .refgaddr = __LC_LPP,
        .refselmk = 1UL << 48,
        .refcmpmk = 1UL << 48,
        .reserved = __PF_RES_FIELD
};

int __pfault_init(void)
{
        int rc = -EOPNOTSUPP;

        if (pfault_disable)
                return rc;
        diag_stat_inc(DIAG_STAT_X258);
        asm_inline volatile(
                "       diag    %[refbk],%[rc],0x258\n"
                "0:     nopr    %%r7\n"
                EX_TABLE(0b, 0b)
                : [rc] "+d" (rc)
                : [refbk] "a" (virt_to_phys(&pfault_init_refbk)), "m" (pfault_init_refbk)
                : "cc");
        return rc;
}

static struct pfault_refbk pfault_fini_refbk = {
        .refdiagc = 0x258,
        .reffcode = 1,
        .refdwlen = 5,
        .refversn = 2,
};

void __pfault_fini(void)
{
        if (pfault_disable)
                return;
        diag_stat_inc(DIAG_STAT_X258);
        asm_inline volatile(
                "       diag    %[refbk],0,0x258\n"
                "0:     nopr    %%r7\n"
                EX_TABLE(0b, 0b)
                :
                : [refbk] "a" (virt_to_phys(&pfault_fini_refbk)), "m" (pfault_fini_refbk)
                : "cc");
}

static DEFINE_SPINLOCK(pfault_lock);
static LIST_HEAD(pfault_list);

#define PF_COMPLETE     0x0080

/*
 * The mechanism of our pfault code: if Linux is running as guest, runs a user
 * space process and the user space process accesses a page that the host has
 * paged out we get a pfault interrupt.
 *
 * This allows us, within the guest, to schedule a different process. Without
 * this mechanism the host would have to suspend the whole virtual cpu until
 * the page has been paged in.
 *
 * So when we get such an interrupt then we set the state of the current task
 * to uninterruptible and also set the need_resched flag. Both happens within
 * interrupt context(!). If we later on want to return to user space we
 * recognize the need_resched flag and then call schedule().  It's not very
 * obvious how this works...
 *
 * Of course we have a lot of additional fun with the completion interrupt (->
 * host signals that a page of a process has been paged in and the process can
 * continue to run). This interrupt can arrive on any cpu and, since we have
 * virtual cpus, actually appear before the interrupt that signals that a page
 * is missing.
 */
static void pfault_interrupt(struct ext_code ext_code,
                             unsigned int param32, unsigned long param64)
{
        struct task_struct *tsk;
        __u16 subcode;
        pid_t pid;

        /*
         * Get the external interruption subcode & pfault initial/completion
         * signal bit. VM stores this in the 'cpu address' field associated
         * with the external interrupt.
         */
        subcode = ext_code.subcode;
        if ((subcode & 0xff00) != __SUBCODE_MASK)
                return;
        inc_irq_stat(IRQEXT_PFL);
        /* Get the token (= pid of the affected task). */
        pid = param64 & LPP_PID_MASK;
        rcu_read_lock();
        tsk = find_task_by_pid_ns(pid, &init_pid_ns);
        if (tsk)
                get_task_struct(tsk);
        rcu_read_unlock();
        if (!tsk)
                return;
        spin_lock(&pfault_lock);
        if (subcode & PF_COMPLETE) {
                /* signal bit is set -> a page has been swapped in by VM */
                if (tsk->thread.pfault_wait == 1) {
                        /*
                         * Initial interrupt was faster than the completion
                         * interrupt. pfault_wait is valid. Set pfault_wait
                         * back to zero and wake up the process. This can
                         * safely be done because the task is still sleeping
                         * and can't produce new pfaults.
                         */
                        tsk->thread.pfault_wait = 0;
                        list_del(&tsk->thread.list);
                        wake_up_process(tsk);
                        put_task_struct(tsk);
                } else {
                        /*
                         * Completion interrupt was faster than initial
                         * interrupt. Set pfault_wait to -1 so the initial
                         * interrupt doesn't put the task to sleep.
                         * If the task is not running, ignore the completion
                         * interrupt since it must be a leftover of a PFAULT
                         * CANCEL operation which didn't remove all pending
                         * completion interrupts.
                         */
                        if (task_is_running(tsk))
                                tsk->thread.pfault_wait = -1;
                }
        } else {
                /* signal bit not set -> a real page is missing. */
                if (WARN_ON_ONCE(tsk != current))
                        goto out;
                if (tsk->thread.pfault_wait == 1) {
                        /* Already on the list with a reference: put to sleep */
                        goto block;
                } else if (tsk->thread.pfault_wait == -1) {
                        /*
                         * Completion interrupt was faster than the initial
                         * interrupt (pfault_wait == -1). Set pfault_wait
                         * back to zero and exit.
                         */
                        tsk->thread.pfault_wait = 0;
                } else {
                        /*
                         * Initial interrupt arrived before completion
                         * interrupt. Let the task sleep.
                         * An extra task reference is needed since a different
                         * cpu may set the task state to TASK_RUNNING again
                         * before the scheduler is reached.
                         */
                        get_task_struct(tsk);
                        tsk->thread.pfault_wait = 1;
                        list_add(&tsk->thread.list, &pfault_list);
block:
                        /*
                         * Since this must be a userspace fault, there
                         * is no kernel task state to trample. Rely on the
                         * return to userspace schedule() to block.
                         */
                        __set_current_state(TASK_UNINTERRUPTIBLE);
                        set_need_resched_current();
                }
        }
out:
        spin_unlock(&pfault_lock);
        put_task_struct(tsk);
}

static int pfault_cpu_dead(unsigned int cpu)
{
        struct thread_struct *thread, *next;
        struct task_struct *tsk;

        spin_lock_irq(&pfault_lock);
        list_for_each_entry_safe(thread, next, &pfault_list, list) {
                thread->pfault_wait = 0;
                list_del(&thread->list);
                tsk = container_of(thread, struct task_struct, thread);
                wake_up_process(tsk);
                put_task_struct(tsk);
        }
        spin_unlock_irq(&pfault_lock);
        return 0;
}

static int __init pfault_irq_init(void)
{
        int rc;

        rc = register_external_irq(EXT_IRQ_CP_SERVICE, pfault_interrupt);
        if (rc)
                goto out_extint;
        rc = pfault_init() == 0 ? 0 : -EOPNOTSUPP;
        if (rc)
                goto out_pfault;
        irq_subclass_register(IRQ_SUBCLASS_SERVICE_SIGNAL);
        cpuhp_setup_state_nocalls(CPUHP_S390_PFAULT_DEAD, "s390/pfault:dead",
                                  NULL, pfault_cpu_dead);
        return 0;

out_pfault:
        unregister_external_irq(EXT_IRQ_CP_SERVICE, pfault_interrupt);
out_extint:
        pfault_disable = 1;
        return rc;
}
early_initcall(pfault_irq_init);