/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * This file contains low-level cache management functions
 * used for sleep and CPU speed changes on Apple machines.
 * (In fact the only thing that is Apple-specific is that we assume
 * that we can read from ROM at physical address 0xfff00000.)
 *
 *    Copyright (C) 2004 Paul Mackerras (paulus@samba.org) and
 *                       Benjamin Herrenschmidt (benh@kernel.crashing.org)
 */

#include <asm/processor.h>
#include <asm/ppc_asm.h>
#include <asm/cputable.h>
#include <asm/feature-fixups.h>

/*
 * Flush and disable all data caches (dL1, L2, L3). This is used
 * when going to sleep, when doing a PMU based cpufreq transition,
 * or when "offlining" a CPU on SMP machines. This code is over
 * paranoid, but I've had enough issues with various CPU revs and
 * bugs that I decided it was worth being over cautious
 */

_GLOBAL(flush_disable_caches)
#ifndef CONFIG_PPC_BOOK3S_32
        blr
#else
BEGIN_FTR_SECTION
        b       flush_disable_745x
END_FTR_SECTION_IFSET(CPU_FTR_SPEC7450)
BEGIN_FTR_SECTION
        b       flush_disable_75x
END_FTR_SECTION_IFSET(CPU_FTR_L2CR)
        b       __flush_disable_L1

/* This is the code for G3 and 74[01]0 */
flush_disable_75x:
        mflr    r10

        /* Turn off EE and DR in MSR */
        mfmsr   r11
        rlwinm  r0,r11,0,~MSR_EE
        rlwinm  r0,r0,0,~MSR_DR
        sync
        mtmsr   r0
        isync

        /* Stop DST streams */
BEGIN_FTR_SECTION
        PPC_DSSALL
        sync
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)

        /* Stop DPM */
        mfspr   r8,SPRN_HID0            /* Save SPRN_HID0 in r8 */
        rlwinm  r4,r8,0,12,10           /* Turn off HID0[DPM] */
        sync
        mtspr   SPRN_HID0,r4            /* Disable DPM */
        sync

        /* Disp-flush L1. We have a weird problem here that I never
         * totally figured out. On 750FX, using the ROM for the flush
         * results in a non-working flush. We use that workaround for
         * now until I finally understand what's going on. --BenH
         */

        /* ROM base by default */
        lis     r4,0xfff0
        mfpvr   r3
        srwi    r3,r3,16
        cmplwi  cr0,r3,0x7000
        bne+    1f
        /* RAM base on 750FX */
        li      r4,0
1:      li      r4,0x4000
        mtctr   r4
1:      lwz     r0,0(r4)
        addi    r4,r4,32
        bdnz    1b
        sync
        isync

        /* Disable / invalidate / enable L1 data */
        mfspr   r3,SPRN_HID0
        rlwinm  r3,r3,0,~(HID0_DCE | HID0_ICE)
        mtspr   SPRN_HID0,r3
        sync
        isync
        ori     r3,r3,(HID0_DCE|HID0_DCI|HID0_ICE|HID0_ICFI)
        sync
        isync
        mtspr   SPRN_HID0,r3
        xori    r3,r3,(HID0_DCI|HID0_ICFI)
        mtspr   SPRN_HID0,r3
        sync

        /* Get the current enable bit of the L2CR into r4 */
        mfspr   r5,SPRN_L2CR
        /* Set to data-only (pre-745x bit) */
        oris    r3,r5,L2CR_L2DO@h
        b       2f
        /* When disabling L2, code must be in L1 */
        .balign 32
1:      mtspr   SPRN_L2CR,r3
3:      sync
        isync
        b       1f
2:      b       3f
3:      sync
        isync
        b       1b
1:      /* disp-flush L2. The interesting thing here is that the L2 can be
         * up to 2Mb ... so using the ROM, we'll end up wrapping back to memory
         * but that is probbaly fine. We disp-flush over 4Mb to be safe
         */
        lis     r4,2
        mtctr   r4
        lis     r4,0xfff0
1:      lwz     r0,0(r4)
        addi    r4,r4,32
        bdnz    1b
        sync
        isync
        lis     r4,2
        mtctr   r4
        lis     r4,0xfff0
1:      dcbf    0,r4
        addi    r4,r4,32
        bdnz    1b
        sync
        isync

        /* now disable L2 */
        rlwinm  r5,r5,0,~L2CR_L2E
        b       2f
        /* When disabling L2, code must be in L1 */
        .balign 32
1:      mtspr   SPRN_L2CR,r5
3:      sync
        isync
        b       1f
2:      b       3f
3:      sync
        isync
        b       1b
1:      sync
        isync
        /* Invalidate L2. This is pre-745x, we clear the L2I bit ourselves */
        oris    r4,r5,L2CR_L2I@h
        mtspr   SPRN_L2CR,r4
        sync
        isync

        /* Wait for the invalidation to complete */
1:      mfspr   r3,SPRN_L2CR
        rlwinm. r0,r3,0,31,31
        bne     1b

        /* Clear L2I */
        xoris   r4,r4,L2CR_L2I@h
        sync
        mtspr   SPRN_L2CR,r4
        sync

        /* now disable the L1 data cache */
        mfspr   r0,SPRN_HID0
        rlwinm  r0,r0,0,~(HID0_DCE|HID0_ICE)
        mtspr   SPRN_HID0,r0
        sync
        isync

        /* Restore HID0[DPM] to whatever it was before */
        sync
        mfspr   r0,SPRN_HID0
        rlwimi  r0,r8,0,11,11           /* Turn back HID0[DPM] */
        mtspr   SPRN_HID0,r0
        sync

        /* restore DR and EE */
        sync
        mtmsr   r11
        isync

        mtlr    r10
        blr
_ASM_NOKPROBE_SYMBOL(flush_disable_75x)

/* This code is for 745x processors */
flush_disable_745x:
        /* Turn off EE and DR in MSR */
        mfmsr   r11
        rlwinm  r0,r11,0,~MSR_EE
        rlwinm  r0,r0,0,~MSR_DR
        sync
        mtmsr   r0
        isync

        /* Stop prefetch streams */
        PPC_DSSALL
        sync

        /* Disable L2 prefetching */
        mfspr   r0,SPRN_MSSCR0
        rlwinm  r0,r0,0,0,29
        mtspr   SPRN_MSSCR0,r0
        sync
        isync
        lis     r4,0
        dcbf    0,r4
        dcbf    0,r4
        dcbf    0,r4
        dcbf    0,r4
        dcbf    0,r4
        dcbf    0,r4
        dcbf    0,r4
        dcbf    0,r4

        /* Due to a bug with the HW flush on some CPU revs, we occasionally
         * experience data corruption. I'm adding a displacement flush along
         * with a dcbf loop over a few Mb to "help". The problem isn't totally
         * fixed by this in theory, but at least, in practice, I couldn't reproduce
         * it even with a big hammer...
         */

        lis     r4,0x0002
        mtctr   r4
        li      r4,0
1:
        lwz     r0,0(r4)
        addi    r4,r4,32                /* Go to start of next cache line */
        bdnz    1b
        isync

        /* Now, flush the first 4MB of memory */
        lis     r4,0x0002
        mtctr   r4
        li      r4,0
        sync
1:
        dcbf    0,r4
        addi    r4,r4,32                /* Go to start of next cache line */
        bdnz    1b

        /* Flush and disable the L1 data cache */
        mfspr   r6,SPRN_LDSTCR
        lis     r3,0xfff0       /* read from ROM for displacement flush */
        li      r4,0xfe         /* start with only way 0 unlocked */
        li      r5,128          /* 128 lines in each way */
1:      mtctr   r5
        rlwimi  r6,r4,0,24,31
        mtspr   SPRN_LDSTCR,r6
        sync
        isync
2:      lwz     r0,0(r3)        /* touch each cache line */
        addi    r3,r3,32
        bdnz    2b
        rlwinm  r4,r4,1,24,30   /* move on to the next way */
        ori     r4,r4,1
        cmpwi   r4,0xff         /* all done? */
        bne     1b
        /* now unlock the L1 data cache */
        li      r4,0
        rlwimi  r6,r4,0,24,31
        sync
        mtspr   SPRN_LDSTCR,r6
        sync
        isync

        /* Flush the L2 cache using the hardware assist */
        mfspr   r3,SPRN_L2CR
        cmpwi   r3,0            /* check if it is enabled first */
        bge     4f
        oris    r0,r3,(L2CR_L2IO_745x|L2CR_L2DO_745x)@h
        b       2f
        /* When disabling/locking L2, code must be in L1 */
        .balign 32
1:      mtspr   SPRN_L2CR,r0    /* lock the L2 cache */
3:      sync
        isync
        b       1f
2:      b       3f
3:      sync
        isync
        b       1b
1:      sync
        isync
        ori     r0,r3,L2CR_L2HWF_745x
        sync
        mtspr   SPRN_L2CR,r0    /* set the hardware flush bit */
3:      mfspr   r0,SPRN_L2CR    /* wait for it to go to 0 */
        andi.   r0,r0,L2CR_L2HWF_745x
        bne     3b
        sync
        rlwinm  r3,r3,0,~L2CR_L2E
        b       2f
        /* When disabling L2, code must be in L1 */
        .balign 32
1:      mtspr   SPRN_L2CR,r3    /* disable the L2 cache */
3:      sync
        isync
        b       1f
2:      b       3f
3:      sync
        isync
        b       1b
1:      sync
        isync
        oris    r4,r3,L2CR_L2I@h
        mtspr   SPRN_L2CR,r4
        sync
        isync
1:      mfspr   r4,SPRN_L2CR
        andis.  r0,r4,L2CR_L2I@h
        bne     1b
        sync

BEGIN_FTR_SECTION
        /* Flush the L3 cache using the hardware assist */
4:      mfspr   r3,SPRN_L3CR
        cmpwi   r3,0            /* check if it is enabled */
        bge     6f
        oris    r0,r3,L3CR_L3IO@h
        ori     r0,r0,L3CR_L3DO
        sync
        mtspr   SPRN_L3CR,r0    /* lock the L3 cache */
        sync
        isync
        ori     r0,r0,L3CR_L3HWF
        sync
        mtspr   SPRN_L3CR,r0    /* set the hardware flush bit */
5:      mfspr   r0,SPRN_L3CR    /* wait for it to go to zero */
        andi.   r0,r0,L3CR_L3HWF
        bne     5b
        rlwinm  r3,r3,0,~L3CR_L3E
        sync
        mtspr   SPRN_L3CR,r3    /* disable the L3 cache */
        sync
        ori     r4,r3,L3CR_L3I
        mtspr   SPRN_L3CR,r4
1:      mfspr   r4,SPRN_L3CR
        andi.   r0,r4,L3CR_L3I
        bne     1b
        sync
END_FTR_SECTION_IFSET(CPU_FTR_L3CR)

6:      mfspr   r0,SPRN_HID0    /* now disable the L1 data cache */
        rlwinm  r0,r0,0,~HID0_DCE
        mtspr   SPRN_HID0,r0
        sync
        isync
        mtmsr   r11             /* restore DR and EE */
        isync
        blr
_ASM_NOKPROBE_SYMBOL(flush_disable_745x)
#endif  /* CONFIG_PPC_BOOK3S_32 */