/*
        Copyright (c) 2002, Thomas Kurschel


        Part of Radeon accelerant

        CP initialization/sync/cleanup.

        It also handles command buffer synchronization.

        non-local memory is used as following:
        - 2048 dwords for ring buffer
        - 253 indirect buffers a 4k (1024 dwords)
        - 8 dwords for returned data (i.e. current read ptr)
          & 6 dwords for "scratch registers"

        usage of scratch registers:
        - reg 0 = reached engine.count

        with a granularity of 4 KByte, we need 2+253+1=256 blocks, which is exactly 1 MB
*/

#include "radeon_driver.h"
#include "CPMicroCode.h"
#include "mmio.h"
#include "cp_regs.h"
#include "pll_regs.h"
#include "rbbm_regs.h"
#include "buscntrl_regs.h"
#include "config_regs.h"
#include "memcntrl_regs.h"
#include "utils.h"
#include "pll_access.h"

#include "log_coll.h"
#include "log_enum.h"

#include <string.h>

#if 0

// macros for user-space

#define ALLOC_MEM( asize, mem_type, aglobal, handle, offset ) \
        { \
                radeon_alloc_mem am; \
\
                am.magic = RADEON_PRIVATE_DATA_MAGIC; \
                am.size = (asize) * 4; \
                am.memory_type = (mt_nonlocal); \
                am.global = (aglobal); \
\
                res = ioctl( ai->fd, RADEON_ALLOC_MEM, &am ); \
                if( res == B_OK ) \
                        *(handle) = am.handle; \
                        *(offset) = am.offset; \
        }

#define MEM2CPU( mem ) \
        ((uint32 *)(ai->mapped_memory[(mem).memory_type].data + (mem).offset))

#define MEM2GC( mem ) ((mem).offset + si->memory[(mem).memory_type].virtual_addr_start)

#define FREE_MEM( mem_type, handle ) \
        { \
                radeon_free_mem fm; \
\
                fm.magic = RADEON_PRIVATE_DATA_MAGIC; \
                fm.memory_type = mem_type; \
                fm.handle = offset; \
\
                ioctl( ai->fd, RADEON_FREE_MEM, &fm ); \
        }

#else

// macros for kernel-space

// allocate memory
// if memory_type is non-local, it is replaced with default non-local type
#define ALLOC_MEM( asize, mem_type, aglobal, handle, offset ) \
        if( mem_type == mt_nonlocal ) \
                mem_type = di->si->nonlocal_type; \
        res = mem_alloc( di->memmgr[mem_type], asize, NULL, handle, offset );

// get address as seen by program to access allocated memory
// (memory_type must _not_ be non-local, see ALLOC_MEM)
#define MEM2CPU( memory_type, offset ) \
        ((uint8 *)(memory_type == mt_local ? di->si->local_mem : \
        (memory_type == mt_PCI ? di->pci_gart.buffer.ptr : di->agp_gart.buffer.ptr)) \
        + (offset))

// get graphics card's virtual address of allocated memory
// (memory_type must _not_ be non-local, see ALLOC_MEM)
#define MEM2GC( memory_type, offset ) \
        (di->si->memory[(memory_type)].virtual_addr_start + (offset))

// free memory
// if memory_type is non-local, it is replaced with default non-local type
#define FREE_MEM( mem_type, handle ) \
        mem_free( \
                di->memmgr[ mem_type == mt_nonlocal ? di->si->nonlocal_type : mem_type], \
                handle, NULL );

#endif


void Radeon_DiscardAllIndirectBuffers( device_info *di );

#define RADEON_SCRATCH_REG_OFFSET       32


void Radeon_FlushPixelCache( device_info *di );

// wait until engine is idle;
// acquire_lock -       true, if lock must be hold
//                                      false, if lock is already acquired
// keep_lock -          true, keep lock on exit (only valid if acquire_lock is true)
void Radeon_WaitForIdle( device_info *di, bool acquire_lock, bool keep_lock )
{
        if( acquire_lock )
                ACQUIRE_BEN( di->si->cp.lock );

        Radeon_WaitForFifo( di, 64 );

        while( 1 ) {
                bigtime_t start_time = system_time();

                do {
                        if( (INREG( di->regs, RADEON_RBBM_STATUS ) & RADEON_RBBM_ACTIVE) == 0 ) {
                                Radeon_FlushPixelCache( di );

                                if( acquire_lock && !keep_lock)
                                        RELEASE_BEN( di->si->cp.lock );

                                return;
                        }

                        snooze( 1 );
                } while( system_time() - start_time < 1000000 );

                SHOW_ERROR( 3,
                        "Engine didn't become idle (rbbm_status=%" B_PRIx32 ", "
                        "cp_stat=%" B_PRIx32 ", "
                        "tlb_address=%" B_PRIx32 ", "
                        "tlb_data=%" B_PRIx32 ")",
                        INREG( di->regs, RADEON_RBBM_STATUS ),
                        INREG( di->regs, RADEON_CP_STAT ),
                        INREG( di->regs, RADEON_AIC_TLB_ADDR ),
                        INREG( di->regs, RADEON_AIC_TLB_DATA ));

                LOG( di->si->log, _Radeon_WaitForIdle );

                Radeon_ResetEngine( di );
        }
}


// wait until "entries" FIFO entries are empty
// lock must be hold
void Radeon_WaitForFifo( device_info *di, int entries )
{
        while( 1 ) {
                bigtime_t start_time = system_time();

                do {
                        int slots = INREG( di->regs, RADEON_RBBM_STATUS ) & RADEON_RBBM_FIFOCNT_MASK;

                        if ( slots >= entries )
                                return;

                        snooze( 1 );
                } while( system_time() - start_time < 1000000 );

                LOG( di->si->log, _Radeon_WaitForFifo );

                Radeon_ResetEngine( di );
        }
}

// flush pixel cache of graphics card
void Radeon_FlushPixelCache( device_info *di )
{
        bigtime_t start_time;

        OUTREGP( di->regs, RADEON_RB2D_DSTCACHE_CTLSTAT, RADEON_RB2D_DC_FLUSH_ALL,
                ~RADEON_RB2D_DC_FLUSH_ALL );

        start_time = system_time();

        do {
                if( (INREG( di->regs, RADEON_RB2D_DSTCACHE_CTLSTAT )
                         & RADEON_RB2D_DC_BUSY) == 0 )
                        return;

                snooze( 1 );
        } while( system_time() - start_time < 1000000 );

        LOG( di->si->log, _Radeon_FlushPixelCache );

        SHOW_ERROR0( 0, "pixel cache didn't become empty" );
}

// reset graphics card's engine
// lock must be hold
void Radeon_ResetEngine( device_info *di )
{
        vuint8 *regs = di->regs;
        shared_info *si = di->si;
        uint32 clock_cntl_index, mclk_cntl, rbbm_soft_reset, host_path_cntl;
        uint32 cur_read_ptr;

        SHOW_FLOW0( 3, "" );

        Radeon_FlushPixelCache( di );

        clock_cntl_index = INREG( regs, RADEON_CLOCK_CNTL_INDEX );
        RADEONPllErrataAfterIndex( regs, di->asic );    // drm has no errata here!
        mclk_cntl = Radeon_INPLL( regs, di->asic, RADEON_MCLK_CNTL );

        // enable clock of units to be reset
        Radeon_OUTPLL( regs, di->asic, RADEON_MCLK_CNTL, mclk_cntl |
      RADEON_FORCEON_MCLKA |
      RADEON_FORCEON_MCLKB |
      RADEON_FORCEON_YCLKA |
      RADEON_FORCEON_YCLKB |
      RADEON_FORCEON_MC |
      RADEON_FORCEON_AIC );

        // do the reset
    host_path_cntl = INREG( regs, RADEON_HOST_PATH_CNTL );
        rbbm_soft_reset = INREG( regs, RADEON_RBBM_SOFT_RESET );

        OUTREG( regs, RADEON_RBBM_SOFT_RESET, (rbbm_soft_reset |
                        RADEON_SOFT_RESET_CP |
                        RADEON_SOFT_RESET_HI |
                        RADEON_SOFT_RESET_SE |
                        RADEON_SOFT_RESET_RE |
                        RADEON_SOFT_RESET_PP |
                        RADEON_SOFT_RESET_E2 |
                        RADEON_SOFT_RESET_RB ) );
        INREG( regs, RADEON_RBBM_SOFT_RESET);
        OUTREG( regs, RADEON_RBBM_SOFT_RESET, rbbm_soft_reset &
                ~( RADEON_SOFT_RESET_CP |
                   RADEON_SOFT_RESET_HI |
                   RADEON_SOFT_RESET_SE |
                   RADEON_SOFT_RESET_RE |
                   RADEON_SOFT_RESET_PP |
                   RADEON_SOFT_RESET_E2 |
                   RADEON_SOFT_RESET_RB ) );
        INREG( regs, RADEON_RBBM_SOFT_RESET);

    OUTREG( regs, RADEON_HOST_PATH_CNTL, host_path_cntl | RADEON_HDP_SOFT_RESET );
    INREG( regs, RADEON_HOST_PATH_CNTL );
    OUTREG( regs, RADEON_HOST_PATH_CNTL, host_path_cntl );

        Radeon_OUTPLL( regs, di->asic, RADEON_MCLK_CNTL, mclk_cntl );
        OUTREG( regs, RADEON_CLOCK_CNTL_INDEX, clock_cntl_index );
        //RADEONPllErrataAfterIndex( regs, di->asic ); // drm doesn't do this here!
        OUTREG( regs, RADEON_RBBM_SOFT_RESET, rbbm_soft_reset);

        if ( di->acc_dma )
        {
                // reset ring buffer
                cur_read_ptr = INREG( regs, RADEON_CP_RB_RPTR );
                OUTREG( regs, RADEON_CP_RB_WPTR, cur_read_ptr );

                //if( si->cp.ring.head ) {
                // during init, there are no feedback data
                if( si->cp.feedback.mem_handle != 0 ) {
                        *(uint32 *)MEM2CPU( si->cp.feedback.mem_type, si->cp.feedback.head_mem_offset) =
                                cur_read_ptr;
                        //      *si->cp.ring.head = cur_read_ptr;
                        si->cp.ring.tail = cur_read_ptr;
                }

                // mark all buffers as being finished
                Radeon_DiscardAllIndirectBuffers( di );
        }

        ++si->engine.count;
        return;
}


// upload Micro-Code of CP
static void loadMicroEngineRAMData( device_info *di )
{
        int i;
        const uint32 (*microcode)[2];

        SHOW_FLOW0( 3, "" );

        switch( di->asic ) {
        case rt_r300:
        case rt_rv350:
        case rt_r350:
        case rt_rv380:
        case rt_r420:
                microcode = r300_cp_microcode;
                break;
        case rt_r200:
                microcode = r200_cp_microcode;
                break;
        case rt_rs100:
        default:
                microcode = radeon_cp_microcode;
        }

        Radeon_WaitForIdle( di, false, false );

        OUTREG( di->regs, RADEON_CP_ME_RAM_ADDR, 0 );

        for ( i = 0 ; i < 256 ; i++ ) {
                OUTREG( di->regs, RADEON_CP_ME_RAM_DATAH, microcode[i][1] );
                OUTREG( di->regs, RADEON_CP_ME_RAM_DATAL, microcode[i][0] );
        }
}

// aring_size - size of ring in dwords
static status_t initRingBuffer( device_info *di, int aring_size )
{
        status_t res;
        shared_info *si = di->si;
        CP_info *cp = &si->cp;
        vuint8 *regs = di->regs;
        uint32 offset;
        memory_type_e memory_type;

        memset( &cp->ring, 0, sizeof( cp->ring ));

        // ring and indirect buffers can be either in AGP or PCI GART
        // (it seems that they cannot be in graphics memory, at least
        //  I had serious coherency problems when I tried that)
        memory_type = mt_nonlocal;

        ALLOC_MEM( aring_size * 4, memory_type, true,
                &cp->ring.mem_handle, &offset );

        if( res != B_OK ) {
                SHOW_ERROR0( 0, "Cannot allocate ring buffer" );
                return res;
        }

        // setup CP buffer
        cp->ring.mem_type = memory_type;
        cp->ring.mem_offset = offset;
        cp->ring.vm_base = MEM2GC( memory_type, offset );
        cp->ring.size = aring_size;
        cp->ring.tail_mask = aring_size - 1;
        OUTREG( regs, RADEON_CP_RB_BASE, cp->ring.vm_base );
        SHOW_INFO( 3, "CP buffer address=%" B_PRIx32, cp->ring.vm_base );

        // set ring buffer size
        // (it's log2 of qwords)
        OUTREG( regs, RADEON_CP_RB_CNTL, radeon_log2( cp->ring.size / 2 ));
        SHOW_INFO( 3, "CP buffer size mask=%d", radeon_log2( cp->ring.size / 2 ) );

        // set write pointer delay to zero;
        // we assume that memory synchronization is done correctly my MoBo
        // and Radeon_SendCP contains a hack that hopefully fixes such problems
        OUTREG( regs, RADEON_CP_RB_WPTR_DELAY, 0 );

        memset( MEM2CPU( cp->ring.mem_type, cp->ring.mem_offset), 0, cp->ring.size * 4 );

        // set CP buffer pointers
        OUTREG( regs, RADEON_CP_RB_RPTR, 0 );
        OUTREG( regs, RADEON_CP_RB_WPTR, 0 );
        //*cp->ring.head = 0;
        cp->ring.tail = 0;

        return B_OK;
}

static void uninitRingBuffer( device_info *di )
{
        vuint8 *regs = di->regs;

        // abort any activity
        Radeon_ResetEngine( di );

        // disable CP BM
        OUTREG( regs, RADEON_CP_CSQ_CNTL, RADEON_CSQ_PRIDIS_INDDIS );
        // read-back for flushing
        INREG( regs, RADEON_CP_CSQ_CNTL );

        FREE_MEM( mt_nonlocal, di->si->cp.ring.mem_handle );
}

static status_t initCPFeedback( device_info *di )
{
        CP_info *cp = &di->si->cp;
        vuint8 *regs = di->regs;
        uint32 offset;
        memory_type_e memory_type;
        status_t res;

        // status information should be in PCI memory, so CPU can
        // poll it without locking the bus (PCI memory is the only
        // cachable memory available)
        memory_type = mt_PCI;

        ALLOC_MEM( RADEON_SCRATCH_REG_OFFSET + 0x40, memory_type, true,
                &cp->feedback.mem_handle, &offset );

        if( res != B_OK ) {
                SHOW_ERROR0( 0, "Cannot allocate buffers for status information" );
                return res;
        }

        // setup CP read pointer buffer
        cp->feedback.mem_type = memory_type;
        cp->feedback.head_mem_offset = offset;
        cp->feedback.head_vm_address = MEM2GC( memory_type, cp->feedback.head_mem_offset );
        OUTREG( regs, RADEON_CP_RB_RPTR_ADDR, cp->feedback.head_vm_address );
        SHOW_INFO( 3, "CP read pointer buffer==%" B_PRIx32,
                cp->feedback.head_vm_address );

        // setup scratch register buffer
        cp->feedback.scratch_mem_offset = offset + RADEON_SCRATCH_REG_OFFSET;
        cp->feedback.scratch_vm_start = MEM2GC( memory_type, cp->feedback.scratch_mem_offset );
        OUTREG( regs, RADEON_SCRATCH_ADDR, cp->feedback.scratch_vm_start );
        OUTREG( regs, RADEON_SCRATCH_UMSK, 0x3f );

        *(uint32 *)MEM2CPU( cp->feedback.mem_type, cp->feedback.head_mem_offset) = 0;
        memset( MEM2CPU( cp->feedback.mem_type, cp->feedback.scratch_mem_offset), 0, 0x40 );
        //*cp->ring.head = 0;

        return B_OK;
}

static void uninitCPFeedback( device_info *di )
{
        vuint8 *regs = di->regs;

        // don't allow any scratch buffer update
        OUTREG( regs, RADEON_SCRATCH_UMSK, 0x0 );

        FREE_MEM( mt_PCI, di->si->cp.feedback.mem_handle );
}

static status_t initIndirectBuffers( device_info *di )
{
        CP_info *cp = &di->si->cp;
        uint32 offset;
        memory_type_e memory_type;
        int i;
        status_t res;

        memory_type = mt_nonlocal;

        ALLOC_MEM( NUM_INDIRECT_BUFFERS * INDIRECT_BUFFER_SIZE * 4, memory_type,
                true, &cp->buffers.mem_handle, &offset );

        if( res != B_OK ) {
                SHOW_ERROR0( 0, "Cannot allocate indirect buffers" );
                return B_ERROR;
        }

        cp->buffers.mem_type = memory_type;
        cp->buffers.mem_offset = offset;
        cp->buffers.vm_start = MEM2GC( memory_type, cp->buffers.mem_offset );

        for( i = 0; i < NUM_INDIRECT_BUFFERS - 1; ++i ) {
                cp->buffers.buffers[i].next = i + 1;
        }

        cp->buffers.buffers[i].next = -1;

        cp->buffers.free_list = 0;
        cp->buffers.oldest = -1;
        cp->buffers.newest = -1;
        cp->buffers.active_state = -1;
        cp->buffers.cur_tag = 0;

        memset( MEM2CPU( cp->buffers.mem_type, cp->buffers.mem_offset), 0,
                NUM_INDIRECT_BUFFERS * INDIRECT_BUFFER_SIZE * 4 );

        return B_OK;
}

static void uninitIndirectBuffers( device_info *di )
{
        FREE_MEM( mt_nonlocal, di->si->cp.buffers.mem_handle );
}

// initialize CP so it's ready for BM
status_t Radeon_InitCP( device_info *di )
{
        thread_id thid;
    thread_info thinfo;
        status_t res;

        SHOW_FLOW0( 3, "" );

        // this is _really_ necessary so functions like ResetEngine() know
        // that the CP is not set up yet
        memset( &di->si->cp, 0, sizeof( di->si->cp ));

        if( (res = INIT_BEN( di->si->cp.lock, "Radeon CP" )) < 0 )
                return res;

        // HACK: change owner of benaphore semaphore to team of calling thread;
        // reason: user code cannot acquire kernel semaphores, but the accelerant
        // is in user space; interestingly, it's enough to change the semaphore's
        // owner to _any_ non-system team (that's the only security check done by
        // the kernel)
        thid = find_thread( NULL );
    get_thread_info( thid, &thinfo );
    set_sem_owner( di->si->cp.lock.sem, thinfo.team );

        // init raw CP
        if ( di->acc_dma ) loadMicroEngineRAMData( di );

        // do soft-reset
        Radeon_ResetEngine( di );

        // after warm-reset, the CP may still be active and thus react to
        // register writes during initialization unpredictably, so we better
        // stop it first
        OUTREG( di->regs, RADEON_CP_CSQ_CNTL, RADEON_CSQ_PRIDIS_INDDIS );
        INREG( di->regs, RADEON_CP_CSQ_CNTL );

        // reset CP to make disabling active
        Radeon_ResetEngine( di );

        if ( di->acc_dma )
        {
                res = initRingBuffer( di, CP_RING_SIZE );
                if( res < 0 )
                        goto err4;

                res = initCPFeedback( di );
                if( res < 0 )
                        goto err3;

                res = initIndirectBuffers( di );
                if( res < 0 )
                        goto err2;

                // tell CP to use BM
                Radeon_WaitForIdle( di, false, false );

                // enable direct and indirect CP bus mastering
                OUTREG( di->regs, RADEON_CP_CSQ_CNTL, RADEON_CSQ_PRIBM_INDBM );

                // allow bus mastering in general
                OUTREGP( di->regs, RADEON_BUS_CNTL, 0, ~RADEON_BUS_MASTER_DIS );
        }


        // don't allow mixing of 2D/3D/scratch/wait_until commands
        // (in fact, this doesn't seem to make any difference as we do a
        // manual sync in all these cases anyway)
        OUTREG( di->regs, RADEON_ISYNC_CNTL,
                RADEON_ISYNC_ANY2D_IDLE3D |
                RADEON_ISYNC_ANY3D_IDLE2D |
                RADEON_ISYNC_WAIT_IDLEGUI |
                RADEON_ISYNC_CPSCRATCH_IDLEGUI );

        SHOW_FLOW( 3, "bus_cntl=%" B_PRIx32, INREG( di->regs, RADEON_BUS_CNTL ));

        SHOW_FLOW0( 3, "Done" );

        return B_OK;

//err:
//      uninitIndirectBuffers( ai );
err2:
        uninitCPFeedback( di );
err3:
        uninitRingBuffer( di );
err4:
        DELETE_BEN( di->si->cp.lock );
        return res;
}


// shutdown CP, freeing any memory
void Radeon_UninitCP( device_info *di )
{
        vuint8 *regs = di->regs;

        // abort any pending commands
        Radeon_ResetEngine( di );

        // disable CP BM
        OUTREG( regs, RADEON_CP_CSQ_CNTL, RADEON_CSQ_PRIDIS_INDDIS );
        // read-back for flushing
        INREG( regs, RADEON_CP_CSQ_CNTL );

        if ( di->acc_dma )
        {
                uninitRingBuffer( di );
                uninitCPFeedback( di );
                uninitIndirectBuffers( di );
        }

        DELETE_BEN( di->si->cp.lock );
}


// mark all indirect buffers as being free;
// this should only be called after a reset;
// lock must be hold
void Radeon_DiscardAllIndirectBuffers( device_info *di )
{
        CP_info *cp = &di->si->cp;

        // during init, there is no indirect buffer
        if( cp->buffers.mem_handle == 0 )
                return;

        // mark all sent indirect buffers as free
        while( cp->buffers.oldest != -1 ) {
                indirect_buffer *oldest_buffer =
                        &cp->buffers.buffers[cp->buffers.oldest];
                int tmp_oldest_buffer;

                SHOW_FLOW( 0, "%d", cp->buffers.oldest );

                // remove buffer from "used" list
                tmp_oldest_buffer = oldest_buffer->next;

                if( tmp_oldest_buffer == -1 )
                        cp->buffers.newest = -1;

                // put it on free list
                oldest_buffer->next = cp->buffers.free_list;
                cp->buffers.free_list = cp->buffers.oldest;

                cp->buffers.oldest = tmp_oldest_buffer;
        }
}

// lets hide this in here, as it's got lots of lovely register headers already...
// does it go here, or in the accelerant anyway?
// for now i'm assuming you turn on dynamic clocks, and they take care of themselves onwards...
// so doing it at driver init seems sensible after a valid detection of course...
void Radeon_SetDynamicClock( device_info *di, int mode)
{
    vuint8 *regs = di->regs;
    radeon_type asic = di->asic;
    uint32 tmp;

    switch(mode) {
        case 0: /* Turn everything OFF (ForceON to everything)*/
                if ( di->num_crtc != 2 ) {
                        tmp = Radeon_INPLL(regs, asic, RADEON_SCLK_CNTL);
                        tmp |= (RADEON_SCLK_FORCE_CP   | RADEON_SCLK_FORCE_HDP |
                                RADEON_SCLK_FORCE_DISP1 | RADEON_SCLK_FORCE_TOP |
                                RADEON_SCLK_FORCE_E2   | RADEON_SCLK_FORCE_SE  |
                                RADEON_SCLK_FORCE_IDCT | RADEON_SCLK_FORCE_VIP |
                                RADEON_SCLK_FORCE_RE   | RADEON_SCLK_FORCE_PB  |
                                RADEON_SCLK_FORCE_TAM  | RADEON_SCLK_FORCE_TDM |
                                RADEON_SCLK_FORCE_RB);
                        Radeon_OUTPLL(regs, asic, RADEON_SCLK_CNTL, tmp);
                } else if (asic == rt_rv350) {
                        /* for RV350/M10, no delays are required. */
                        tmp = Radeon_INPLL(regs, asic, R300_SCLK_CNTL2);
                        tmp |= (R300_SCLK_FORCE_TCL |
                                R300_SCLK_FORCE_GA  |
                                R300_SCLK_FORCE_CBA);
                        Radeon_OUTPLL(regs, asic, R300_SCLK_CNTL2, tmp);

                        tmp = Radeon_INPLL(regs, asic, RADEON_SCLK_CNTL);
                        tmp |= (RADEON_SCLK_FORCE_DISP2 | RADEON_SCLK_FORCE_CP      |
                                RADEON_SCLK_FORCE_HDP   | RADEON_SCLK_FORCE_DISP1   |
                                RADEON_SCLK_FORCE_TOP   | RADEON_SCLK_FORCE_E2      |
                                R300_SCLK_FORCE_VAP     | RADEON_SCLK_FORCE_IDCT    |
                                RADEON_SCLK_FORCE_VIP   | R300_SCLK_FORCE_SR        |
                                R300_SCLK_FORCE_PX      | R300_SCLK_FORCE_TX        |
                                R300_SCLK_FORCE_US      | RADEON_SCLK_FORCE_TV_SCLK |
                                R300_SCLK_FORCE_SU      | RADEON_SCLK_FORCE_OV0);
                        Radeon_OUTPLL(regs, asic, RADEON_SCLK_CNTL, tmp);

                        tmp = Radeon_INPLL(regs, asic, RADEON_SCLK_MORE_CNTL);
                        tmp |= RADEON_SCLK_MORE_FORCEON;
                        Radeon_OUTPLL(regs, asic, RADEON_SCLK_MORE_CNTL, tmp);

                        tmp = Radeon_INPLL(regs, asic, RADEON_MCLK_CNTL);
                        tmp |= (RADEON_FORCEON_MCLKA |
                                RADEON_FORCEON_MCLKB |
                                RADEON_FORCEON_YCLKA |
                                RADEON_FORCEON_YCLKB |
                                RADEON_FORCEON_MC);
                        Radeon_OUTPLL(regs, asic, RADEON_MCLK_CNTL, tmp);

                        tmp = Radeon_INPLL(regs, asic, RADEON_VCLK_ECP_CNTL);
                        tmp &= ~(RADEON_PIXCLK_ALWAYS_ONb  |
                                RADEON_PIXCLK_DAC_ALWAYS_ONb |
                        R300_DISP_DAC_PIXCLK_DAC_BLANK_OFF);
                        Radeon_OUTPLL(regs, asic, RADEON_VCLK_ECP_CNTL, tmp);

                        tmp = Radeon_INPLL(regs, asic, RADEON_PIXCLKS_CNTL);
                        tmp &= ~(RADEON_PIX2CLK_ALWAYS_ONb         |
                                RADEON_PIX2CLK_DAC_ALWAYS_ONb     |
                                RADEON_DISP_TVOUT_PIXCLK_TV_ALWAYS_ONb |
                                R300_DVOCLK_ALWAYS_ONb            |
                                RADEON_PIXCLK_BLEND_ALWAYS_ONb    |
                                RADEON_PIXCLK_GV_ALWAYS_ONb       |
                                R300_PIXCLK_DVO_ALWAYS_ONb        |
                                RADEON_PIXCLK_LVDS_ALWAYS_ONb     |
                                RADEON_PIXCLK_TMDS_ALWAYS_ONb     |
                                R300_PIXCLK_TRANS_ALWAYS_ONb      |
                                R300_PIXCLK_TVO_ALWAYS_ONb        |
                                R300_P2G2CLK_ALWAYS_ONb            |
                                R300_P2G2CLK_DAC_ALWAYS_ONb           |
                                R300_DISP_DAC_PIXCLK_DAC2_BLANK_OFF);
                        Radeon_OUTPLL(regs, asic, RADEON_PIXCLKS_CNTL, tmp);
                }  else {
                        tmp = Radeon_INPLL(regs, asic, RADEON_SCLK_CNTL);
                        tmp |= (RADEON_SCLK_FORCE_CP | RADEON_SCLK_FORCE_E2);
                        tmp |= RADEON_SCLK_FORCE_SE;

                        if ( di->num_crtc != 2 ) {
                                tmp |= ( RADEON_SCLK_FORCE_RB    |
                                RADEON_SCLK_FORCE_TDM   |
                                RADEON_SCLK_FORCE_TAM   |
                                RADEON_SCLK_FORCE_PB    |
                                RADEON_SCLK_FORCE_RE    |
                                RADEON_SCLK_FORCE_VIP   |
                                RADEON_SCLK_FORCE_IDCT  |
                                RADEON_SCLK_FORCE_TOP   |
                                RADEON_SCLK_FORCE_DISP1 |
                                RADEON_SCLK_FORCE_DISP2 |
                                RADEON_SCLK_FORCE_HDP    );
                        } else if ((asic == rt_r300) || (asic == rt_r350)) {
                                tmp |= ( RADEON_SCLK_FORCE_HDP   |
                                        RADEON_SCLK_FORCE_DISP1 |
                                        RADEON_SCLK_FORCE_DISP2 |
                                        RADEON_SCLK_FORCE_TOP   |
                                        RADEON_SCLK_FORCE_IDCT  |
                                        RADEON_SCLK_FORCE_VIP);
                        }
                        Radeon_OUTPLL(regs, asic, RADEON_SCLK_CNTL, tmp);

                        snooze(16000);

                        if ((asic == rt_r300) || (asic == rt_r350)) {
                                tmp = Radeon_INPLL(regs, asic, R300_SCLK_CNTL2);
                                tmp |= ( R300_SCLK_FORCE_TCL |
                                        R300_SCLK_FORCE_GA  |
                                        R300_SCLK_FORCE_CBA);
                                Radeon_OUTPLL(regs, asic, R300_SCLK_CNTL2, tmp);
                                snooze(16000);
                        }

                        if (di->is_igp) {
                                tmp = Radeon_INPLL(regs, asic, RADEON_MCLK_CNTL);
                                tmp &= ~(RADEON_FORCEON_MCLKA |
                                        RADEON_FORCEON_YCLKA);
                                Radeon_OUTPLL(regs, asic, RADEON_MCLK_CNTL, tmp);
                                snooze(16000);
                        }

                        if ((asic == rt_rv200) ||
                                (asic == rt_rv250) ||
                                (asic == rt_rv280)) {
                                tmp = Radeon_INPLL(regs, asic, RADEON_SCLK_MORE_CNTL);
                                tmp |= RADEON_SCLK_MORE_FORCEON;
                                Radeon_OUTPLL(regs, asic, RADEON_SCLK_MORE_CNTL, tmp);
                                snooze(16000);
                        }

                        tmp = Radeon_INPLL(regs, asic, RADEON_PIXCLKS_CNTL);
                        tmp &= ~(RADEON_PIX2CLK_ALWAYS_ONb         |
                                RADEON_PIX2CLK_DAC_ALWAYS_ONb     |
                                RADEON_PIXCLK_BLEND_ALWAYS_ONb    |
                                RADEON_PIXCLK_GV_ALWAYS_ONb       |
                                RADEON_PIXCLK_DIG_TMDS_ALWAYS_ONb |
                                RADEON_PIXCLK_LVDS_ALWAYS_ONb     |
                                RADEON_PIXCLK_TMDS_ALWAYS_ONb);

                        Radeon_OUTPLL(regs, asic, RADEON_PIXCLKS_CNTL, tmp);
                        snooze(16000);

                        tmp = Radeon_INPLL(regs, asic, RADEON_VCLK_ECP_CNTL);
                        tmp &= ~(RADEON_PIXCLK_ALWAYS_ONb  |
                                RADEON_PIXCLK_DAC_ALWAYS_ONb);
                        Radeon_OUTPLL(regs, asic, RADEON_VCLK_ECP_CNTL, tmp);
                }
                SHOW_FLOW0( 3, "Dynamic Clock Scaling Disabled" );
                break;
        case 1:
                if ( di->num_crtc != 2 ) {
                        tmp = Radeon_INPLL(regs, asic, RADEON_SCLK_CNTL);
                        if ((INREG( regs, RADEON_CONFIG_CNTL) & RADEON_CFG_ATI_REV_ID_MASK) > RADEON_CFG_ATI_REV_A13) {
                                tmp &= ~(RADEON_SCLK_FORCE_CP | RADEON_SCLK_FORCE_RB);
                        }
                        tmp &= ~(RADEON_SCLK_FORCE_HDP  | RADEON_SCLK_FORCE_DISP1 |
                                RADEON_SCLK_FORCE_TOP  | RADEON_SCLK_FORCE_SE   |
                                RADEON_SCLK_FORCE_IDCT | RADEON_SCLK_FORCE_RE   |
                                RADEON_SCLK_FORCE_PB   | RADEON_SCLK_FORCE_TAM  |
                                RADEON_SCLK_FORCE_TDM);
                        Radeon_OUTPLL(regs, asic, RADEON_SCLK_CNTL, tmp);
                } else if ((asic == rt_r300)
                                || (asic == rt_r350)
                                || (asic == rt_rv350)) {
                        if (asic == rt_rv350) {
                                tmp = Radeon_INPLL(regs, asic, R300_SCLK_CNTL2);
                                tmp &= ~(R300_SCLK_FORCE_TCL |
                                        R300_SCLK_FORCE_GA  |
                                        R300_SCLK_FORCE_CBA);
                                tmp |=  (R300_SCLK_TCL_MAX_DYN_STOP_LAT |
                                        R300_SCLK_GA_MAX_DYN_STOP_LAT  |
                                        R300_SCLK_CBA_MAX_DYN_STOP_LAT);
                                Radeon_OUTPLL(regs, asic, R300_SCLK_CNTL2, tmp);

                                tmp = Radeon_INPLL(regs, asic, RADEON_SCLK_CNTL);
                                tmp &= ~(RADEON_SCLK_FORCE_DISP2 | RADEON_SCLK_FORCE_CP      |
                                        RADEON_SCLK_FORCE_HDP   | RADEON_SCLK_FORCE_DISP1   |
                                        RADEON_SCLK_FORCE_TOP   | RADEON_SCLK_FORCE_E2      |
                                        R300_SCLK_FORCE_VAP     | RADEON_SCLK_FORCE_IDCT    |
                                        RADEON_SCLK_FORCE_VIP   | R300_SCLK_FORCE_SR        |
                                        R300_SCLK_FORCE_PX      | R300_SCLK_FORCE_TX        |
                                        R300_SCLK_FORCE_US      | RADEON_SCLK_FORCE_TV_SCLK |
                                        R300_SCLK_FORCE_SU      | RADEON_SCLK_FORCE_OV0);
                                        tmp |=  RADEON_DYN_STOP_LAT_MASK;
                                Radeon_OUTPLL(regs, asic, RADEON_SCLK_CNTL, tmp);

                                tmp = Radeon_INPLL(regs, asic, RADEON_SCLK_MORE_CNTL);
                                tmp &= ~RADEON_SCLK_MORE_FORCEON;
                                tmp |=  RADEON_SCLK_MORE_MAX_DYN_STOP_LAT;
                                Radeon_OUTPLL(regs, asic, RADEON_SCLK_MORE_CNTL, tmp);

                                tmp = Radeon_INPLL(regs, asic, RADEON_VCLK_ECP_CNTL);
                                tmp |= (RADEON_PIXCLK_ALWAYS_ONb |
                                        RADEON_PIXCLK_DAC_ALWAYS_ONb);
                                Radeon_OUTPLL(regs, asic, RADEON_VCLK_ECP_CNTL, tmp);

                                tmp = Radeon_INPLL(regs, asic, RADEON_PIXCLKS_CNTL);
                                tmp |= (RADEON_PIX2CLK_ALWAYS_ONb         |
                                        RADEON_PIX2CLK_DAC_ALWAYS_ONb     |
                                        RADEON_DISP_TVOUT_PIXCLK_TV_ALWAYS_ONb |
                                        R300_DVOCLK_ALWAYS_ONb            |
                                        RADEON_PIXCLK_BLEND_ALWAYS_ONb    |
                                        RADEON_PIXCLK_GV_ALWAYS_ONb       |
                                        R300_PIXCLK_DVO_ALWAYS_ONb        |
                                        RADEON_PIXCLK_LVDS_ALWAYS_ONb     |
                                        RADEON_PIXCLK_TMDS_ALWAYS_ONb     |
                                        R300_PIXCLK_TRANS_ALWAYS_ONb      |
                                        R300_PIXCLK_TVO_ALWAYS_ONb        |
                                        R300_P2G2CLK_ALWAYS_ONb           |
                                        R300_P2G2CLK_DAC_ALWAYS_ONb);
                                Radeon_OUTPLL(regs, asic, RADEON_PIXCLKS_CNTL, tmp);

                                tmp = Radeon_INPLL(regs, asic, RADEON_MCLK_MISC);
                                tmp |= (RADEON_MC_MCLK_DYN_ENABLE |
                                        RADEON_IO_MCLK_DYN_ENABLE);
                                Radeon_OUTPLL(regs, asic, RADEON_MCLK_MISC, tmp);

                                tmp = Radeon_INPLL(regs, asic, RADEON_MCLK_CNTL);
                                tmp |= (RADEON_FORCEON_MCLKA |
                                        RADEON_FORCEON_MCLKB);

                                tmp &= ~(RADEON_FORCEON_YCLKA  |
                                        RADEON_FORCEON_YCLKB  |
                                        RADEON_FORCEON_MC);

                                /* Some releases of vbios have set DISABLE_MC_MCLKA
                                and DISABLE_MC_MCLKB bits in the vbios table.  Setting these
                                bits will cause H/W hang when reading video memory with dynamic clocking
                                enabled. */
                                if ((tmp & R300_DISABLE_MC_MCLKA) &&
                                (tmp & R300_DISABLE_MC_MCLKB)) {
                                        /* If both bits are set, then check the active channels */
                                        tmp = Radeon_INPLL(regs, asic, RADEON_MCLK_CNTL);
                                        if (di->ram.width == 64) {
                                                if (INREG( regs, RADEON_MEM_CNTL) & R300_MEM_USE_CD_CH_ONLY)
                                                tmp &= ~R300_DISABLE_MC_MCLKB;
                                                else
                                                tmp &= ~R300_DISABLE_MC_MCLKA;
                                        } else {
                                                tmp &= ~(R300_DISABLE_MC_MCLKA |
                                                R300_DISABLE_MC_MCLKB);
                                        }
                                }

                                Radeon_OUTPLL(regs, asic, RADEON_MCLK_CNTL, tmp);
                        } else {
                                tmp = Radeon_INPLL(regs, asic, RADEON_SCLK_CNTL);
                                tmp &= ~(R300_SCLK_FORCE_VAP);
                                tmp |= RADEON_SCLK_FORCE_CP;
                                Radeon_OUTPLL(regs, asic, RADEON_SCLK_CNTL, tmp);
                                snooze(15000);

                                tmp = Radeon_INPLL(regs, asic, R300_SCLK_CNTL2);
                                tmp &= ~(R300_SCLK_FORCE_TCL |
                                R300_SCLK_FORCE_GA  |
                                R300_SCLK_FORCE_CBA);
                                Radeon_OUTPLL(regs, asic, R300_SCLK_CNTL2, tmp);
                        }
                } else {
                        tmp = Radeon_INPLL(regs, asic, RADEON_CLK_PWRMGT_CNTL);

                        tmp &= ~(RADEON_ACTIVE_HILO_LAT_MASK     |
                                RADEON_DISP_DYN_STOP_LAT_MASK   |
                                RADEON_DYN_STOP_MODE_MASK);

                        tmp |= (RADEON_ENGIN_DYNCLK_MODE |
                        (0x01 << RADEON_ACTIVE_HILO_LAT_SHIFT));
                        Radeon_OUTPLL(regs, asic, RADEON_CLK_PWRMGT_CNTL, tmp);
                        snooze(15000);

                        tmp = Radeon_INPLL(regs, asic, RADEON_CLK_PIN_CNTL);
                        tmp |= RADEON_SCLK_DYN_START_CNTL;
                        Radeon_OUTPLL(regs, asic, RADEON_CLK_PIN_CNTL, tmp);
                        snooze(15000);

                        /* When DRI is enabled, setting DYN_STOP_LAT to zero can cause some R200
                        to lockup randomly, leave them as set by BIOS.
                        */
                        tmp = Radeon_INPLL(regs, asic, RADEON_SCLK_CNTL);
                        /*tmp &= RADEON_SCLK_SRC_SEL_MASK;*/
                        tmp &= ~RADEON_SCLK_FORCEON_MASK;

                        /*RAGE_6::A11 A12 A12N1 A13, RV250::A11 A12, R300*/
                        if (((asic == rt_rv250) &&
                                ((INREG( regs, RADEON_CONFIG_CNTL) & RADEON_CFG_ATI_REV_ID_MASK) <
                                  RADEON_CFG_ATI_REV_A13)) ||
                                ((asic == rt_rv100) &&
                                ((INREG( regs, RADEON_CONFIG_CNTL) & RADEON_CFG_ATI_REV_ID_MASK) <=
                                  RADEON_CFG_ATI_REV_A13)))
                        {
                                tmp |= RADEON_SCLK_FORCE_CP;
                                tmp |= RADEON_SCLK_FORCE_VIP;
                        }

                        Radeon_OUTPLL(regs, asic, RADEON_SCLK_CNTL, tmp);

                        if ((asic == rt_rv200) ||
                                (asic == rt_rv250) ||
                                (asic == rt_rv280)) {
                                tmp = Radeon_INPLL(regs, asic, RADEON_SCLK_MORE_CNTL);
                                tmp &= ~RADEON_SCLK_MORE_FORCEON;

                                /* RV200::A11 A12 RV250::A11 A12 */
                                if (((asic == rt_rv200) ||
                                         (asic == rt_rv250)) &&
                                        ((INREG( regs, RADEON_CONFIG_CNTL) & RADEON_CFG_ATI_REV_ID_MASK) <
                                          RADEON_CFG_ATI_REV_A13))
                                {
                                        tmp |= RADEON_SCLK_MORE_FORCEON;
                                }
                                Radeon_OUTPLL(regs, asic, RADEON_SCLK_MORE_CNTL, tmp);
                                snooze(15000);
                        }

                        /* RV200::A11 A12, RV250::A11 A12 */
                        if (((asic == rt_rv200) ||
                                 (asic == rt_rv250)) &&
                                ((INREG( regs, RADEON_CONFIG_CNTL) & RADEON_CFG_ATI_REV_ID_MASK) <
                                  RADEON_CFG_ATI_REV_A13))
                        {
                                tmp = Radeon_INPLL(regs, asic, RADEON_PLL_PWRMGT_CNTL);
                                tmp |= RADEON_TCL_BYPASS_DISABLE;
                                Radeon_OUTPLL(regs, asic, RADEON_PLL_PWRMGT_CNTL, tmp);
                        }
                        snooze(15000);

                        /*enable dynamic mode for display clocks (PIXCLK and PIX2CLK)*/
                        tmp = Radeon_INPLL(regs, asic, RADEON_PIXCLKS_CNTL);
                        tmp |=  (RADEON_PIX2CLK_ALWAYS_ONb         |
                                RADEON_PIX2CLK_DAC_ALWAYS_ONb     |
                                RADEON_PIXCLK_BLEND_ALWAYS_ONb    |
                                RADEON_PIXCLK_GV_ALWAYS_ONb       |
                                RADEON_PIXCLK_DIG_TMDS_ALWAYS_ONb |
                                RADEON_PIXCLK_LVDS_ALWAYS_ONb     |
                                RADEON_PIXCLK_TMDS_ALWAYS_ONb);

                        Radeon_OUTPLL(regs, asic, RADEON_PIXCLKS_CNTL, tmp);
                        snooze(15000);

                        tmp = Radeon_INPLL(regs, asic, RADEON_VCLK_ECP_CNTL);
                        tmp |= (RADEON_PIXCLK_ALWAYS_ONb  |
                                RADEON_PIXCLK_DAC_ALWAYS_ONb);

                        Radeon_OUTPLL(regs, asic, RADEON_VCLK_ECP_CNTL, tmp);
                        snooze(15000);
                }
                SHOW_FLOW0( 3, "Dynamic Clock Scaling Enabled" );
                break;
        default:
                break;
        }
}