/*
 * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de.
 * Copyright 2002-2007, Axel Dörfler, axeld@pinc-software.de. All rights reserved.
 * Distributed under the terms of the MIT License.
 *
 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
 * Distributed under the terms of the NewOS License.
 */


#include "paging/32bit/X86PagingMethod32Bit.h"

#include <stdlib.h>
#include <string.h>

#include <AutoDeleter.h>

#include <arch/smp.h>
#include <arch_system_info.h>
#include <boot/kernel_args.h>
#include <interrupts.h>
#include <thread.h>
#include <vm/vm.h>
#include <vm/VMAddressSpace.h>

#include "paging/32bit/X86PagingStructures32Bit.h"
#include "paging/32bit/X86VMTranslationMap32Bit.h"
#include "paging/x86_physical_page_mapper.h"
#include "paging/x86_physical_page_mapper_large_memory.h"


//#define TRACE_X86_PAGING_METHOD_32_BIT
#ifdef TRACE_X86_PAGING_METHOD_32_BIT
#       define TRACE(x...) dprintf(x)
#else
#       define TRACE(x...) ;
#endif


#define MAX_INITIAL_POOLS       \
        (ROUNDUP(SMP_MAX_CPUS * TOTAL_SLOTS_PER_CPU + EXTRA_SLOTS, 1024) / 1024)


using X86LargePhysicalPageMapper::PhysicalPageSlot;


// #pragma mark - X86PagingMethod32Bit::PhysicalPageSlotPool


struct X86PagingMethod32Bit::PhysicalPageSlotPool final
        : X86LargePhysicalPageMapper::PhysicalPageSlotPool {
public:
        virtual                                         ~PhysicalPageSlotPool();

                        status_t                        InitInitial(kernel_args* args);
                        status_t                        InitInitialPostArea(kernel_args* args);

                        void                            Init(area_id dataArea, void* data,
                                                                        area_id virtualArea, addr_t virtualBase);

        virtual status_t                        AllocatePool(
                                                                        X86LargePhysicalPageMapper
                                                                                ::PhysicalPageSlotPool*& _pool);
        virtual void                            Map(phys_addr_t physicalAddress,
                                                                        addr_t virtualAddress);

public:
        static  PhysicalPageSlotPool sInitialPhysicalPagePool[MAX_INITIAL_POOLS];

private:
        area_id                                 fDataArea;
        area_id                                 fVirtualArea;
        addr_t                                  fVirtualBase;
        page_table_entry*               fPageTable;
};


X86PagingMethod32Bit::PhysicalPageSlotPool
        X86PagingMethod32Bit::PhysicalPageSlotPool::sInitialPhysicalPagePool[
                MAX_INITIAL_POOLS];


X86PagingMethod32Bit::PhysicalPageSlotPool::~PhysicalPageSlotPool()
{
}


status_t
X86PagingMethod32Bit::PhysicalPageSlotPool::InitInitial(kernel_args* args)
{
        // allocate a virtual address range for the pages to be mapped into
        addr_t virtualBase = vm_allocate_early(args, 1024 * B_PAGE_SIZE, 0, 0,
                kPageTableAlignment);
        if (virtualBase == 0) {
                panic("LargeMemoryPhysicalPageMapper::Init(): Failed to reserve "
                        "physical page pool space in virtual address space!");
                return B_ERROR;
        }

        // allocate memory for the page table and data
        size_t areaSize = B_PAGE_SIZE + sizeof(PhysicalPageSlot[1024]);
        page_table_entry* pageTable = (page_table_entry*)vm_allocate_early(args,
                areaSize, ~0L, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0);
        if (pageTable == 0) {
                panic("X86PagingMethod32Bit::PhysicalPageSlotPool::InitInitial(): "
                        "Failed to allocate memory for page table!");
                return B_ERROR;
        }

        // prepare the page table
        _EarlyPreparePageTables(pageTable, virtualBase, 1024 * B_PAGE_SIZE);

        // init the pool structure and add the initial pool
        Init(-1, pageTable, -1, (addr_t)virtualBase);

        return B_OK;
}


status_t
X86PagingMethod32Bit::PhysicalPageSlotPool::InitInitialPostArea(
        kernel_args* args)
{
        // create an area for the (already allocated) data
        size_t areaSize = B_PAGE_SIZE + sizeof(PhysicalPageSlot[1024]);
        void* temp = fPageTable;
        area_id area = create_area("physical page pool", &temp,
                B_EXACT_ADDRESS, areaSize, B_ALREADY_WIRED,
                B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
        if (area < B_OK) {
                panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to "
                        "create area for physical page pool.");
                return area;
        }
        fDataArea = area;

        // create an area for the virtual address space
        temp = (void*)fVirtualBase;
        area = vm_create_null_area(VMAddressSpace::KernelID(),
                "physical page pool space", &temp, B_EXACT_ADDRESS,
                1024 * B_PAGE_SIZE, 0);
        if (area < B_OK) {
                panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to "
                        "create area for physical page pool space.");
                return area;
        }
        fVirtualArea = area;

        return B_OK;
}


void
X86PagingMethod32Bit::PhysicalPageSlotPool::Init(area_id dataArea, void* data,
        area_id virtualArea, addr_t virtualBase)
{
        fDataArea = dataArea;
        fVirtualArea = virtualArea;
        fVirtualBase = virtualBase;
        fPageTable = (page_table_entry*)data;

        // init slot list
        fSlots = (PhysicalPageSlot*)(fPageTable + 1024);
        addr_t slotAddress = virtualBase;
        for (int32 i = 0; i < 1024; i++, slotAddress += B_PAGE_SIZE) {
                PhysicalPageSlot* slot = &fSlots[i];
                slot->next = slot + 1;
                slot->pool = this;
                slot->address = slotAddress;
        }

        fSlots[1023].next = NULL;
                // terminate list
}


void
X86PagingMethod32Bit::PhysicalPageSlotPool::Map(phys_addr_t physicalAddress,
        addr_t virtualAddress)
{
        page_table_entry& pte = fPageTable[
                (virtualAddress - fVirtualBase) / B_PAGE_SIZE];
        pte = (physicalAddress & X86_PTE_ADDRESS_MASK)
                | X86_PTE_WRITABLE | X86_PTE_GLOBAL | X86_PTE_PRESENT;

        invalidate_TLB(virtualAddress);
}


status_t
X86PagingMethod32Bit::PhysicalPageSlotPool::AllocatePool(
        X86LargePhysicalPageMapper::PhysicalPageSlotPool*& _pool)
{
        // create the pool structure
        PhysicalPageSlotPool* pool = new(std::nothrow) PhysicalPageSlotPool;
        if (pool == NULL)
                return B_NO_MEMORY;
        ObjectDeleter<PhysicalPageSlotPool> poolDeleter(pool);

        // create an area that can contain the page table and the slot
        // structures
        size_t areaSize = B_PAGE_SIZE + sizeof(PhysicalPageSlot[1024]);
        void* data;
        virtual_address_restrictions virtualRestrictions = {};
        virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS;
        physical_address_restrictions physicalRestrictions = {};
        area_id dataArea = create_area_etc(B_SYSTEM_TEAM, "physical page pool",
                PAGE_ALIGN(areaSize), B_FULL_LOCK,
                B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT, 0,
                &virtualRestrictions, &physicalRestrictions, &data);
        if (dataArea < 0)
                return dataArea;

        // create the null area for the virtual address space
        void* virtualBase;
        area_id virtualArea = vm_create_null_area(
                VMAddressSpace::KernelID(), "physical page pool space",
                &virtualBase, B_ANY_KERNEL_BLOCK_ADDRESS, 1024 * B_PAGE_SIZE,
                CREATE_AREA_PRIORITY_VIP);
        if (virtualArea < 0) {
                delete_area(dataArea);
                return virtualArea;
        }

        // prepare the page table
        memset(data, 0, B_PAGE_SIZE);

        // get the page table's physical address
        phys_addr_t physicalTable;
        X86VMTranslationMap32Bit* map = static_cast<X86VMTranslationMap32Bit*>(
                VMAddressSpace::Kernel()->TranslationMap());
        uint32 dummyFlags;
        cpu_status state = disable_interrupts();
        map->QueryInterrupt((addr_t)data, &physicalTable, &dummyFlags);
        restore_interrupts(state);

        // put the page table into the page directory
        int32 index = (addr_t)virtualBase / (B_PAGE_SIZE * 1024);
        page_directory_entry* entry
                = &map->PagingStructures32Bit()->pgdir_virt[index];
        PutPageTableInPageDir(entry, physicalTable,
                B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
        X86PagingStructures32Bit::UpdateAllPageDirs(index, *entry);

        // init the pool structure
        pool->Init(dataArea, data, virtualArea, (addr_t)virtualBase);
        poolDeleter.Detach();
        _pool = pool;
        return B_OK;
}


// #pragma mark - X86PagingMethod32Bit


X86PagingMethod32Bit::X86PagingMethod32Bit()
        :
        fPageHole(NULL),
        fPageHolePageDir(NULL),
        fKernelPhysicalPageDirectory(0),
        fKernelVirtualPageDirectory(NULL),
        fPhysicalPageMapper(NULL),
        fKernelPhysicalPageMapper(NULL)
{
}


X86PagingMethod32Bit::~X86PagingMethod32Bit()
{
}


status_t
X86PagingMethod32Bit::Init(kernel_args* args,
        VMPhysicalPageMapper** _physicalPageMapper)
{
        TRACE("X86PagingMethod32Bit::Init(): entry\n");

        // Ignore all memory beyond the maximum 32-bit address.
        static const phys_addr_t kLimit = 1ULL << 32;
        for (uint32 i = 0; i < args->num_physical_memory_ranges; i++) {
                addr_range& range = args->physical_memory_range[i];
                if (range.start >= kLimit)
                        range.size = 0;
                else if ((range.start + range.size) > kLimit)
                        range.size = kLimit - range.start;
        }

        // page hole set up in stage2
        fPageHole = (page_table_entry*)(addr_t)args->arch_args.page_hole;
        // calculate where the pgdir would be
        fPageHolePageDir = (page_directory_entry*)
                (((addr_t)args->arch_args.page_hole)
                        + (B_PAGE_SIZE * 1024 - B_PAGE_SIZE));
        // clear out the bottom 2 GB, unmap everything
        memset(fPageHolePageDir + FIRST_USER_PGDIR_ENT, 0,
                sizeof(page_directory_entry) * NUM_USER_PGDIR_ENTS);

        fKernelPhysicalPageDirectory = args->arch_args.phys_pgdir;
        fKernelVirtualPageDirectory = (page_directory_entry*)(addr_t)
                args->arch_args.vir_pgdir;

#ifdef TRACE_X86_PAGING_METHOD_32_BIT
        TRACE("page hole: %p, page dir: %p\n", fPageHole, fPageHolePageDir);
        TRACE("page dir: %p (physical: %#" B_PRIx32 ")\n",
                fKernelVirtualPageDirectory, fKernelPhysicalPageDirectory);
#endif

        X86PagingStructures32Bit::StaticInit();

        // create the initial pools for the physical page mapper
        int32 poolCount = _GetInitialPoolCount();
        PhysicalPageSlotPool* pool = PhysicalPageSlotPool::sInitialPhysicalPagePool;

        for (int32 i = 0; i < poolCount; i++) {
                new(&pool[i]) PhysicalPageSlotPool;
                status_t error = pool[i].InitInitial(args);
                if (error != B_OK) {
                        panic("X86PagingMethod32Bit::Init(): Failed to create initial pool "
                                "for physical page mapper!");
                        return error;
                }
        }

        // create physical page mapper
        large_memory_physical_page_ops_init(args, pool, poolCount, sizeof(*pool),
                fPhysicalPageMapper, fKernelPhysicalPageMapper);
                // TODO: Select the best page mapper!

        // enable global page feature if available
        if (x86_check_feature(IA32_FEATURE_PGE, FEATURE_COMMON)) {
                // this prevents kernel pages from being flushed from TLB on
                // context-switch
                x86_write_cr4(x86_read_cr4() | IA32_CR4_GLOBAL_PAGES);
        }

        TRACE("X86PagingMethod32Bit::Init(): done\n");

        *_physicalPageMapper = fPhysicalPageMapper;
        return B_OK;
}


status_t
X86PagingMethod32Bit::InitPostArea(kernel_args* args)
{
        // now that the vm is initialized, create an area that represents
        // the page hole
        void *temp;
        area_id area;

        // unmap the page hole hack we were using before
        fKernelVirtualPageDirectory[1023] = 0;
        fPageHolePageDir = NULL;
        fPageHole = NULL;

        temp = (void*)fKernelVirtualPageDirectory;
        area = create_area("kernel_pgdir", &temp, B_EXACT_ADDRESS, B_PAGE_SIZE,
                B_ALREADY_WIRED, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
        if (area < B_OK)
                return area;

        int32 poolCount = _GetInitialPoolCount();
        for (int32 i = 0; i < poolCount; i++) {
                status_t error = PhysicalPageSlotPool::sInitialPhysicalPagePool[i]
                        .InitInitialPostArea(args);
                if (error != B_OK)
                        return error;
        }

        return B_OK;
}


status_t
X86PagingMethod32Bit::CreateTranslationMap(bool kernel, VMTranslationMap** _map)
{
        X86VMTranslationMap32Bit* map = new(std::nothrow) X86VMTranslationMap32Bit;
        if (map == NULL)
                return B_NO_MEMORY;

        status_t error = map->Init(kernel);
        if (error != B_OK) {
                delete map;
                return error;
        }

        *_map = map;
        return B_OK;
}


status_t
X86PagingMethod32Bit::MapEarly(kernel_args* args, addr_t virtualAddress,
        phys_addr_t physicalAddress, uint8 attributes)
{
        // XXX horrible back door to map a page quickly regardless of translation
        // map object, etc. used only during VM setup.
        // uses a 'page hole' set up in the stage 2 bootloader. The page hole is
        // created by pointing one of the pgdir entries back at itself, effectively
        // mapping the contents of all of the 4MB of pagetables into a 4 MB region.
        // It's only used here, and is later unmapped.

        // check to see if a page table exists for this range
        int index = VADDR_TO_PDENT(virtualAddress);
        if ((fPageHolePageDir[index] & X86_PDE_PRESENT) == 0) {
                phys_addr_t pgtable;
                page_directory_entry *e;
                // we need to allocate a pgtable
                pgtable = vm_allocate_early_physical_page(args);
                // pgtable is in pages, convert to physical address
                pgtable *= B_PAGE_SIZE;

                TRACE("X86PagingMethod32Bit::MapEarly(): asked for free page for "
                        "pgtable. %#" B_PRIxPHYSADDR "\n", pgtable);

                // put it in the pgdir
                e = &fPageHolePageDir[index];
                PutPageTableInPageDir(e, pgtable, attributes);

                // zero it out in it's new mapping
                memset((unsigned int*)((addr_t)fPageHole
                                + (virtualAddress / B_PAGE_SIZE / 1024) * B_PAGE_SIZE),
                        0, B_PAGE_SIZE);
        }

        ASSERT_PRINT(
                (fPageHole[virtualAddress / B_PAGE_SIZE] & X86_PTE_PRESENT) == 0,
                "virtual address: %#" B_PRIxADDR ", pde: %#" B_PRIx32
                ", existing pte: %#" B_PRIx32, virtualAddress, fPageHolePageDir[index],
                fPageHole[virtualAddress / B_PAGE_SIZE]);

        // now, fill in the pentry
        PutPageTableEntryInTable(fPageHole + virtualAddress / B_PAGE_SIZE,
                physicalAddress, attributes, 0, IS_KERNEL_ADDRESS(virtualAddress));

        return B_OK;
}


bool
X86PagingMethod32Bit::IsKernelPageAccessible(addr_t virtualAddress,
        uint32 protection)
{
        // We only trust the kernel team's page directory. So switch to it first.
        // Always set it to make sure the TLBs don't contain obsolete data.
        uint32 physicalPageDirectory = x86_read_cr3();
        x86_write_cr3(fKernelPhysicalPageDirectory);

        // get the page directory entry for the address
        page_directory_entry pageDirectoryEntry;
        uint32 index = VADDR_TO_PDENT(virtualAddress);

        if (physicalPageDirectory == fKernelPhysicalPageDirectory) {
                pageDirectoryEntry = fKernelVirtualPageDirectory[index];
        } else if (fPhysicalPageMapper != NULL) {
                // map the original page directory and get the entry
                void* handle;
                addr_t virtualPageDirectory;
                status_t error = fPhysicalPageMapper->GetPageDebug(
                        physicalPageDirectory, &virtualPageDirectory, &handle);
                if (error == B_OK) {
                        pageDirectoryEntry
                                = ((page_directory_entry*)virtualPageDirectory)[index];
                        fPhysicalPageMapper->PutPageDebug(virtualPageDirectory, handle);
                } else
                        pageDirectoryEntry = 0;
        } else
                pageDirectoryEntry = 0;

        // map the page table and get the entry
        page_table_entry pageTableEntry;
        index = VADDR_TO_PTENT(virtualAddress);

        if ((pageDirectoryEntry & X86_PDE_PRESENT) != 0
                        && fPhysicalPageMapper != NULL) {
                void* handle;
                addr_t virtualPageTable;
                status_t error = fPhysicalPageMapper->GetPageDebug(
                        pageDirectoryEntry & X86_PDE_ADDRESS_MASK, &virtualPageTable,
                        &handle);
                if (error == B_OK) {
                        pageTableEntry = ((page_table_entry*)virtualPageTable)[index];
                        fPhysicalPageMapper->PutPageDebug(virtualPageTable, handle);
                } else
                        pageTableEntry = 0;
        } else
                pageTableEntry = 0;

        // switch back to the original page directory
        if (physicalPageDirectory != fKernelPhysicalPageDirectory)
                x86_write_cr3(physicalPageDirectory);

        if ((pageTableEntry & X86_PTE_PRESENT) == 0)
                return false;

        // present means kernel-readable, so check for writable
        return (protection & B_KERNEL_WRITE_AREA) == 0
                || (pageTableEntry & X86_PTE_WRITABLE) != 0;
}


/*static*/ void
X86PagingMethod32Bit::PutPageTableInPageDir(page_directory_entry* entry,
        phys_addr_t pgtablePhysical, uint32 attributes)
{
        *entry = (pgtablePhysical & X86_PDE_ADDRESS_MASK)
                | X86_PDE_PRESENT
                | X86_PDE_WRITABLE
                | X86_PDE_USER;
                // TODO: we ignore the attributes of the page table - for compatibility
                // with BeOS we allow having user accessible areas in the kernel address
                // space. This is currently being used by some drivers, mainly for the
                // frame buffer. Our current real time data implementation makes use of
                // this fact, too.
                // We might want to get rid of this possibility one day, especially if
                // we intend to port it to a platform that does not support this.
}


/*static*/ void
X86PagingMethod32Bit::PutPageTableEntryInTable(page_table_entry* entry,
        phys_addr_t physicalAddress, uint32 attributes, uint32 memoryType,
        bool globalPage)
{
        page_table_entry page = (physicalAddress & X86_PTE_ADDRESS_MASK)
                | X86_PTE_PRESENT | (globalPage ? X86_PTE_GLOBAL : 0)
                | MemoryTypeToPageTableEntryFlags(memoryType);

        // if the page is user accessible, it's automatically
        // accessible in kernel space, too (but with the same
        // protection)
        if ((attributes & B_USER_PROTECTION) != 0) {
                page |= X86_PTE_USER;
                if ((attributes & B_WRITE_AREA) != 0)
                        page |= X86_PTE_WRITABLE;
        } else if ((attributes & B_KERNEL_WRITE_AREA) != 0)
                page |= X86_PTE_WRITABLE;

        // put it in the page table
        *(volatile page_table_entry*)entry = page;
}


inline int32
X86PagingMethod32Bit::_GetInitialPoolCount()
{
        int32 requiredSlots = smp_get_num_cpus() * TOTAL_SLOTS_PER_CPU
                        + EXTRA_SLOTS;
        return (requiredSlots + 1023) / 1024;
}


/*static*/ void
X86PagingMethod32Bit::_EarlyPreparePageTables(page_table_entry* pageTables,
        addr_t address, size_t size)
{
        memset(pageTables, 0, B_PAGE_SIZE * (size / (B_PAGE_SIZE * 1024)));

        // put the array of pgtables directly into the kernel pagedir
        // these will be wired and kept mapped into virtual space to be easy to get
        // to
        {
                addr_t virtualTable = (addr_t)pageTables;

                page_directory_entry* pageHolePageDir
                        = X86PagingMethod32Bit::Method()->PageHolePageDir();

                for (size_t i = 0; i < (size / (B_PAGE_SIZE * 1024));
                                i++, virtualTable += B_PAGE_SIZE) {
                        phys_addr_t physicalTable = 0;
                        _EarlyQuery(virtualTable, &physicalTable);
                        page_directory_entry* entry = &pageHolePageDir[
                                (address / (B_PAGE_SIZE * 1024)) + i];
                        PutPageTableInPageDir(entry, physicalTable,
                                B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
                }
        }
}


//! TODO: currently assumes this translation map is active
/*static*/ status_t
X86PagingMethod32Bit::_EarlyQuery(addr_t virtualAddress,
        phys_addr_t *_physicalAddress)
{
        X86PagingMethod32Bit* method = X86PagingMethod32Bit::Method();
        int index = VADDR_TO_PDENT(virtualAddress);
        if ((method->PageHolePageDir()[index] & X86_PDE_PRESENT) == 0) {
                // no pagetable here
                return B_ERROR;
        }

        page_table_entry* entry = method->PageHole() + virtualAddress / B_PAGE_SIZE;
        if ((*entry & X86_PTE_PRESENT) == 0) {
                // page mapping not valid
                return B_ERROR;
        }

        *_physicalAddress = *entry & X86_PTE_ADDRESS_MASK;
        return B_OK;
}