/*
 * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de.
 * Copyright 2002-2007, Axel Dörfler, axeld@pinc-software.de. All rights reserved.
 * Distributed under the terms of the MIT License.
 *
 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
 * Distributed under the terms of the NewOS License.
 */


#include "paging/pae/X86PagingMethodPAE.h"

#include <stdlib.h>
#include <string.h>

#include <AutoDeleter.h>

#include <arch/smp.h>
#include <boot/kernel_args.h>
#include <util/AutoLock.h>
#include <vm/vm.h>
#include <vm/vm_page.h>
#include <vm/VMAddressSpace.h>

#include "paging/32bit/paging.h"
#include "paging/32bit/X86PagingMethod32Bit.h"
#include "paging/pae/X86PagingStructuresPAE.h"
#include "paging/pae/X86VMTranslationMapPAE.h"
#include "paging/x86_physical_page_mapper.h"
#include "paging/x86_physical_page_mapper_large_memory.h"


//#define TRACE_X86_PAGING_METHOD_PAE
#ifdef TRACE_X86_PAGING_METHOD_PAE
#       define TRACE(x...) dprintf(x)
#else
#       define TRACE(x...) ;
#endif


#if B_HAIKU_PHYSICAL_BITS == 64


#define MAX_INITIAL_POOLS       \
        (ROUNDUP(SMP_MAX_CPUS * TOTAL_SLOTS_PER_CPU + EXTRA_SLOTS,      \
                        kPAEPageTableEntryCount)        \
                / kPAEPageTableEntryCount)


using X86LargePhysicalPageMapper::PhysicalPageSlot;


// number of 32 bit pages that will be cached
static const page_num_t kMaxFree32BitPagesCount = 32;


// #pragma mark - ToPAESwitcher


struct X86PagingMethodPAE::ToPAESwitcher {
        ToPAESwitcher(kernel_args* args)
                :
                fKernelArgs(args)
        {
                // page hole set up in the boot loader
                fPageHole = (page_table_entry*)
                        (addr_t)fKernelArgs->arch_args.page_hole;

                // calculate where the page dir would be
                fPageHolePageDir = (page_directory_entry*)
                        (((addr_t)fKernelArgs->arch_args.page_hole)
                                + (B_PAGE_SIZE * 1024 - B_PAGE_SIZE));

                fPhysicalPageDir = fKernelArgs->arch_args.phys_pgdir;

                TRACE("page hole: %p\n", fPageHole);
                TRACE("page dir:  %p (physical: %#" B_PRIxPHYSADDR ")\n",
                        fPageHolePageDir, fPhysicalPageDir);
        }

        void Switch(pae_page_directory_pointer_table_entry*& _virtualPDPT,
                phys_addr_t& _physicalPDPT, void*& _pageStructures,
                size_t& _pageStructuresSize, pae_page_directory_entry** pageDirs,
                phys_addr_t* physicalPageDirs, addr_t& _freeVirtualSlot,
                pae_page_table_entry*& _freeVirtualSlotPTE)
        {
                // count the page tables we have to translate
                uint32 pageTableCount = 0;
                for (uint32 i = FIRST_KERNEL_PGDIR_ENT;
                        i < FIRST_KERNEL_PGDIR_ENT + NUM_KERNEL_PGDIR_ENTS; i++) {
                        page_directory_entry entry = fPageHolePageDir[i];
                        if ((entry & X86_PDE_PRESENT) != 0)
                                pageTableCount++;
                }

                TRACE("page tables to translate: %" B_PRIu32 "\n", pageTableCount);

                // The pages we need to allocate to do our job:
                // + 1 page dir pointer table
                // + 4 page dirs
                // + 2 * page tables (each has 512 instead of 1024 entries)
                // + 1 page for the free virtual slot (no physical page needed)
                uint32 pagesNeeded = 1 + 4 + pageTableCount * 2 + 1;

                // We need additional PAE page tables for the new pages we're going to
                // allocate: Two tables for every 1024 pages to map, i.e. 2 additional
                // pages for every 1022 pages we want to allocate. We also need 32 bit
                // page tables, but we don't need additional virtual space for them,
                // since we can access then via the page hole.
                pagesNeeded += ((pagesNeeded + 1021) / 1022) * 2;

                TRACE("pages needed: %" B_PRIu32 "\n", pagesNeeded);

                // allocate the pages we need
                _AllocateNeededPages(pagesNeeded);

                // prepare the page directory pointer table
                phys_addr_t physicalPDPT = 0;
                pae_page_directory_pointer_table_entry* pdpt
                        = (pae_page_directory_pointer_table_entry*)_NextPage(true,
                                physicalPDPT);

                for (int32 i = 0; i < 4; i++) {
                        fPageDirs[i] = (pae_page_directory_entry*)_NextPage(true,
                                fPhysicalPageDirs[i]);

                        pdpt[i] = X86_PAE_PDPTE_PRESENT
                                | (fPhysicalPageDirs[i] & X86_PAE_PDPTE_ADDRESS_MASK);
                }

                // Since we have to enable PAE in two steps -- setting cr3 to the PDPT
                // and setting the cr4 PAE bit -- we copy the kernel page dir entries to
                // the PDPT page, so after setting cr3, we continue to have working
                // kernel mappings. This requires that the PDPTE registers and the
                // page dir entries don't interect, obviously.
                ASSERT(4 * sizeof(pae_page_directory_pointer_table_entry)
                        <= FIRST_KERNEL_PGDIR_ENT * sizeof(page_directory_entry));

                // translate the page tables
                for (uint32 i = FIRST_KERNEL_PGDIR_ENT;
                        i < FIRST_KERNEL_PGDIR_ENT + NUM_KERNEL_PGDIR_ENTS; i++) {
                        if ((fPageHolePageDir[i] & X86_PDE_PRESENT) != 0) {
                                // two PAE page tables per 32 bit page table
                                _TranslatePageTable((addr_t)i * 1024 * B_PAGE_SIZE);
                                _TranslatePageTable(((addr_t)i * 1024 + 512) * B_PAGE_SIZE);

                                // copy the page directory entry to the PDPT page
                                ((page_directory_entry*)pdpt)[i] = fPageHolePageDir[i];
                        }
                }

                TRACE("free virtual slot: %#" B_PRIxADDR ", PTE: %p\n",
                        fFreeVirtualSlot, fFreeVirtualSlotPTE);

                // enable PAE on all CPUs
                call_all_cpus_sync(&_EnablePAE, (void*)(addr_t)physicalPDPT);

                // if available enable NX-bit (No eXecute)
                if (x86_check_feature(IA32_FEATURE_AMD_EXT_NX, FEATURE_EXT_AMD))
                        call_all_cpus_sync(&_EnableExecutionDisable, NULL);

                // set return values
                _virtualPDPT = pdpt;
                _physicalPDPT = physicalPDPT;
                _pageStructures = fAllocatedPages;
                _pageStructuresSize = (size_t)fUsedPagesCount * B_PAGE_SIZE;
                memcpy(pageDirs, fPageDirs, sizeof(fPageDirs));
                memcpy(physicalPageDirs, fPhysicalPageDirs, sizeof(fPhysicalPageDirs));

                _freeVirtualSlot = fFreeVirtualSlot;
                _freeVirtualSlotPTE = fFreeVirtualSlotPTE;
        }

private:
        static void _EnablePAE(void* physicalPDPT, int cpu)
        {
                x86_write_cr3((addr_t)physicalPDPT);
                x86_write_cr4(x86_read_cr4() | IA32_CR4_PAE | IA32_CR4_GLOBAL_PAGES);
        }

        static void _EnableExecutionDisable(void* dummy, int cpu)
        {
                x86_write_msr(IA32_MSR_EFER, x86_read_msr(IA32_MSR_EFER)
                        | IA32_MSR_EFER_NX);
        }

        void _TranslatePageTable(addr_t virtualBase)
        {
                page_table_entry* entry = &fPageHole[virtualBase / B_PAGE_SIZE];

                // allocate a PAE page table
                phys_addr_t physicalTable = 0;
                pae_page_table_entry* paeTable = (pae_page_table_entry*)_NextPage(false,
                        physicalTable);

                // enter it into the page dir
                pae_page_directory_entry* pageDirEntry = PageDirEntryForAddress(
                        fPageDirs, virtualBase);
                PutPageTableInPageDir(pageDirEntry, physicalTable,
                        B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);

                pae_page_table_entry* paeEntry = paeTable;
                for (uint32 i = 0; i < kPAEPageTableEntryCount;
                                i++, entry++, paeEntry++) {
                        if ((*entry & X86_PTE_PRESENT) != 0
                                && _IsVirtualAddressAllocated(virtualBase + i * B_PAGE_SIZE)) {
                                // Note, we use the fact that the PAE flags are defined to the
                                // same values.
                                *paeEntry = *entry & (X86_PTE_PRESENT
                                        | X86_PTE_WRITABLE
                                        | X86_PTE_USER
                                        | X86_PTE_WRITE_THROUGH
                                        | X86_PTE_CACHING_DISABLED
                                        | X86_PTE_GLOBAL
                                        | X86_PTE_ADDRESS_MASK);
                        } else
                                *paeEntry = 0;
                }

                if (fFreeVirtualSlot / kPAEPageTableRange
                                == virtualBase / kPAEPageTableRange) {
                        fFreeVirtualSlotPTE = paeTable
                                + fFreeVirtualSlot / B_PAGE_SIZE % kPAEPageTableEntryCount;
                }
        }

        void _AllocateNeededPages(uint32 pagesNeeded)
        {
                size_t virtualSize = ROUNDUP(pagesNeeded, 1024) * B_PAGE_SIZE;
                addr_t virtualBase = vm_allocate_early(fKernelArgs, virtualSize, 0, 0,
                        kPageTableAlignment);
                if (virtualBase == 0) {
                        panic("Failed to reserve virtual address space for the switch to "
                                "PAE!");
                }

                TRACE("virtual space: %#" B_PRIxADDR ", size: %#" B_PRIxSIZE "\n",
                        virtualBase, virtualSize);

                // allocate pages for the 32 bit page tables and prepare the tables
                uint32 oldPageTableCount = virtualSize / B_PAGE_SIZE / 1024;
                for (uint32 i = 0; i < oldPageTableCount; i++) {
                        // allocate a page
                        phys_addr_t physicalTable = _Allocate32BitPage();

                        // put the page into the page dir
                        page_directory_entry* entry = &fPageHolePageDir[
                                virtualBase / B_PAGE_SIZE / 1024 + i];
                        X86PagingMethod32Bit::PutPageTableInPageDir(entry, physicalTable,
                                B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);

                        // clear the table
                        memset((void*)((addr_t)fPageHole
                                        + (virtualBase / B_PAGE_SIZE / 1024 + i) * B_PAGE_SIZE),
                                0, B_PAGE_SIZE);
                }

                // We don't need a physical page for the free virtual slot.
                pagesNeeded--;

                // allocate and map the pages we need
                for (uint32 i = 0; i < pagesNeeded; i++) {
                        // allocate a page
                        phys_addr_t physicalAddress = _Allocate32BitPage();

                        // put the page into the page table
                        page_table_entry* entry = fPageHole + virtualBase / B_PAGE_SIZE + i;
                        X86PagingMethod32Bit::PutPageTableEntryInTable(entry,
                                physicalAddress, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0,
                                true);

                        // Write the page's physical address into the page itself, so we
                        // don't need to look it up later.
                        *(phys_addr_t*)(virtualBase + i * B_PAGE_SIZE) = physicalAddress;
                }

                fAllocatedPages = (uint8*)virtualBase;
                fAllocatedPagesCount = pagesNeeded;
                fUsedPagesCount = 0;
                fFreeVirtualSlot
                        = (addr_t)(fAllocatedPages + pagesNeeded * B_PAGE_SIZE);
        }

        phys_addr_t _Allocate32BitPage()
        {
                phys_addr_t physicalAddress
                        = (phys_addr_t)vm_allocate_early_physical_page(fKernelArgs, 0xffffffff)
                                * B_PAGE_SIZE;
                if (physicalAddress == 0 || physicalAddress > 0xffffffff) {
                        panic("Failed to allocate 32-bit-addressable page for the switch "
                                "to PAE!");
                        return 0;
                }
                return physicalAddress;
        }

        void* _NextPage(bool clearPage, phys_addr_t& _physicalAddress)
        {
                if (fUsedPagesCount >= fAllocatedPagesCount) {
                        panic("X86PagingMethodPAE::ToPAESwitcher::_NextPage(): no more "
                                "allocated pages!");
                        return NULL;
                }

                void* page = fAllocatedPages + (fUsedPagesCount++) * B_PAGE_SIZE;
                _physicalAddress = *((phys_addr_t*)page);

                if (clearPage)
                        memset(page, 0, B_PAGE_SIZE);

                return page;
        }

        bool _IsVirtualAddressAllocated(addr_t address) const
        {
                for (uint32 i = 0; i < fKernelArgs->num_virtual_allocated_ranges; i++) {
                        addr_t start = fKernelArgs->virtual_allocated_range[i].start;
                        addr_t end = start + fKernelArgs->virtual_allocated_range[i].size;
                        if (address < start)
                                return false;
                        if (address <= end - 1)
                                return true;
                }

                return false;
        }

private:
        kernel_args*                            fKernelArgs;
        page_table_entry*                       fPageHole;
        page_directory_entry*           fPageHolePageDir;
        phys_addr_t                                     fPhysicalPageDir;
        uint8*                                          fAllocatedPages;
        uint32                                          fAllocatedPagesCount;
        uint32                                          fUsedPagesCount;
        addr_t                                          fFreeVirtualSlot;
        pae_page_table_entry*           fFreeVirtualSlotPTE;
        pae_page_directory_entry*       fPageDirs[4];
        phys_addr_t                                     fPhysicalPageDirs[4];
};


// #pragma mark - PhysicalPageSlotPool


struct X86PagingMethodPAE::PhysicalPageSlotPool final
        : X86LargePhysicalPageMapper::PhysicalPageSlotPool {
public:
        virtual                                         ~PhysicalPageSlotPool();

                        status_t                        InitInitial(X86PagingMethodPAE* method,
                                                                        kernel_args* args);
                        status_t                        InitInitialPostArea(kernel_args* args);

                        void                            Init(area_id dataArea,
                                                                        pae_page_table_entry* pageTable,
                                                                        area_id virtualArea, addr_t virtualBase);

        virtual status_t                        AllocatePool(
                                                                        X86LargePhysicalPageMapper
                                                                                ::PhysicalPageSlotPool*& _pool);
        virtual void                            Map(phys_addr_t physicalAddress,
                                                                        addr_t virtualAddress);

public:
        static  PhysicalPageSlotPool sInitialPhysicalPagePool[MAX_INITIAL_POOLS];

private:
                        area_id                         fDataArea;
                        area_id                         fVirtualArea;
                        addr_t                          fVirtualBase;
                        pae_page_table_entry* fPageTable;
};


X86PagingMethodPAE::PhysicalPageSlotPool
        X86PagingMethodPAE::PhysicalPageSlotPool::sInitialPhysicalPagePool[
                MAX_INITIAL_POOLS];


X86PagingMethodPAE::PhysicalPageSlotPool::~PhysicalPageSlotPool()
{
}


status_t
X86PagingMethodPAE::PhysicalPageSlotPool::InitInitial(
        X86PagingMethodPAE* method, kernel_args* args)
{
        // allocate a virtual address range for the pages to be mapped into
        addr_t virtualBase = vm_allocate_early(args, kPAEPageTableRange, 0, 0,
                kPAEPageTableRange);
        if (virtualBase == 0) {
                panic("LargeMemoryPhysicalPageMapper::Init(): Failed to reserve "
                        "physical page pool space in virtual address space!");
                return B_ERROR;
        }

        // allocate memory for the page table and data
        size_t areaSize = B_PAGE_SIZE
                + sizeof(PhysicalPageSlot[kPAEPageTableEntryCount]);
        pae_page_table_entry* pageTable = (pae_page_table_entry*)vm_allocate_early(
                args, areaSize, ~0L, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0);
        if (pageTable == 0) {
                panic("X86PagingMethodPAE::PhysicalPageSlotPool::InitInitial(): Failed "
                        "to allocate memory for page table!");
                return B_ERROR;
        }

        // clear the page table and put it in the page dir
        memset(pageTable, 0, B_PAGE_SIZE);

        phys_addr_t physicalTable = 0;
        method->_EarlyQuery((addr_t)pageTable, &physicalTable);

        pae_page_directory_entry* entry = PageDirEntryForAddress(
                method->KernelVirtualPageDirs(), virtualBase);
        PutPageTableInPageDir(entry, physicalTable,
                B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);

        // init the pool structure and add the initial pool
        Init(-1, pageTable, -1, (addr_t)virtualBase);

        return B_OK;
}


status_t
X86PagingMethodPAE::PhysicalPageSlotPool::InitInitialPostArea(
        kernel_args* args)
{
        // create an area for the (already allocated) data
        size_t areaSize = B_PAGE_SIZE
                + sizeof(PhysicalPageSlot[kPAEPageTableEntryCount]);
        void* temp = fPageTable;
        area_id area = create_area("physical page pool", &temp,
                B_EXACT_ADDRESS, areaSize, B_ALREADY_WIRED,
                B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
        if (area < B_OK) {
                panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to "
                        "create area for physical page pool.");
                return area;
        }
        fDataArea = area;

        // create an area for the virtual address space
        temp = (void*)fVirtualBase;
        area = vm_create_null_area(VMAddressSpace::KernelID(),
                "physical page pool space", &temp, B_EXACT_ADDRESS,
                kPAEPageTableRange, 0);
        if (area < B_OK) {
                panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to "
                        "create area for physical page pool space.");
                return area;
        }
        fVirtualArea = area;

        return B_OK;
}


void
X86PagingMethodPAE::PhysicalPageSlotPool::Init(area_id dataArea,
        pae_page_table_entry* pageTable, area_id virtualArea, addr_t virtualBase)
{
        fDataArea = dataArea;
        fVirtualArea = virtualArea;
        fVirtualBase = virtualBase;
        fPageTable = pageTable;

        // init slot list
        fSlots = (PhysicalPageSlot*)(fPageTable + kPAEPageTableEntryCount);
        addr_t slotAddress = virtualBase;
        for (uint32 i = 0; i < kPAEPageTableEntryCount;
                        i++, slotAddress += B_PAGE_SIZE) {
                PhysicalPageSlot* slot = &fSlots[i];
                slot->next = slot + 1;
                slot->pool = this;
                slot->address = slotAddress;
        }

        fSlots[kPAEPageTableEntryCount - 1].next = NULL;
                // terminate list
}


void
X86PagingMethodPAE::PhysicalPageSlotPool::Map(phys_addr_t physicalAddress,
        addr_t virtualAddress)
{
        pae_page_table_entry& pte = fPageTable[
                (virtualAddress - fVirtualBase) / B_PAGE_SIZE];
        pte = (physicalAddress & X86_PAE_PTE_ADDRESS_MASK)
                | X86_PAE_PTE_WRITABLE | X86_PAE_PTE_GLOBAL | X86_PAE_PTE_PRESENT;

        invalidate_TLB(virtualAddress);
}


status_t
X86PagingMethodPAE::PhysicalPageSlotPool::AllocatePool(
        X86LargePhysicalPageMapper::PhysicalPageSlotPool*& _pool)
{
        // create the pool structure
        PhysicalPageSlotPool* pool = new(std::nothrow) PhysicalPageSlotPool;
        if (pool == NULL)
                return B_NO_MEMORY;
        ObjectDeleter<PhysicalPageSlotPool> poolDeleter(pool);

        // create an area that can contain the page table and the slot
        // structures
        size_t areaSize = B_PAGE_SIZE
                + sizeof(PhysicalPageSlot[kPAEPageTableEntryCount]);
        void* data;
        virtual_address_restrictions virtualRestrictions = {};
        virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS;
        physical_address_restrictions physicalRestrictions = {};
        area_id dataArea = create_area_etc(B_SYSTEM_TEAM, "physical page pool",
                PAGE_ALIGN(areaSize), B_FULL_LOCK,
                B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT, 0,
                &virtualRestrictions, &physicalRestrictions, &data);
        if (dataArea < 0)
                return dataArea;

        // create the null area for the virtual address space
        void* virtualBase;
        area_id virtualArea = vm_create_null_area(
                VMAddressSpace::KernelID(), "physical page pool space",
                &virtualBase, B_ANY_KERNEL_BLOCK_ADDRESS, kPAEPageTableRange,
                CREATE_AREA_PRIORITY_VIP);
        if (virtualArea < 0) {
                delete_area(dataArea);
                return virtualArea;
        }

        // prepare the page table
        memset(data, 0, B_PAGE_SIZE);

        // get the page table's physical address
        phys_addr_t physicalTable;
        X86VMTranslationMapPAE* map = static_cast<X86VMTranslationMapPAE*>(
                VMAddressSpace::Kernel()->TranslationMap());
        uint32 dummyFlags;
        cpu_status state = disable_interrupts();
        map->QueryInterrupt((addr_t)data, &physicalTable, &dummyFlags);
        restore_interrupts(state);

        // put the page table into the page directory
        pae_page_directory_entry* pageDirEntry
                = X86PagingMethodPAE::PageDirEntryForAddress(
                        map->PagingStructuresPAE()->VirtualPageDirs(), (addr_t)virtualBase);
        PutPageTableInPageDir(pageDirEntry, physicalTable,
                B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);

        // init the pool structure
        pool->Init(dataArea, (pae_page_table_entry*)data, virtualArea,
                (addr_t)virtualBase);
        poolDeleter.Detach();
        _pool = pool;
        return B_OK;
}


// #pragma mark - X86PagingMethodPAE


X86PagingMethodPAE::X86PagingMethodPAE()
        :
        fPhysicalPageMapper(NULL),
        fKernelPhysicalPageMapper(NULL),
        fFreePages(NULL),
        fFreePagesCount(0)
{
        mutex_init(&fFreePagesLock, "x86 PAE free pages");
}


X86PagingMethodPAE::~X86PagingMethodPAE()
{
}


status_t
X86PagingMethodPAE::Init(kernel_args* args,
        VMPhysicalPageMapper** _physicalPageMapper)
{
        // Ignore all memory beyond the maximum PAE address.
        static const phys_addr_t kLimit = 1ULL << 36;
        for (uint32 i = 0; i < args->num_physical_memory_ranges; i++) {
                addr_range& range = args->physical_memory_range[i];
                if (range.start >= kLimit)
                        range.size = 0;
                else if ((range.start + range.size) > kLimit)
                        range.size = kLimit - range.start;
        }

        // switch to PAE
        ToPAESwitcher(args).Switch(fKernelVirtualPageDirPointerTable,
                fKernelPhysicalPageDirPointerTable, fEarlyPageStructures,
                fEarlyPageStructuresSize, fKernelVirtualPageDirs,
                fKernelPhysicalPageDirs, fFreeVirtualSlot, fFreeVirtualSlotPTE);

        // create the initial pools for the physical page mapper
        int32 poolCount = _GetInitialPoolCount();
        PhysicalPageSlotPool* pool = PhysicalPageSlotPool::sInitialPhysicalPagePool;

        for (int32 i = 0; i < poolCount; i++) {
                new(&pool[i]) PhysicalPageSlotPool;
                status_t error = pool[i].InitInitial(this, args);
                if (error != B_OK) {
                        panic("X86PagingMethodPAE::Init(): Failed to create initial pool "
                                "for physical page mapper!");
                        return error;
                }
        }

        // create physical page mapper
        large_memory_physical_page_ops_init(args, pool, poolCount, sizeof(*pool),
                fPhysicalPageMapper, fKernelPhysicalPageMapper);

        *_physicalPageMapper = fPhysicalPageMapper;
        return B_OK;
}


status_t
X86PagingMethodPAE::InitPostArea(kernel_args* args)
{
        // wrap the kernel paging structures in an area
        area_id area = create_area("kernel paging structs", &fEarlyPageStructures,
                B_EXACT_ADDRESS, fEarlyPageStructuresSize, B_ALREADY_WIRED,
                B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
        if (area < B_OK)
                return area;

        // let the initial page pools create areas for its structures
        int32 poolCount = _GetInitialPoolCount();
        for (int32 i = 0; i < poolCount; i++) {
                status_t error = PhysicalPageSlotPool::sInitialPhysicalPagePool[i]
                        .InitInitialPostArea(args);
                if (error != B_OK)
                        return error;
        }

        // The early physical page mapping mechanism is no longer needed. Unmap the
        // slot.
        *fFreeVirtualSlotPTE = 0;
        invalidate_TLB(fFreeVirtualSlot);

        fFreeVirtualSlotPTE = NULL;
        fFreeVirtualSlot = 0;

        return B_OK;
}


status_t
X86PagingMethodPAE::CreateTranslationMap(bool kernel, VMTranslationMap** _map)
{
        X86VMTranslationMapPAE* map = new(std::nothrow) X86VMTranslationMapPAE;
        if (map == NULL)
                return B_NO_MEMORY;

        status_t error = map->Init(kernel);
        if (error != B_OK) {
                delete map;
                return error;
        }

        *_map = map;
        return B_OK;
}


status_t
X86PagingMethodPAE::MapEarly(kernel_args* args, addr_t virtualAddress,
        phys_addr_t physicalAddress, uint8 attributes)
{
        // check to see if a page table exists for this range
        pae_page_directory_entry* pageDirEntry = PageDirEntryForAddress(
                fKernelVirtualPageDirs, virtualAddress);
        pae_page_table_entry* pageTable;
        if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) {
                // we need to allocate a page table
                phys_addr_t physicalPageTable = vm_allocate_early_physical_page(args) * B_PAGE_SIZE;

                TRACE("X86PagingMethodPAE::MapEarly(): asked for free page for "
                        "page table: %#" B_PRIxPHYSADDR "\n", physicalPageTable);

                // put it in the page dir
                PutPageTableInPageDir(pageDirEntry, physicalPageTable, attributes);

                // zero it out
                pageTable = _EarlyGetPageTable(physicalPageTable);
                memset(pageTable, 0, B_PAGE_SIZE);
        } else {
                // table already exists -- map it
                pageTable = _EarlyGetPageTable(
                        *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK);
        }

        pae_page_table_entry* entry = pageTable
                + virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount;

        ASSERT_PRINT(
                (*entry & X86_PAE_PTE_PRESENT) == 0,
                "virtual address: %#" B_PRIxADDR ", pde: %#" B_PRIx64
                ", existing pte: %#" B_PRIx64, virtualAddress, *pageDirEntry, *entry);

        // now, fill in the pentry
        PutPageTableEntryInTable(entry, physicalAddress, attributes, 0,
                IS_KERNEL_ADDRESS(virtualAddress));

        return B_OK;
}


bool
X86PagingMethodPAE::IsKernelPageAccessible(addr_t virtualAddress,
        uint32 protection)
{
        // we can't check much without the physical page mapper
        if (fPhysicalPageMapper == NULL)
                return false;

        // We only trust the kernel team's page directories. So switch to the
        // kernel PDPT first. Always set it to make sure the TLBs don't contain
        // obsolete data.
        uint32 physicalPDPT = x86_read_cr3();
        x86_write_cr3(fKernelPhysicalPageDirPointerTable);

        // get the PDPT entry for the address
        pae_page_directory_pointer_table_entry pdptEntry = 0;
        if (physicalPDPT == fKernelPhysicalPageDirPointerTable) {
                pdptEntry = fKernelVirtualPageDirPointerTable[
                        virtualAddress / kPAEPageDirRange];
        } else {
                // map the original PDPT and get the entry
                void* handle;
                addr_t virtualPDPT;
                status_t error = fPhysicalPageMapper->GetPageDebug(physicalPDPT,
                        &virtualPDPT, &handle);
                if (error == B_OK) {
                        pdptEntry = ((pae_page_directory_pointer_table_entry*)
                                virtualPDPT)[virtualAddress / kPAEPageDirRange];
                        fPhysicalPageMapper->PutPageDebug(virtualPDPT, handle);
                }
        }

        // map the page dir and get the entry
        pae_page_directory_entry pageDirEntry = 0;
        if ((pdptEntry & X86_PAE_PDPTE_PRESENT) != 0) {
                void* handle;
                addr_t virtualPageDir;
                status_t error = fPhysicalPageMapper->GetPageDebug(
                        pdptEntry & X86_PAE_PDPTE_ADDRESS_MASK, &virtualPageDir, &handle);
                if (error == B_OK) {
                        pageDirEntry = ((pae_page_directory_entry*)virtualPageDir)[
                                virtualAddress / kPAEPageTableRange % kPAEPageDirEntryCount];
                        fPhysicalPageMapper->PutPageDebug(virtualPageDir, handle);
                }
        }

        // map the page table and get the entry
        pae_page_table_entry pageTableEntry = 0;
        if ((pageDirEntry & X86_PAE_PDE_PRESENT) != 0) {
                void* handle;
                addr_t virtualPageTable;
                status_t error = fPhysicalPageMapper->GetPageDebug(
                        pageDirEntry & X86_PAE_PDE_ADDRESS_MASK, &virtualPageTable,
                        &handle);
                if (error == B_OK) {
                        pageTableEntry = ((pae_page_table_entry*)virtualPageTable)[
                                virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount];
                        fPhysicalPageMapper->PutPageDebug(virtualPageTable, handle);
                }
        }

        // switch back to the original page directory
        if (physicalPDPT != fKernelPhysicalPageDirPointerTable)
                x86_write_cr3(physicalPDPT);

        if ((pageTableEntry & X86_PAE_PTE_PRESENT) == 0)
                return false;

        // present means kernel-readable, so check for writable
        return (protection & B_KERNEL_WRITE_AREA) == 0
                || (pageTableEntry & X86_PAE_PTE_WRITABLE) != 0;
}


/*static*/ void
X86PagingMethodPAE::PutPageTableInPageDir(pae_page_directory_entry* entry,
        phys_addr_t physicalTable, uint32 attributes)
{
        SetTableEntry(entry, (physicalTable & X86_PAE_PDE_ADDRESS_MASK)
                | X86_PAE_PDE_PRESENT
                | X86_PAE_PDE_WRITABLE
                | X86_PAE_PDE_USER);
                // TODO: We ignore the attributes of the page table -- for compatibility
                // with BeOS we allow having user accessible areas in the kernel address
                // space. This is currently being used by some drivers, mainly for the
                // frame buffer. Our current real time data implementation makes use of
                // this fact, too.
                // We might want to get rid of this possibility one day, especially if
                // we intend to port it to a platform that does not support this.
}


/*static*/ void
X86PagingMethodPAE::PutPageTableEntryInTable(pae_page_table_entry* entry,
        phys_addr_t physicalAddress, uint32 attributes, uint32 memoryType,
        bool globalPage)
{
        pae_page_table_entry page = (physicalAddress & X86_PAE_PTE_ADDRESS_MASK)
                | X86_PAE_PTE_PRESENT | (globalPage ? X86_PAE_PTE_GLOBAL : 0)
                | MemoryTypeToPageTableEntryFlags(memoryType);

        // if the page is user accessible, it's automatically
        // accessible in kernel space, too (but with the same
        // protection)
        if ((attributes & B_USER_PROTECTION) != 0) {
                page |= X86_PAE_PTE_USER;
                if ((attributes & B_WRITE_AREA) != 0)
                        page |= X86_PAE_PTE_WRITABLE;
                if ((attributes & B_EXECUTE_AREA) == 0
                        && x86_check_feature(IA32_FEATURE_AMD_EXT_NX, FEATURE_EXT_AMD)) {
                        page |= X86_PAE_PTE_NOT_EXECUTABLE;
                }
        } else if ((attributes & B_KERNEL_WRITE_AREA) != 0)
                page |= X86_PAE_PTE_WRITABLE;

        // put it in the page table
        SetTableEntry(entry, page);
}


void*
X86PagingMethodPAE::Allocate32BitPage(phys_addr_t& _physicalAddress,
        void*& _handle)
{
        // get a free page
        MutexLocker locker(fFreePagesLock);
        vm_page* page;
        if (fFreePages != NULL) {
                page = fFreePages;
                fFreePages = page->cache_next;
                fFreePagesCount--;
                locker.Unlock();
        } else {
                // no pages -- allocate one
                locker.Unlock();

                physical_address_restrictions restrictions = {};
                restrictions.high_address = 0x100000000LL;
                page = vm_page_allocate_page_run(PAGE_STATE_UNUSED, 1, &restrictions,
                        VM_PRIORITY_SYSTEM);
                if (page == NULL)
                        return NULL;

                DEBUG_PAGE_ACCESS_END(page);
        }

        // map the page
        phys_addr_t physicalAddress
                = (phys_addr_t)page->physical_page_number * B_PAGE_SIZE;
        addr_t virtualAddress;
        if (fPhysicalPageMapper->GetPage(physicalAddress, &virtualAddress, &_handle)
                        != B_OK) {
                // mapping failed -- free page
                locker.Lock();
                page->cache_next = fFreePages;
                fFreePages = page;
                fFreePagesCount++;
                return NULL;
        }

        _physicalAddress = physicalAddress;
        return (void*)virtualAddress;
}


void
X86PagingMethodPAE::Free32BitPage(void* address, phys_addr_t physicalAddress,
        void* handle)
{
        // unmap the page
        fPhysicalPageMapper->PutPage((addr_t)address, handle);

        // free it
        vm_page* page = vm_lookup_page(physicalAddress / B_PAGE_SIZE);
        MutexLocker locker(fFreePagesLock);
        if (fFreePagesCount < kMaxFree32BitPagesCount) {
                // cache not full yet -- cache it
                page->cache_next = fFreePages;
                fFreePages = page;
                fFreePagesCount++;
        } else {
                // cache full -- free it
                locker.Unlock();
                DEBUG_PAGE_ACCESS_START(page);
                vm_page_free(NULL, page);
        }
}


inline int32
X86PagingMethodPAE::_GetInitialPoolCount()
{
        int32 requiredSlots = smp_get_num_cpus() * TOTAL_SLOTS_PER_CPU
                        + EXTRA_SLOTS;
        return (requiredSlots + kPAEPageTableEntryCount - 1)
                / kPAEPageTableEntryCount;
}


bool
X86PagingMethodPAE::_EarlyQuery(addr_t virtualAddress,
        phys_addr_t* _physicalAddress)
{
        pae_page_directory_entry* pageDirEntry = PageDirEntryForAddress(
                fKernelVirtualPageDirs, virtualAddress);
        if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) {
                // no pagetable here
                return false;
        }

        pae_page_table_entry* entry = _EarlyGetPageTable(
                        *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK)
                + virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount;
        if ((*entry & X86_PAE_PTE_PRESENT) == 0) {
                // page mapping not valid
                return false;
        }

        *_physicalAddress = *entry & X86_PAE_PTE_ADDRESS_MASK;
        return true;
}


pae_page_table_entry*
X86PagingMethodPAE::_EarlyGetPageTable(phys_addr_t address)
{
        *fFreeVirtualSlotPTE = (address & X86_PAE_PTE_ADDRESS_MASK)
                | X86_PAE_PTE_PRESENT | X86_PAE_PTE_WRITABLE | X86_PAE_PTE_GLOBAL;

        invalidate_TLB(fFreeVirtualSlot);

        return (pae_page_table_entry*)fFreeVirtualSlot;
}


#endif  // B_HAIKU_PHYSICAL_BITS == 64