#include <string.h>
#include <stdlib.h>
#include <algorithm>
#include <KernelExport.h>
#include <OS.h>
#include <AutoDeleter.h>
#include <BinarySemaphore.h>
#include <arch/cpu.h>
#include <arch/vm_translation_map.h>
#include <block_cache.h>
#include <boot/kernel_args.h>
#include <condition_variable.h>
#include <elf.h>
#include <heap.h>
#include <kernel.h>
#include <low_resource_manager.h>
#include <thread.h>
#include <tracing.h>
#include <util/AutoLock.h>
#include <vm/vm.h>
#include <vm/vm_priv.h>
#include <vm/vm_page.h>
#include <vm/VMAddressSpace.h>
#include <vm/VMArea.h>
#include <vm/VMCache.h>
#include "VMAnonymousCache.h"
#include "VMPageQueue.h"
#include "ModifiedPageQueue.h"
#ifdef TRACE_VM_PAGE
# define TRACE(x) dprintf x
#else
# define TRACE(x) ;
#endif
#ifdef TRACE_VM_DAEMONS
#define TRACE_DAEMON(x...) dprintf(x)
#else
#define TRACE_DAEMON(x...) do {} while (false)
#endif
#define PAGE_ASSERT(page, condition) \
ASSERT_PRINT((condition), "page: %p", (page))
#define SCRUB_SIZE 32
static const size_t kPageReserveForPriority[] = {
VM_PAGE_RESERVE_USER,
VM_PAGE_RESERVE_SYSTEM,
0
};
static uint32 sFreePagesTarget;
static uint32 sFreeOrCachedPagesTarget;
static uint32 sInactivePagesTarget;
static const bigtime_t kIdleScanWaitInterval = 1000000LL;
static const bigtime_t kBusyScanWaitInterval = 500000LL;
static const uint32 kIdleRunsForFullQueue = 20;
static const int32 kPageUsageMax = 64;
static const int32 kPageUsageAdvance = 3;
static const int32 kPageUsageDecline = 1;
int32 gMappedPagesCount;
static VMPageQueue sFreePageQueue;
static VMPageQueue sClearPageQueue;
static ModifiedPageQueue sDefaultModifiedPageQueue;
static VMPageQueue sInactivePageQueue;
static VMPageQueue sActivePageQueue;
static VMPageQueue sCachedPageQueue;
static vm_page *sPages;
static page_num_t sPhysicalPageOffset;
static page_num_t sNumPages;
static page_num_t sNonExistingPages;
static uint64 sIgnoredPages;
static int32 sUnreservedFreePages;
static int32 sUnsatisfiedPageReservations;
static int32 sModifiedTemporaryPages;
static ConditionVariable sFreePageCondition;
static mutex sPageDeficitLock = MUTEX_INITIALIZER("page deficit");
static rw_lock sFreePageQueuesLock
= RW_LOCK_INITIALIZER("free/clear page queues");
#ifdef TRACK_PAGE_USAGE_STATS
static page_num_t sPageUsageArrays[512];
static page_num_t* sPageUsage = sPageUsageArrays;
static page_num_t sPageUsagePageCount;
static page_num_t* sNextPageUsage = sPageUsageArrays + 256;
static page_num_t sNextPageUsagePageCount;
#endif
#if VM_PAGE_ALLOCATION_TRACKING_AVAILABLE
struct caller_info {
addr_t caller;
size_t count;
};
static const int32 kCallerInfoTableSize = 1024;
static caller_info sCallerInfoTable[kCallerInfoTableSize];
static int32 sCallerInfoCount = 0;
static caller_info* get_caller_info(addr_t caller);
RANGE_MARKER_FUNCTION_PROTOTYPES(vm_page)
static const addr_t kVMPageCodeAddressRange[] = {
RANGE_MARKER_FUNCTION_ADDRESS_RANGE(vm_page)
};
#endif
RANGE_MARKER_FUNCTION_BEGIN(vm_page)
struct page_stats {
int32 totalFreePages;
int32 unsatisfiedReservations;
int32 cachedPages;
};
struct PageReservationWaiter
: public DoublyLinkedListLinkImpl<PageReservationWaiter> {
Thread* thread;
uint32 dontTouch;
uint32 requested;
uint32 reserved;
bool operator<(const PageReservationWaiter& other) const
{
if (dontTouch != other.dontTouch)
return dontTouch < other.dontTouch;
return thread->priority > other.thread->priority;
}
};
typedef DoublyLinkedList<PageReservationWaiter> PageReservationWaiterList;
static PageReservationWaiterList sPageReservationWaiters;
static BinarySemaphore sPageDaemonCondition;
#if PAGE_ALLOCATION_TRACING
namespace PageAllocationTracing {
class ReservePages : public AbstractTraceEntry {
public:
ReservePages(uint32 count)
:
fCount(count)
{
Initialized();
}
virtual void AddDump(TraceOutput& out)
{
out.Print("page reserve: %" B_PRIu32, fCount);
}
private:
uint32 fCount;
};
class UnreservePages : public AbstractTraceEntry {
public:
UnreservePages(uint32 count)
:
fCount(count)
{
Initialized();
}
virtual void AddDump(TraceOutput& out)
{
out.Print("page unreserve: %" B_PRId32, fCount);
}
private:
uint32 fCount;
};
class AllocatePage
: public TRACE_ENTRY_SELECTOR(PAGE_ALLOCATION_TRACING_STACK_TRACE) {
public:
AllocatePage(page_num_t pageNumber)
:
TraceEntryBase(PAGE_ALLOCATION_TRACING_STACK_TRACE, 0, true),
fPageNumber(pageNumber)
{
Initialized();
}
virtual void AddDump(TraceOutput& out)
{
out.Print("page alloc: %#" B_PRIxPHYSADDR, fPageNumber);
}
private:
page_num_t fPageNumber;
};
class AllocatePageRun
: public TRACE_ENTRY_SELECTOR(PAGE_ALLOCATION_TRACING_STACK_TRACE) {
public:
AllocatePageRun(page_num_t startPage, uint32 length)
:
TraceEntryBase(PAGE_ALLOCATION_TRACING_STACK_TRACE, 0, true),
fStartPage(startPage),
fLength(length)
{
Initialized();
}
virtual void AddDump(TraceOutput& out)
{
out.Print("page alloc run: start %#" B_PRIxPHYSADDR " length: %"
B_PRIu32, fStartPage, fLength);
}
private:
page_num_t fStartPage;
uint32 fLength;
};
class FreePage
: public TRACE_ENTRY_SELECTOR(PAGE_ALLOCATION_TRACING_STACK_TRACE) {
public:
FreePage(page_num_t pageNumber)
:
TraceEntryBase(PAGE_ALLOCATION_TRACING_STACK_TRACE, 0, true),
fPageNumber(pageNumber)
{
Initialized();
}
virtual void AddDump(TraceOutput& out)
{
out.Print("page free: %#" B_PRIxPHYSADDR, fPageNumber);
}
private:
page_num_t fPageNumber;
};
class ScrubbingPages : public AbstractTraceEntry {
public:
ScrubbingPages(uint32 count)
:
fCount(count)
{
Initialized();
}
virtual void AddDump(TraceOutput& out)
{
out.Print("page scrubbing: %" B_PRId32, fCount);
}
private:
uint32 fCount;
};
class ScrubbedPages : public AbstractTraceEntry {
public:
ScrubbedPages(uint32 count)
:
fCount(count)
{
Initialized();
}
virtual void AddDump(TraceOutput& out)
{
out.Print("page scrubbed: %" B_PRId32, fCount);
}
private:
uint32 fCount;
};
class StolenPage : public AbstractTraceEntry {
public:
StolenPage()
{
Initialized();
}
virtual void AddDump(TraceOutput& out)
{
out.Print("page stolen");
}
};
}
# define TA(x) new(std::nothrow) PageAllocationTracing::x
#else
# define TA(x)
#endif
#if PAGE_DAEMON_TRACING
namespace PageDaemonTracing {
class ActivatePage : public AbstractTraceEntry {
public:
ActivatePage(vm_page* page)
:
fCache(page->cache),
fPage(page)
{
Initialized();
}
virtual void AddDump(TraceOutput& out)
{
out.Print("page activated: %p, cache: %p", fPage, fCache);
}
private:
VMCache* fCache;
vm_page* fPage;
};
class DeactivatePage : public AbstractTraceEntry {
public:
DeactivatePage(vm_page* page)
:
fCache(page->cache),
fPage(page)
{
Initialized();
}
virtual void AddDump(TraceOutput& out)
{
out.Print("page deactivated: %p, cache: %p", fPage, fCache);
}
private:
VMCache* fCache;
vm_page* fPage;
};
class FreedPageSwap : public AbstractTraceEntry {
public:
FreedPageSwap(vm_page* page)
:
fCache(page->cache),
fPage(page)
{
Initialized();
}
virtual void AddDump(TraceOutput& out)
{
out.Print("page swap freed: %p, cache: %p", fPage, fCache);
}
private:
VMCache* fCache;
vm_page* fPage;
};
}
# define TD(x) new(std::nothrow) PageDaemonTracing::x
#else
# define TD(x)
#endif
#if PAGE_STATE_TRACING
namespace PageStateTracing {
class SetPageState : public AbstractTraceEntry {
public:
SetPageState(vm_page* page, uint8 newState)
:
fPage(page),
fOldState(page->State()),
fNewState(newState),
fBusy(page->busy),
fWired(page->WiredCount() > 0),
fMapped(!page->mappings.IsEmpty()),
fAccessed(page->accessed),
fModified(page->modified)
{
#if PAGE_STATE_TRACING_STACK_TRACE
fStackTrace = capture_tracing_stack_trace(
PAGE_STATE_TRACING_STACK_TRACE, 0, true);
#endif
Initialized();
}
#if PAGE_STATE_TRACING_STACK_TRACE
virtual void DumpStackTrace(TraceOutput& out)
{
out.PrintStackTrace(fStackTrace);
}
#endif
virtual void AddDump(TraceOutput& out)
{
out.Print("page set state: %p (%c%c%c%c%c): %s -> %s", fPage,
fBusy ? 'b' : '-',
fWired ? 'w' : '-',
fMapped ? 'm' : '-',
fAccessed ? 'a' : '-',
fModified ? 'm' : '-',
page_state_to_string(fOldState),
page_state_to_string(fNewState));
}
private:
vm_page* fPage;
#if PAGE_STATE_TRACING_STACK_TRACE
tracing_stack_trace* fStackTrace;
#endif
uint8 fOldState;
uint8 fNewState;
bool fBusy : 1;
bool fWired : 1;
bool fMapped : 1;
bool fAccessed : 1;
bool fModified : 1;
};
}
# define TPS(x) new(std::nothrow) PageStateTracing::x
#else
# define TPS(x)
#endif
#if VM_PAGE_ALLOCATION_TRACKING_AVAILABLE
namespace BKernel {
class AllocationTrackingCallback {
public:
virtual ~AllocationTrackingCallback();
virtual bool ProcessTrackingInfo(
AllocationTrackingInfo* info,
page_num_t pageNumber) = 0;
};
}
using BKernel::AllocationTrackingCallback;
class AllocationCollectorCallback : public AllocationTrackingCallback {
public:
AllocationCollectorCallback(bool resetInfos)
:
fResetInfos(resetInfos)
{
}
virtual bool ProcessTrackingInfo(AllocationTrackingInfo* info,
page_num_t pageNumber)
{
if (!info->IsInitialized())
return true;
addr_t caller = 0;
AbstractTraceEntryWithStackTrace* traceEntry = info->TraceEntry();
if (traceEntry != NULL && info->IsTraceEntryValid()) {
caller = tracing_find_caller_in_stack_trace(
traceEntry->StackTrace(), kVMPageCodeAddressRange, 1);
}
caller_info* callerInfo = get_caller_info(caller);
if (callerInfo == NULL) {
kprintf("out of space for caller infos\n");
return false;
}
callerInfo->count++;
if (fResetInfos)
info->Clear();
return true;
}
private:
bool fResetInfos;
};
class AllocationInfoPrinterCallback : public AllocationTrackingCallback {
public:
AllocationInfoPrinterCallback(bool printStackTrace, page_num_t pageFilter,
team_id teamFilter, thread_id threadFilter)
:
fPrintStackTrace(printStackTrace),
fPageFilter(pageFilter),
fTeamFilter(teamFilter),
fThreadFilter(threadFilter)
{
}
virtual bool ProcessTrackingInfo(AllocationTrackingInfo* info,
page_num_t pageNumber)
{
if (!info->IsInitialized())
return true;
if (fPageFilter != 0 && pageNumber != fPageFilter)
return true;
AbstractTraceEntryWithStackTrace* traceEntry = info->TraceEntry();
if (traceEntry != NULL && !info->IsTraceEntryValid())
traceEntry = NULL;
if (traceEntry != NULL) {
if (fTeamFilter != -1 && traceEntry->TeamID() != fTeamFilter)
return true;
if (fThreadFilter != -1 && traceEntry->ThreadID() != fThreadFilter)
return true;
} else {
if (fTeamFilter != -1 || fThreadFilter != -1)
return true;
}
kprintf("page number %#" B_PRIxPHYSADDR, pageNumber);
if (traceEntry != NULL) {
kprintf(", team: %" B_PRId32 ", thread %" B_PRId32
", time %" B_PRId64 "\n", traceEntry->TeamID(),
traceEntry->ThreadID(), traceEntry->Time());
if (fPrintStackTrace)
tracing_print_stack_trace(traceEntry->StackTrace());
} else
kprintf("\n");
return true;
}
private:
bool fPrintStackTrace;
page_num_t fPageFilter;
team_id fTeamFilter;
thread_id fThreadFilter;
};
class AllocationDetailPrinterCallback : public AllocationTrackingCallback {
public:
AllocationDetailPrinterCallback(addr_t caller)
:
fCaller(caller)
{
}
virtual bool ProcessTrackingInfo(AllocationTrackingInfo* info,
page_num_t pageNumber)
{
if (!info->IsInitialized())
return true;
addr_t caller = 0;
AbstractTraceEntryWithStackTrace* traceEntry = info->TraceEntry();
if (traceEntry != NULL && !info->IsTraceEntryValid())
traceEntry = NULL;
if (traceEntry != NULL) {
caller = tracing_find_caller_in_stack_trace(
traceEntry->StackTrace(), kVMPageCodeAddressRange, 1);
}
if (caller != fCaller)
return true;
kprintf("page %#" B_PRIxPHYSADDR "\n", pageNumber);
if (traceEntry != NULL)
tracing_print_stack_trace(traceEntry->StackTrace());
return true;
}
private:
addr_t fCaller;
};
#endif
static void
list_page(vm_page* page)
{
kprintf("0x%08" B_PRIxADDR " ",
(addr_t)(page->physical_page_number * B_PAGE_SIZE));
switch (page->State()) {
case PAGE_STATE_ACTIVE: kprintf("A"); break;
case PAGE_STATE_INACTIVE: kprintf("I"); break;
case PAGE_STATE_MODIFIED: kprintf("M"); break;
case PAGE_STATE_CACHED: kprintf("C"); break;
case PAGE_STATE_FREE: kprintf("F"); break;
case PAGE_STATE_CLEAR: kprintf("L"); break;
case PAGE_STATE_WIRED: kprintf("W"); break;
case PAGE_STATE_UNUSED: kprintf("-"); break;
}
kprintf(" ");
if (page->busy) kprintf("B"); else kprintf("-");
if (page->busy_io) kprintf("I"); else kprintf("-");
if (page->accessed) kprintf("A"); else kprintf("-");
if (page->modified) kprintf("M"); else kprintf("-");
kprintf("-");
kprintf(" usage:%3u", page->usage_count);
kprintf(" wired:%5u", page->WiredCount());
bool first = true;
vm_page_mappings::Iterator iterator = page->mappings.GetIterator();
vm_page_mapping* mapping;
while ((mapping = iterator.Next()) != NULL) {
if (first) {
kprintf(": ");
first = false;
} else
kprintf(", ");
kprintf("%" B_PRId32 " (%s)", mapping->area->id, mapping->area->name);
mapping = mapping->page_link.next;
}
}
static int
dump_page_list(int argc, char **argv)
{
kprintf("page table:\n");
for (page_num_t i = 0; i < sNumPages; i++) {
if (sPages[i].State() != PAGE_STATE_UNUSED) {
list_page(&sPages[i]);
kprintf("\n");
}
}
kprintf("end of page table\n");
return 0;
}
static int
find_page(int argc, char **argv)
{
addr_t address;
int32 index = 1;
if (argc < 2
|| strlen(argv[index]) <= 2
|| argv[index][0] != '0'
|| argv[index][1] != 'x') {
kprintf("usage: find_page <address>\n");
return 0;
}
address = strtoul(argv[index], NULL, 0);
struct vm_page *page = (vm_page*)address;
struct {
const char* name;
VMPageQueue* queue;
} pageQueueInfos[] = {
{ "free", &sFreePageQueue },
{ "clear", &sClearPageQueue },
{ "modified-default", &sDefaultModifiedPageQueue },
{ "modified-cache", page->Cache() != NULL ? page->Cache()->ModifiedQueue() : NULL },
{ "active", &sActivePageQueue },
{ "inactive", &sInactivePageQueue },
{ "cached", &sCachedPageQueue },
{ NULL, NULL }
};
for (int i = 0; pageQueueInfos[i].name; i++) {
if (pageQueueInfos[i].queue == NULL)
continue;
VMPageQueue::Iterator it = pageQueueInfos[i].queue->GetIterator();
while (vm_page* p = it.Next()) {
if (p == page) {
kprintf("found page %p in queue %p (%s)\n", page,
pageQueueInfos[i].queue, pageQueueInfos[i].name);
return 0;
}
}
}
kprintf("page %p isn't in any queue\n", page);
return 0;
}
const char *
page_state_to_string(int state)
{
switch(state) {
case PAGE_STATE_ACTIVE:
return "active";
case PAGE_STATE_INACTIVE:
return "inactive";
case PAGE_STATE_MODIFIED:
return "modified";
case PAGE_STATE_CACHED:
return "cached";
case PAGE_STATE_FREE:
return "free";
case PAGE_STATE_CLEAR:
return "clear";
case PAGE_STATE_WIRED:
return "wired";
case PAGE_STATE_UNUSED:
return "unused";
default:
return "unknown";
}
}
static int
dump_page_long(int argc, char **argv)
{
bool addressIsPointer = true;
bool physical = false;
bool searchMappings = false;
int32 index = 1;
while (index < argc) {
if (argv[index][0] != '-')
break;
if (!strcmp(argv[index], "-p")) {
addressIsPointer = false;
physical = true;
} else if (!strcmp(argv[index], "-v")) {
addressIsPointer = false;
} else if (!strcmp(argv[index], "-m")) {
searchMappings = true;
} else {
print_debugger_command_usage(argv[0]);
return 0;
}
index++;
}
if (index + 1 != argc) {
print_debugger_command_usage(argv[0]);
return 0;
}
uint64 value;
if (!evaluate_debug_expression(argv[index], &value, false))
return 0;
uint64 pageAddress = value;
struct vm_page* page;
if (addressIsPointer) {
page = (struct vm_page *)(addr_t)pageAddress;
} else {
if (!physical) {
VMAddressSpace *addressSpace = VMAddressSpace::Kernel();
if (debug_get_debugged_thread()->team->address_space != NULL)
addressSpace = debug_get_debugged_thread()->team->address_space;
uint32 flags = 0;
phys_addr_t physicalAddress;
if (addressSpace->TranslationMap()->QueryInterrupt(pageAddress,
&physicalAddress, &flags) != B_OK
|| (flags & PAGE_PRESENT) == 0) {
kprintf("Virtual address not mapped to a physical page in this "
"address space.\n");
return 0;
}
pageAddress = physicalAddress;
}
page = vm_lookup_page(pageAddress / B_PAGE_SIZE);
}
if (page == NULL) {
kprintf("Page not found.\n");
return 0;
}
kprintf("PAGE: %p\n", page);
const off_t pageOffset = (addr_t)page - (addr_t)sPages;
const off_t pageIndex = pageOffset / (off_t)sizeof(vm_page);
if (pageIndex < 0) {
kprintf("\taddress is before start of page array!"
" (offset %" B_PRIdOFF ")\n", pageOffset);
} else if ((page_num_t)pageIndex >= sNumPages) {
kprintf("\taddress is after end of page array!"
" (offset %" B_PRIdOFF ")\n", pageOffset);
} else if ((pageIndex * (off_t)sizeof(vm_page)) != pageOffset) {
kprintf("\taddress isn't a multiple of page structure size!"
" (offset %" B_PRIdOFF ", expected align %" B_PRIuSIZE ")\n",
pageOffset, sizeof(vm_page));
}
kprintf("queue_next,prev: %p, %p\n", page->queue_link.next,
page->queue_link.previous);
kprintf("physical_number: %#" B_PRIxPHYSADDR "\n", page->physical_page_number);
kprintf("cache: %p\n", page->Cache());
kprintf("cache_offset: %" B_PRIuPHYSADDR "\n", page->cache_offset);
kprintf("cache_next: %p\n", page->cache_next);
kprintf("state: %s\n", page_state_to_string(page->State()));
kprintf("wired_count: %d\n", page->WiredCount());
kprintf("usage_count: %d\n", page->usage_count);
kprintf("busy: %d\n", page->busy);
kprintf("busy_io: %d\n", page->busy_io);
kprintf("accessed: %d\n", page->accessed);
kprintf("modified: %d\n", page->modified);
#if DEBUG_PAGE_QUEUE
kprintf("queue: %p\n", page->queue);
#endif
#if DEBUG_PAGE_ACCESS
kprintf("accessor: %" B_PRId32 "\n", page->accessing_thread);
#endif
if (pageIndex < 0 || (page_num_t)pageIndex >= sNumPages) {
return 0;
}
kprintf("area mappings:\n");
vm_page_mappings::Iterator iterator = page->mappings.GetIterator();
vm_page_mapping *mapping;
while ((mapping = iterator.Next()) != NULL) {
kprintf(" %p (%" B_PRId32 ")\n", mapping->area, mapping->area->id);
mapping = mapping->page_link.next;
}
if (searchMappings) {
struct Callback : VMTranslationMap::ReverseMappingInfoCallback {
VMAddressSpace* fAddressSpace;
virtual bool HandleVirtualAddress(addr_t virtualAddress)
{
phys_addr_t physicalAddress;
uint32 flags = 0;
if (fAddressSpace->TranslationMap()->QueryInterrupt(virtualAddress,
&physicalAddress, &flags) != B_OK) {
kprintf(" aspace %" B_PRId32 ": %#" B_PRIxADDR " (querying failed)\n",
fAddressSpace->ID(), virtualAddress);
return false;
}
VMArea* area = fAddressSpace->LookupArea(virtualAddress);
kprintf(" aspace %" B_PRId32 ", area %" B_PRId32 ": %#"
B_PRIxADDR " (%c%c%s%s)\n", fAddressSpace->ID(),
area != NULL ? area->id : -1, virtualAddress,
(flags & B_KERNEL_READ_AREA) != 0 ? 'r' : '-',
(flags & B_KERNEL_WRITE_AREA) != 0 ? 'w' : '-',
(flags & PAGE_MODIFIED) != 0 ? " modified" : "",
(flags & PAGE_ACCESSED) != 0 ? " accessed" : "");
return false;
}
} callback;
kprintf("all mappings:\n");
VMAddressSpace* addressSpace = VMAddressSpace::DebugFirst();
while (addressSpace != NULL) {
callback.fAddressSpace = addressSpace;
addressSpace->TranslationMap()->DebugGetReverseMappingInfo(
page->physical_page_number * B_PAGE_SIZE, callback);
addressSpace = VMAddressSpace::DebugNext(addressSpace);
}
}
set_debug_variable("_cache", (addr_t)page->Cache());
#if DEBUG_PAGE_ACCESS
set_debug_variable("_accessor", page->accessing_thread);
#endif
return 0;
}
static int
dump_page_queue(int argc, char **argv)
{
struct VMPageQueue *queue;
if (argc < 2) {
kprintf("usage: page_queue <address/name> [list]\n");
return 0;
}
if (strlen(argv[1]) >= 2 && argv[1][0] == '0' && argv[1][1] == 'x')
queue = (VMPageQueue*)strtoul(argv[1], NULL, 16);
else if (!strcmp(argv[1], "free"))
queue = &sFreePageQueue;
else if (!strcmp(argv[1], "clear"))
queue = &sClearPageQueue;
else if (!strcmp(argv[1], "modified-default"))
queue = &sDefaultModifiedPageQueue;
else if (!strcmp(argv[1], "active"))
queue = &sActivePageQueue;
else if (!strcmp(argv[1], "inactive"))
queue = &sInactivePageQueue;
else if (!strcmp(argv[1], "cached"))
queue = &sCachedPageQueue;
else {
kprintf("page_queue: unknown queue \"%s\".\n", argv[1]);
return 0;
}
kprintf("queue = %p, queue->head = %p, queue->tail = %p, queue->count = %"
B_PRIuPHYSADDR "\n", queue, queue->Head(), queue->Tail(),
queue->Count());
if (argc == 3) {
struct vm_page *page = queue->Head();
kprintf("page cache type state wired usage\n");
for (page_num_t i = 0; page; i++, page = queue->Next(page)) {
kprintf("%p %p %-7s %8s %5d %5d\n", page, page->Cache(),
vm_cache_type_to_string(page->Cache()->type),
page_state_to_string(page->State()),
page->WiredCount(), page->usage_count);
}
}
return 0;
}
static int
dump_page_stats(int argc, char **argv)
{
page_num_t swappableModified = 0;
page_num_t swappableModifiedInactive = 0;
size_t counter[8];
size_t busyCounter[8];
memset(counter, 0, sizeof(counter));
memset(busyCounter, 0, sizeof(busyCounter));
struct page_run {
page_num_t start;
page_num_t end;
page_num_t Length() const { return end - start; }
};
page_run currentFreeRun = { 0, 0 };
page_run currentCachedRun = { 0, 0 };
page_run longestFreeRun = { 0, 0 };
page_run longestCachedRun = { 0, 0 };
for (page_num_t i = 0; i < sNumPages; i++) {
if (sPages[i].State() > 7) {
panic("page %" B_PRIuPHYSADDR " at %p has invalid state!\n", i,
&sPages[i]);
}
uint32 pageState = sPages[i].State();
counter[pageState]++;
if (sPages[i].busy)
busyCounter[pageState]++;
if (pageState == PAGE_STATE_MODIFIED
&& sPages[i].Cache() != NULL
&& sPages[i].Cache()->temporary && sPages[i].WiredCount() == 0) {
swappableModified++;
if (sPages[i].usage_count == 0)
swappableModifiedInactive++;
}
if (pageState == PAGE_STATE_FREE || pageState == PAGE_STATE_CLEAR) {
currentFreeRun.end = i + 1;
currentCachedRun.end = i + 1;
} else {
if (currentFreeRun.Length() > longestFreeRun.Length())
longestFreeRun = currentFreeRun;
currentFreeRun.start = currentFreeRun.end = i + 1;
if (pageState == PAGE_STATE_CACHED) {
currentCachedRun.end = i + 1;
} else {
if (currentCachedRun.Length() > longestCachedRun.Length())
longestCachedRun = currentCachedRun;
currentCachedRun.start = currentCachedRun.end = i + 1;
}
}
}
kprintf("page stats:\n");
kprintf("total: %" B_PRIuPHYSADDR "\n", sNumPages);
kprintf("active: %" B_PRIuSIZE " (busy: %" B_PRIuSIZE ")\n",
counter[PAGE_STATE_ACTIVE], busyCounter[PAGE_STATE_ACTIVE]);
kprintf("inactive: %" B_PRIuSIZE " (busy: %" B_PRIuSIZE ")\n",
counter[PAGE_STATE_INACTIVE], busyCounter[PAGE_STATE_INACTIVE]);
kprintf("cached: %" B_PRIuSIZE " (busy: %" B_PRIuSIZE ")\n",
counter[PAGE_STATE_CACHED], busyCounter[PAGE_STATE_CACHED]);
kprintf("unused: %" B_PRIuSIZE " (busy: %" B_PRIuSIZE ")\n",
counter[PAGE_STATE_UNUSED], busyCounter[PAGE_STATE_UNUSED]);
kprintf("wired: %" B_PRIuSIZE " (busy: %" B_PRIuSIZE ")\n",
counter[PAGE_STATE_WIRED], busyCounter[PAGE_STATE_WIRED]);
kprintf("modified: %" B_PRIuSIZE " (busy: %" B_PRIuSIZE ")\n",
counter[PAGE_STATE_MODIFIED], busyCounter[PAGE_STATE_MODIFIED]);
kprintf("free: %" B_PRIuSIZE "\n", counter[PAGE_STATE_FREE]);
kprintf("clear: %" B_PRIuSIZE "\n", counter[PAGE_STATE_CLEAR]);
kprintf("unreserved free pages: %" B_PRId32 "\n", sUnreservedFreePages);
kprintf("unsatisfied page reservations: %" B_PRId32 "\n",
sUnsatisfiedPageReservations);
kprintf("mapped pages: %" B_PRId32 "\n", gMappedPagesCount);
kprintf("longest free pages run: %" B_PRIuPHYSADDR " pages (at %"
B_PRIuPHYSADDR ")\n", longestFreeRun.Length(),
sPages[longestFreeRun.start].physical_page_number);
kprintf("longest free/cached pages run: %" B_PRIuPHYSADDR " pages (at %"
B_PRIuPHYSADDR ")\n", longestCachedRun.Length(),
sPages[longestCachedRun.start].physical_page_number);
kprintf("waiting threads:\n");
for (PageReservationWaiterList::Iterator it
= sPageReservationWaiters.GetIterator();
PageReservationWaiter* waiter = it.Next();) {
kprintf(" %6" B_PRId32 ": requested: %6" B_PRIu32 ", reserved: %6" B_PRIu32
", don't touch: %6" B_PRIu32 "\n", waiter->thread->id,
waiter->requested, waiter->reserved, waiter->dontTouch);
}
kprintf("\nfree queue: %p, count = %" B_PRIuPHYSADDR "\n", &sFreePageQueue,
sFreePageQueue.Count());
kprintf("clear queue: %p, count = %" B_PRIuPHYSADDR "\n", &sClearPageQueue,
sClearPageQueue.Count());
kprintf("modified-default queue: %p, count = %" B_PRIuPHYSADDR " (%" B_PRId32
" temporary, %" B_PRIuPHYSADDR " swappable, " "inactive: %"
B_PRIuPHYSADDR ")\n", &sDefaultModifiedPageQueue, sDefaultModifiedPageQueue.Count(),
sModifiedTemporaryPages, swappableModified, swappableModifiedInactive);
kprintf("active queue: %p, count = %" B_PRIuPHYSADDR "\n",
&sActivePageQueue, sActivePageQueue.Count());
kprintf("inactive queue: %p, count = %" B_PRIuPHYSADDR "\n",
&sInactivePageQueue, sInactivePageQueue.Count());
kprintf("cached queue: %p, count = %" B_PRIuPHYSADDR "\n",
&sCachedPageQueue, sCachedPageQueue.Count());
return 0;
}
#if VM_PAGE_ALLOCATION_TRACKING_AVAILABLE
static caller_info*
get_caller_info(addr_t caller)
{
for (int32 i = 0; i < sCallerInfoCount; i++) {
if (caller == sCallerInfoTable[i].caller)
return &sCallerInfoTable[i];
}
if (sCallerInfoCount >= kCallerInfoTableSize)
return NULL;
caller_info* info = &sCallerInfoTable[sCallerInfoCount++];
info->caller = caller;
info->count = 0;
return info;
}
static int
caller_info_compare_count(const void* _a, const void* _b)
{
const caller_info* a = (const caller_info*)_a;
const caller_info* b = (const caller_info*)_b;
return (int)(b->count - a->count);
}
static int
dump_page_allocations_per_caller(int argc, char** argv)
{
bool resetAllocationInfos = false;
bool printDetails = false;
addr_t caller = 0;
for (int32 i = 1; i < argc; i++) {
if (strcmp(argv[i], "-d") == 0) {
uint64 callerAddress;
if (++i >= argc
|| !evaluate_debug_expression(argv[i], &callerAddress, true)) {
print_debugger_command_usage(argv[0]);
return 0;
}
caller = callerAddress;
printDetails = true;
} else if (strcmp(argv[i], "-r") == 0) {
resetAllocationInfos = true;
} else {
print_debugger_command_usage(argv[0]);
return 0;
}
}
sCallerInfoCount = 0;
AllocationCollectorCallback collectorCallback(resetAllocationInfos);
AllocationDetailPrinterCallback detailsCallback(caller);
AllocationTrackingCallback& callback = printDetails
? (AllocationTrackingCallback&)detailsCallback
: (AllocationTrackingCallback&)collectorCallback;
for (page_num_t i = 0; i < sNumPages; i++)
callback.ProcessTrackingInfo(&sPages[i].allocation_tracking_info, i);
if (printDetails)
return 0;
qsort(sCallerInfoTable, sCallerInfoCount, sizeof(caller_info),
&caller_info_compare_count);
kprintf("%" B_PRId32 " different callers\n\n", sCallerInfoCount);
size_t totalAllocationCount = 0;
kprintf(" count caller\n");
kprintf("----------------------------------\n");
for (int32 i = 0; i < sCallerInfoCount; i++) {
caller_info& info = sCallerInfoTable[i];
kprintf("%10" B_PRIuSIZE " %p", info.count, (void*)info.caller);
const char* symbol;
const char* imageName;
bool exactMatch;
addr_t baseAddress;
if (elf_debug_lookup_symbol_address(info.caller, &baseAddress, &symbol,
&imageName, &exactMatch) == B_OK) {
kprintf(" %s + %#" B_PRIxADDR " (%s)%s\n", symbol,
info.caller - baseAddress, imageName,
exactMatch ? "" : " (nearest)");
} else
kprintf("\n");
totalAllocationCount += info.count;
}
kprintf("\ntotal page allocations: %" B_PRIuSIZE "\n",
totalAllocationCount);
return 0;
}
static int
dump_page_allocation_infos(int argc, char** argv)
{
page_num_t pageFilter = 0;
team_id teamFilter = -1;
thread_id threadFilter = -1;
bool printStackTraces = false;
for (int32 i = 1; i < argc; i++) {
if (strcmp(argv[i], "--stacktrace") == 0)
printStackTraces = true;
else if (strcmp(argv[i], "-p") == 0) {
uint64 pageNumber;
if (++i >= argc
|| !evaluate_debug_expression(argv[i], &pageNumber, true)) {
print_debugger_command_usage(argv[0]);
return 0;
}
pageFilter = pageNumber;
} else if (strcmp(argv[i], "--team") == 0) {
uint64 team;
if (++i >= argc
|| !evaluate_debug_expression(argv[i], &team, true)) {
print_debugger_command_usage(argv[0]);
return 0;
}
teamFilter = team;
} else if (strcmp(argv[i], "--thread") == 0) {
uint64 thread;
if (++i >= argc
|| !evaluate_debug_expression(argv[i], &thread, true)) {
print_debugger_command_usage(argv[0]);
return 0;
}
threadFilter = thread;
} else {
print_debugger_command_usage(argv[0]);
return 0;
}
}
AllocationInfoPrinterCallback callback(printStackTraces, pageFilter,
teamFilter, threadFilter);
for (page_num_t i = 0; i < sNumPages; i++)
callback.ProcessTrackingInfo(&sPages[i].allocation_tracking_info, i);
return 0;
}
#endif
#ifdef TRACK_PAGE_USAGE_STATS
static void
track_page_usage(vm_page* page)
{
if (page->WiredCount() == 0) {
sNextPageUsage[(int32)page->usage_count + 128]++;
sNextPageUsagePageCount++;
}
}
static void
update_page_usage_stats()
{
std::swap(sPageUsage, sNextPageUsage);
sPageUsagePageCount = sNextPageUsagePageCount;
memset(sNextPageUsage, 0, sizeof(page_num_t) * 256);
sNextPageUsagePageCount = 0;
if (sPageUsagePageCount > 0) {
int64 sum = 0;
for (int32 i = 0; i < 256; i++)
sum += (int64)sPageUsage[i] * (i - 128);
TRACE_DAEMON("average page usage: %f (%lu pages)\n",
(float)sum / sPageUsagePageCount, sPageUsagePageCount);
}
}
static int
dump_page_usage_stats(int argc, char** argv)
{
kprintf("distribution of page usage counts (%lu pages):",
sPageUsagePageCount);
int64 sum = 0;
for (int32 i = 0; i < 256; i++) {
if (i % 8 == 0)
kprintf("\n%4ld:", i - 128);
int64 count = sPageUsage[i];
sum += count * (i - 128);
kprintf(" %9llu", count);
}
kprintf("\n\n");
kprintf("average usage count: %f\n",
sPageUsagePageCount > 0 ? (float)sum / sPageUsagePageCount : 0);
return 0;
}
#endif
inline void
vm_page::InitState(uint8 newState)
{
state = newState;
}
inline void
vm_page::SetState(uint8 newState)
{
TPS(SetPageState(this, newState));
state = newState;
}
static void
get_page_stats(page_stats& _pageStats)
{
_pageStats.totalFreePages = sUnreservedFreePages;
_pageStats.cachedPages = sCachedPageQueue.Count();
_pageStats.unsatisfiedReservations = sUnsatisfiedPageReservations;
}
static bool
do_active_paging(const page_stats& pageStats)
{
return pageStats.totalFreePages + pageStats.cachedPages
< pageStats.unsatisfiedReservations
+ (int32)sFreeOrCachedPagesTarget;
}
bool
vm_page_should_do_active_paging()
{
page_stats pageStats;
get_page_stats(pageStats);
return do_active_paging(pageStats);
}
static uint32
reserve_some_pages(uint32 count, uint32 dontTouch)
{
while (true) {
int32 freePages = atomic_get(&sUnreservedFreePages);
if (freePages <= (int32)dontTouch)
return 0;
int32 toReserve = std::min(count, freePages - dontTouch);
if (atomic_test_and_set(&sUnreservedFreePages,
freePages - toReserve, freePages)
== freePages) {
return toReserve;
}
}
}
static void
wake_up_page_reservation_waiters()
{
ASSERT_LOCKED_MUTEX(&sPageDeficitLock);
while (PageReservationWaiter* waiter = sPageReservationWaiters.Head()) {
int32 reserved = reserve_some_pages(waiter->requested - waiter->reserved,
waiter->dontTouch);
if (reserved == 0)
return;
sUnsatisfiedPageReservations -= reserved;
waiter->reserved += reserved;
if (waiter->reserved != waiter->requested)
return;
sPageReservationWaiters.Remove(waiter);
thread_unblock(waiter->thread, B_OK);
}
}
static inline void
unreserve_pages(uint32 count)
{
atomic_add(&sUnreservedFreePages, count);
if (atomic_get(&sUnsatisfiedPageReservations) != 0) {
MutexLocker pageDeficitLocker(sPageDeficitLock);
wake_up_page_reservation_waiters();
}
}
ModifiedPageQueue*
vm_page_default_modified_queue()
{
return &sDefaultModifiedPageQueue;
}
static VMPageQueue*
page_queue_for(vm_page* page, uint8 state)
{
switch (state) {
case PAGE_STATE_ACTIVE:
return &sActivePageQueue;
case PAGE_STATE_INACTIVE:
return &sInactivePageQueue;
case PAGE_STATE_MODIFIED:
{
VMPageQueue* queue = page->Cache()->ModifiedQueue();
if (queue != NULL)
return queue;
return &sDefaultModifiedPageQueue;
}
case PAGE_STATE_CACHED:
return &sCachedPageQueue;
case PAGE_STATE_FREE:
return &sFreePageQueue;
case PAGE_STATE_CLEAR:
return &sClearPageQueue;
case PAGE_STATE_WIRED:
case PAGE_STATE_UNUSED:
return NULL;
default:
panic("page %p in invalid state %d", page, page->State());
return NULL;
}
}
static void
free_page(vm_page* page, bool clear)
{
DEBUG_PAGE_ACCESS_CHECK(page);
PAGE_ASSERT(page, !page->IsMapped());
if (page->State() == PAGE_STATE_FREE || page->State() == PAGE_STATE_CLEAR) {
panic("free_page(): page %p already free", page);
return;
}
VMPageQueue* fromQueue = page_queue_for(page, page->State());
if (page->CacheRef() != NULL)
panic("to be freed page %p has cache", page);
if (page->IsMapped())
panic("to be freed page %p has mappings", page);
if (fromQueue != NULL)
fromQueue->RemoveUnlocked(page);
TA(FreePage(page->physical_page_number));
#if VM_PAGE_ALLOCATION_TRACKING_AVAILABLE
page->allocation_tracking_info.Clear();
#endif
ReadLocker locker(sFreePageQueuesLock);
DEBUG_PAGE_ACCESS_END(page);
if (clear) {
page->SetState(PAGE_STATE_CLEAR);
sClearPageQueue.PrependUnlocked(page);
} else {
page->SetState(PAGE_STATE_FREE);
sFreePageQueue.PrependUnlocked(page);
sFreePageCondition.NotifyAll();
}
locker.Unlock();
}
static void
set_page_state(vm_page *page, int pageState)
{
DEBUG_PAGE_ACCESS_CHECK(page);
if (pageState == page->State())
return;
if (page->State() == PAGE_STATE_FREE || page->State() == PAGE_STATE_CLEAR) {
panic("set_page_state(): page %p is free/clear", page);
return;
}
if (pageState == PAGE_STATE_FREE || pageState == PAGE_STATE_CLEAR) {
panic("set_page_state(): target state is free/clear");
return;
}
if (pageState == PAGE_STATE_CACHED) {
PAGE_ASSERT(page, !page->IsMapped());
PAGE_ASSERT(page, !page->modified);
}
VMPageQueue* fromQueue = page_queue_for(page, page->State());
VMPageQueue* toQueue = page_queue_for(page, pageState);
VMCache* cache = page->Cache();
if (cache != NULL && cache->temporary) {
if (pageState == PAGE_STATE_MODIFIED)
atomic_add(&sModifiedTemporaryPages, 1);
else if (page->State() == PAGE_STATE_MODIFIED)
atomic_add(&sModifiedTemporaryPages, -1);
}
if (toQueue == fromQueue) {
PAGE_ASSERT(page, cache != NULL);
cache->AssertLocked();
page->SetState(pageState);
} else {
if (fromQueue != NULL)
fromQueue->RemoveUnlocked(page);
page->SetState(pageState);
if (toQueue != NULL)
toQueue->AppendUnlocked(page);
}
}
static void
clear_page(struct vm_page *page)
{
vm_memset_physical(page->physical_page_number << PAGE_SHIFT, 0,
B_PAGE_SIZE);
}
static status_t
mark_page_range_in_use(page_num_t startPage, page_num_t length, bool wired)
{
TRACE(("mark_page_range_in_use: start %#" B_PRIxPHYSADDR ", len %#"
B_PRIxPHYSADDR "\n", startPage, length));
if (sPhysicalPageOffset > startPage) {
dprintf("mark_page_range_in_use(%#" B_PRIxPHYSADDR ", %#" B_PRIxPHYSADDR
"): start page is before free list\n", startPage, length);
if (sPhysicalPageOffset - startPage >= length)
return B_OK;
length -= sPhysicalPageOffset - startPage;
startPage = sPhysicalPageOffset;
}
startPage -= sPhysicalPageOffset;
if (startPage + length > sNumPages) {
dprintf("mark_page_range_in_use(%#" B_PRIxPHYSADDR ", %#" B_PRIxPHYSADDR
"): range would extend past free list\n", startPage, length);
if (startPage >= sNumPages)
return B_OK;
length = sNumPages - startPage;
}
WriteLocker locker(sFreePageQueuesLock);
for (page_num_t i = 0; i < length; i++) {
vm_page *page = &sPages[startPage + i];
switch (page->State()) {
case PAGE_STATE_FREE:
case PAGE_STATE_CLEAR:
{
ASSERT(gKernelStartup);
DEBUG_PAGE_ACCESS_START(page);
VMPageQueue& queue = page->State() == PAGE_STATE_FREE
? sFreePageQueue : sClearPageQueue;
queue.Remove(page);
page->SetState(wired ? PAGE_STATE_WIRED : PAGE_STATE_UNUSED);
page->busy = false;
atomic_add(&sUnreservedFreePages, -1);
DEBUG_PAGE_ACCESS_END(page);
break;
}
case PAGE_STATE_WIRED:
case PAGE_STATE_UNUSED:
break;
case PAGE_STATE_ACTIVE:
case PAGE_STATE_INACTIVE:
case PAGE_STATE_MODIFIED:
case PAGE_STATE_CACHED:
default:
panic("mark_page_range_in_use: page %#" B_PRIxPHYSADDR
" in non-free state %d!\n", startPage + i, page->State());
break;
}
}
return B_OK;
}
static int32
page_scrubber(void *unused)
{
(void)(unused);
TRACE(("page_scrubber starting...\n"));
ConditionVariableEntry entry;
for (;;) {
while (sFreePageQueue.Count() == 0
|| atomic_get(&sUnreservedFreePages)
< (int32)sFreePagesTarget) {
sFreePageCondition.Add(&entry);
entry.Wait();
}
int32 reserved = reserve_some_pages(SCRUB_SIZE,
kPageReserveForPriority[VM_PRIORITY_USER]);
if (reserved == 0)
continue;
ReadLocker locker(sFreePageQueuesLock);
vm_page *page[SCRUB_SIZE];
int32 scrubCount = 0;
for (int32 i = 0; i < reserved; i++) {
page[i] = sFreePageQueue.RemoveHeadUnlocked();
if (page[i] == NULL)
break;
DEBUG_PAGE_ACCESS_START(page[i]);
page[i]->SetState(PAGE_STATE_ACTIVE);
page[i]->busy = true;
scrubCount++;
}
locker.Unlock();
if (scrubCount == 0) {
unreserve_pages(reserved);
continue;
}
TA(ScrubbingPages(scrubCount));
for (int32 i = 0; i < scrubCount; i++)
clear_page(page[i]);
locker.Lock();
for (int32 i = scrubCount - 1; i >= 0; i--) {
page[i]->SetState(PAGE_STATE_CLEAR);
page[i]->busy = false;
DEBUG_PAGE_ACCESS_END(page[i]);
sClearPageQueue.PrependUnlocked(page[i]);
}
locker.Unlock();
unreserve_pages(reserved);
TA(ScrubbedPages(scrubCount));
snooze(100 * 1000);
}
return 0;
}
static void
init_page_marker(vm_page &marker)
{
marker.SetCacheRef(NULL);
marker.InitState(PAGE_STATE_UNUSED);
marker.busy = true;
#if DEBUG_PAGE_QUEUE
marker.queue = NULL;
#endif
#if DEBUG_PAGE_ACCESS
marker.accessing_thread = thread_get_current_thread_id();
#endif
}
static void
remove_page_marker(struct vm_page &marker)
{
DEBUG_PAGE_ACCESS_CHECK(&marker);
if (marker.State() < PAGE_STATE_FIRST_UNQUEUED)
page_queue_for(NULL, marker.State())->RemoveUnlocked(&marker);
marker.SetState(PAGE_STATE_UNUSED);
}
#if ENABLE_SWAP_SUPPORT
static bool
free_page_swap_space(vm_page *page)
{
DEBUG_PAGE_ACCESS_CHECK(page);
PAGE_ASSERT(page, page->State() == PAGE_STATE_ACTIVE);
VMCache* cache = page->Cache();
if (cache->temporary && cache->StoreHasPage(page->cache_offset << PAGE_SHIFT)) {
if (swap_free_page_swap_space(page)) {
page->modified = true;
TD(FreedPageSwap(page));
return true;
}
}
return false;
}
#endif
static vm_page *
find_cached_page_candidate(struct vm_page &marker)
{
DEBUG_PAGE_ACCESS_CHECK(&marker);
InterruptsSpinLocker locker(sCachedPageQueue.GetLock());
vm_page *page;
if (marker.State() == PAGE_STATE_UNUSED) {
page = sCachedPageQueue.Head();
} else {
if (marker.State() != PAGE_STATE_CACHED) {
panic("invalid marker %p state", &marker);
return NULL;
}
page = sCachedPageQueue.Next(&marker);
sCachedPageQueue.Remove(&marker);
marker.SetState(PAGE_STATE_UNUSED);
}
while (page != NULL) {
if (!page->busy) {
marker.SetState(PAGE_STATE_CACHED);
sCachedPageQueue.InsertAfter(page, &marker);
return page;
}
page = sCachedPageQueue.Next(page);
}
return NULL;
}
static bool
free_cached_page(vm_page *page, bool dontWait)
{
if (vm_cache_acquire_locked_page_cache(page, dontWait) == NULL)
return false;
VMCache* cache = page->Cache();
AutoLocker<VMCache> cacheLocker(cache, true);
MethodDeleter<VMCache, void, &VMCache::ReleaseRefLocked> _2(cache);
if (page->busy || page->State() != PAGE_STATE_CACHED)
return false;
DEBUG_PAGE_ACCESS_START(page);
PAGE_ASSERT(page, !page->IsMapped());
PAGE_ASSERT(page, !page->modified);
cache->RemovePage(page);
sCachedPageQueue.RemoveUnlocked(page);
return true;
}
static uint32
free_cached_pages(uint32 pagesToFree, bool dontWait)
{
vm_page marker(-1);
init_page_marker(marker);
forbid_page_faults();
uint32 pagesFreed = 0;
while (pagesFreed < pagesToFree) {
vm_page *page = find_cached_page_candidate(marker);
if (page == NULL)
break;
if (free_cached_page(page, dontWait)) {
ReadLocker locker(sFreePageQueuesLock);
page->SetState(PAGE_STATE_FREE);
DEBUG_PAGE_ACCESS_END(page);
sFreePageQueue.PrependUnlocked(page);
locker.Unlock();
TA(StolenPage());
pagesFreed++;
}
}
remove_page_marker(marker);
permit_page_faults();
sFreePageCondition.NotifyAll();
return pagesFreed;
}
static void
idle_scan_active_pages(page_stats& pageStats)
{
VMPageQueue& queue = sActivePageQueue;
uint32 maxToScan = queue.Count() / kIdleRunsForFullQueue + 1;
while (maxToScan > 0) {
maxToScan--;
vm_page* page = queue.Head();
if (page == NULL)
break;
VMCache* cache = vm_cache_acquire_locked_page_cache(page, true);
if (cache == NULL)
continue;
if (page->State() != PAGE_STATE_ACTIVE) {
cache->ReleaseRefAndUnlock();
continue;
}
DEBUG_PAGE_ACCESS_START(page);
if (page->busy) {
vm_page_requeue(page, true, NULL);
DEBUG_PAGE_ACCESS_END(page);
cache->ReleaseRefAndUnlock();
continue;
}
int32 usageCount;
if (page->WiredCount() > 0 || page->usage_count > 0 || !cache->temporary)
usageCount = vm_clear_page_mapping_accessed_flags(page);
else
usageCount = vm_remove_all_page_mappings_if_unaccessed(page);
if (usageCount > 0) {
usageCount += page->usage_count + kPageUsageAdvance;
if (usageCount > kPageUsageMax)
usageCount = kPageUsageMax;
#if ENABLE_SWAP_SUPPORT
free_page_swap_space(page);
#endif
} else {
usageCount += page->usage_count - (int32)kPageUsageDecline;
if (usageCount < 0) {
usageCount = 0;
set_page_state(page, PAGE_STATE_INACTIVE);
}
}
page->usage_count = usageCount;
DEBUG_PAGE_ACCESS_END(page);
cache->ReleaseRefAndUnlock();
}
}
static void
full_scan_inactive_pages(page_stats& pageStats, int32 despairLevel)
{
int32 pagesToFree = pageStats.unsatisfiedReservations
+ sFreeOrCachedPagesTarget
- (pageStats.totalFreePages + pageStats.cachedPages);
if (pagesToFree <= 0)
return;
bigtime_t time = system_time();
uint32 pagesScanned = 0;
uint32 pagesToCached = 0;
uint32 pagesToModified = 0;
uint32 pagesToActive = 0;
uint32 maxToFlush = despairLevel <= 1 ? 32 : 10000;
vm_page marker(-1);
init_page_marker(marker);
VMPageQueue& queue = sInactivePageQueue;
InterruptsSpinLocker queueLocker(queue.GetLock());
uint32 maxToScan = queue.Count();
ModifiedPageQueue* modifiedQueue = NULL;
vm_page* nextPage = queue.Head();
while (pagesToFree > 0 && maxToScan > 0) {
maxToScan--;
vm_page* page = nextPage;
if (page == NULL)
break;
nextPage = queue.Next(page);
if (page->busy)
continue;
queue.InsertAfter(page, &marker);
queueLocker.Unlock();
VMCache* cache = vm_cache_acquire_locked_page_cache(page, true);
if (cache == NULL || page->busy
|| page->State() != PAGE_STATE_INACTIVE) {
if (cache != NULL)
cache->ReleaseRefAndUnlock();
queueLocker.Lock();
nextPage = queue.Next(&marker);
queue.Remove(&marker);
continue;
}
pagesScanned++;
DEBUG_PAGE_ACCESS_START(page);
int32 usageCount;
if (page->WiredCount() > 0)
usageCount = vm_clear_page_mapping_accessed_flags(page);
else
usageCount = vm_remove_all_page_mappings_if_unaccessed(page);
if (usageCount > 0) {
usageCount += page->usage_count + kPageUsageAdvance;
if (usageCount > kPageUsageMax)
usageCount = kPageUsageMax;
} else {
usageCount += page->usage_count - (int32)kPageUsageDecline;
if (usageCount < 0)
usageCount = 0;
}
page->usage_count = usageCount;
bool isMapped = page->IsMapped();
if (usageCount > 0) {
if (isMapped) {
set_page_state(page, PAGE_STATE_ACTIVE);
pagesToActive++;
} else
vm_page_requeue(page, true, NULL);
} else if (isMapped) {
vm_page_requeue(page, true, NULL);
} else if (!page->modified) {
set_page_state(page, PAGE_STATE_CACHED);
pagesToFree--;
pagesToCached++;
} else if (maxToFlush > 0) {
if (pagesToModified != 0 && modifiedQueue != cache->ModifiedQueue()) {
if (modifiedQueue == NULL)
modifiedQueue = &sDefaultModifiedPageQueue;
modifiedQueue->NotifyWriter();
modifiedQueue = cache->ModifiedQueue();
}
set_page_state(page, PAGE_STATE_MODIFIED);
maxToFlush--;
pagesToModified++;
} else
vm_page_requeue(page, true, NULL);
DEBUG_PAGE_ACCESS_END(page);
cache->ReleaseRefAndUnlock();
queueLocker.Lock();
nextPage = queue.Next(&marker);
queue.Remove(&marker);
}
queueLocker.Unlock();
time = system_time() - time;
TRACE_DAEMON(" -> inactive scan (%7" B_PRId64 " us): scanned: %7" B_PRIu32
", moved: %" B_PRIu32 " -> cached, %" B_PRIu32 " -> modified, %"
B_PRIu32 " -> active\n", time, pagesScanned, pagesToCached,
pagesToModified, pagesToActive);
if (pagesToModified > 0) {
if (modifiedQueue == NULL)
modifiedQueue = &sDefaultModifiedPageQueue;
modifiedQueue->NotifyWriter();
}
}
static void
full_scan_active_pages(page_stats& pageStats, int32 despairLevel)
{
vm_page marker(-1);
init_page_marker(marker);
VMPageQueue& queue = sActivePageQueue;
InterruptsSpinLocker queueLocker(queue.GetLock());
uint32 maxToScan = queue.Count();
int32 pagesToDeactivate = pageStats.unsatisfiedReservations
+ sFreeOrCachedPagesTarget
- (pageStats.totalFreePages + pageStats.cachedPages)
+ std::max((int32)sInactivePagesTarget - (int32)maxToScan, (int32)0);
if (pagesToDeactivate <= 0)
return;
bigtime_t time = system_time();
uint32 pagesAccessed = 0;
uint32 pagesToInactive = 0;
uint32 pagesScanned = 0;
vm_page* nextPage = queue.Head();
while (pagesToDeactivate > 0 && maxToScan > 0) {
maxToScan--;
vm_page* page = nextPage;
if (page == NULL)
break;
nextPage = queue.Next(page);
if (page->busy)
continue;
queue.InsertAfter(page, &marker);
queueLocker.Unlock();
VMCache* cache = vm_cache_acquire_locked_page_cache(page, true);
if (cache == NULL || page->busy || page->State() != PAGE_STATE_ACTIVE) {
if (cache != NULL)
cache->ReleaseRefAndUnlock();
queueLocker.Lock();
nextPage = queue.Next(&marker);
queue.Remove(&marker);
continue;
}
pagesScanned++;
DEBUG_PAGE_ACCESS_START(page);
int32 usageCount = vm_clear_page_mapping_accessed_flags(page);
if (usageCount > 0) {
usageCount += page->usage_count + kPageUsageAdvance;
if (usageCount > kPageUsageMax)
usageCount = kPageUsageMax;
pagesAccessed++;
#if ENABLE_SWAP_SUPPORT
free_page_swap_space(page);
#endif
} else {
usageCount += page->usage_count - (int32)kPageUsageDecline;
if (usageCount <= 0) {
usageCount = 0;
set_page_state(page, PAGE_STATE_INACTIVE);
pagesToInactive++;
}
}
page->usage_count = usageCount;
DEBUG_PAGE_ACCESS_END(page);
cache->ReleaseRefAndUnlock();
queueLocker.Lock();
nextPage = queue.Next(&marker);
queue.Remove(&marker);
}
time = system_time() - time;
TRACE_DAEMON(" -> active scan (%7" B_PRId64 " us): scanned: %7" B_PRIu32
", moved: %" B_PRIu32 " -> inactive, encountered %" B_PRIu32 " accessed"
" ones\n", time, pagesScanned, pagesToInactive, pagesAccessed);
}
static void
page_daemon_idle_scan(page_stats& pageStats)
{
TRACE_DAEMON("page daemon: idle run\n");
if (pageStats.totalFreePages < (int32)sFreePagesTarget) {
uint32 freed = free_cached_pages(
sFreePagesTarget - pageStats.totalFreePages, false);
if (freed > 0)
unreserve_pages(freed);
get_page_stats(pageStats);
}
get_page_stats(pageStats);
idle_scan_active_pages(pageStats);
}
static void
page_daemon_full_scan(page_stats& pageStats, int32 despairLevel)
{
TRACE_DAEMON("page daemon: full run: free: %" B_PRIu32 ", cached: %"
B_PRIu32 ", to free: %" B_PRIu32 "\n", pageStats.totalFreePages,
pageStats.cachedPages, pageStats.unsatisfiedReservations
+ sFreeOrCachedPagesTarget
- (pageStats.totalFreePages + pageStats.cachedPages));
full_scan_inactive_pages(pageStats, despairLevel);
get_page_stats(pageStats);
int32 pagesToFree = pageStats.unsatisfiedReservations + sFreePagesTarget
- (pageStats.totalFreePages);
if (pagesToFree > 0) {
uint32 freed = free_cached_pages(pagesToFree, true);
if (freed > 0)
unreserve_pages(freed);
}
get_page_stats(pageStats);
full_scan_active_pages(pageStats, despairLevel);
}
static status_t
page_daemon(void* )
{
int32 despairLevel = 0;
while (true) {
sPageDaemonCondition.ClearActivated();
page_stats pageStats;
get_page_stats(pageStats);
if (!do_active_paging(pageStats)) {
despairLevel = 0;
page_daemon_idle_scan(pageStats);
sPageDaemonCondition.Wait(kIdleScanWaitInterval, false);
} else {
despairLevel = std::max(despairLevel + 1, (int32)3);
page_daemon_full_scan(pageStats, despairLevel);
if (despairLevel > 1)
snooze(kBusyScanWaitInterval);
}
}
return B_OK;
}
static uint32
reserve_pages(uint32 missing, int priority, bool dontWait)
{
const uint32 requested = missing;
const int32 dontTouch = kPageReserveForPriority[priority];
while (true) {
missing -= reserve_some_pages(missing, dontTouch);
if (missing == 0)
return 0;
if (sUnsatisfiedPageReservations == 0) {
missing -= free_cached_pages(missing, dontWait);
if (missing == 0)
return 0;
}
if (dontWait)
return missing;
MutexLocker pageDeficitLocker(sPageDeficitLock);
if (atomic_get(&sUnreservedFreePages) > dontTouch) {
pageDeficitLocker.Unlock();
continue;
}
const bool notifyDaemon = (sUnsatisfiedPageReservations == 0);
sUnsatisfiedPageReservations += missing;
PageReservationWaiter waiter;
waiter.thread = thread_get_current_thread();
waiter.dontTouch = dontTouch;
waiter.requested = requested;
waiter.reserved = requested - missing;
PageReservationWaiter* otherWaiter = NULL;
for (PageReservationWaiterList::Iterator it
= sPageReservationWaiters.GetIterator();
(otherWaiter = it.Next()) != NULL;) {
if (waiter < *otherWaiter)
break;
}
if (otherWaiter != NULL && sPageReservationWaiters.Head() == otherWaiter) {
if (otherWaiter->reserved >= missing) {
otherWaiter->reserved -= missing;
return 0;
}
missing -= otherWaiter->reserved;
waiter.reserved += otherWaiter->reserved;
otherWaiter->reserved = 0;
} else if (!sPageReservationWaiters.IsEmpty() && waiter.reserved != 0) {
sUnsatisfiedPageReservations -= missing;
missing += waiter.reserved;
atomic_add(&sUnreservedFreePages, waiter.reserved);
wake_up_page_reservation_waiters();
continue;
}
sPageReservationWaiters.InsertBefore(otherWaiter, &waiter);
thread_prepare_to_block(waiter.thread, 0, THREAD_BLOCK_TYPE_OTHER,
"waiting for pages");
if (notifyDaemon)
sPageDaemonCondition.WakeUp();
pageDeficitLocker.Unlock();
low_resource(B_KERNEL_RESOURCE_PAGES, missing, B_RELATIVE_TIMEOUT, 0);
thread_block();
ASSERT(waiter.requested == waiter.reserved);
return 0;
}
}
void
vm_page_init_num_pages(kernel_args *args)
{
sPhysicalPageOffset = args->physical_memory_range[0].start / B_PAGE_SIZE;
page_num_t physicalPagesEnd = sPhysicalPageOffset
+ args->physical_memory_range[0].size / B_PAGE_SIZE;
sNonExistingPages = 0;
sIgnoredPages = args->ignored_physical_memory / B_PAGE_SIZE;
for (uint32 i = 1; i < args->num_physical_memory_ranges; i++) {
page_num_t start = args->physical_memory_range[i].start / B_PAGE_SIZE;
if (start > physicalPagesEnd)
sNonExistingPages += start - physicalPagesEnd;
physicalPagesEnd = start
+ args->physical_memory_range[i].size / B_PAGE_SIZE;
#ifdef LIMIT_AVAILABLE_MEMORY
page_num_t available
= physicalPagesEnd - sPhysicalPageOffset - sNonExistingPages;
if (available > LIMIT_AVAILABLE_MEMORY * (1024 * 1024 / B_PAGE_SIZE)) {
physicalPagesEnd = sPhysicalPageOffset + sNonExistingPages
+ LIMIT_AVAILABLE_MEMORY * (1024 * 1024 / B_PAGE_SIZE);
break;
}
#endif
}
TRACE(("first phys page = %#" B_PRIxPHYSADDR ", end %#" B_PRIxPHYSADDR "\n",
sPhysicalPageOffset, physicalPagesEnd));
sNumPages = physicalPagesEnd - sPhysicalPageOffset;
}
status_t
vm_page_init(kernel_args *args)
{
TRACE(("vm_page_init: entry\n"));
sDefaultModifiedPageQueue.Init();
sInactivePageQueue.Init();
sActivePageQueue.Init();
sCachedPageQueue.Init();
sFreePageQueue.Init();
sClearPageQueue.Init();
new (&sPageReservationWaiters) PageReservationWaiterList;
sPages = (vm_page *)vm_allocate_early(args, sNumPages * sizeof(vm_page),
~0L, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0);
TRACE(("vm_init: putting free_page_table @ %p, # ents %" B_PRIuPHYSADDR
" (size %#" B_PRIxPHYSADDR ")\n", sPages, sNumPages,
(phys_addr_t)(sNumPages * sizeof(vm_page))));
for (uint32 i = 0; i < sNumPages; i++) {
new(&sPages[i]) vm_page(sPhysicalPageOffset + i);
sFreePageQueue.Append(&sPages[i]);
#if VM_PAGE_ALLOCATION_TRACKING_AVAILABLE
sPages[i].allocation_tracking_info.Clear();
#endif
}
sUnreservedFreePages = sNumPages;
TRACE(("initialized table\n"));
phys_addr_t previousEnd = 0;
for (uint32 i = 0; i < args->num_physical_memory_ranges; i++) {
phys_addr_t base = args->physical_memory_range[i].start;
phys_size_t size = args->physical_memory_range[i].size;
if (base > previousEnd) {
mark_page_range_in_use(previousEnd / B_PAGE_SIZE,
(base - previousEnd) / B_PAGE_SIZE, false);
}
previousEnd = base + size;
}
for (uint32 i = 0; i < args->num_physical_allocated_ranges; i++) {
mark_page_range_in_use(
args->physical_allocated_range[i].start / B_PAGE_SIZE,
args->physical_allocated_range[i].size / B_PAGE_SIZE, true);
}
args->num_physical_allocated_ranges = 0;
vm_unreserve_memory(vm_page_num_free_pages() * B_PAGE_SIZE);
sFreePagesTarget = VM_PAGE_RESERVE_USER
+ std::max((page_num_t)32, (sNumPages - sNonExistingPages) / 1024);
if (sUnreservedFreePages < (16 * 1024)) {
sFreeOrCachedPagesTarget = sFreePagesTarget + 128;
sInactivePagesTarget = sFreePagesTarget / 3;
} else {
sFreeOrCachedPagesTarget = 2 * sFreePagesTarget;
sInactivePagesTarget = sFreePagesTarget / 2;
}
TRACE(("vm_page_init: exit\n"));
return B_OK;
}
status_t
vm_page_init_post_area(kernel_args *args)
{
void *dummy;
dummy = sPages;
create_area("page structures", &dummy, B_EXACT_ADDRESS,
PAGE_ALIGN(sNumPages * sizeof(vm_page)), B_ALREADY_WIRED,
B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
add_debugger_command("list_pages", &dump_page_list,
"List physical pages");
add_debugger_command("page_stats", &dump_page_stats,
"Dump statistics about page usage");
add_debugger_command_etc("page", &dump_page_long,
"Dump page info",
"[ \"-p\" | \"-v\" ] [ \"-m\" ] <address>\n"
"Prints information for the physical page. If neither \"-p\" nor\n"
"\"-v\" are given, the provided address is interpreted as address of\n"
"the vm_page data structure for the page in question. If \"-p\" is\n"
"given, the address is the physical address of the page. If \"-v\" is\n"
"given, the address is interpreted as virtual address in the current\n"
"thread's address space and for the page it is mapped to (if any)\n"
"information are printed. If \"-m\" is specified, the command will\n"
"search all known address spaces for mappings to that page and print\n"
"them.\n", 0);
add_debugger_command("page_queue", &dump_page_queue, "Dump page queue");
add_debugger_command("find_page", &find_page,
"Find out which queue a page is actually in");
#ifdef TRACK_PAGE_USAGE_STATS
add_debugger_command_etc("page_usage", &dump_page_usage_stats,
"Dumps statistics about page usage counts",
"\n"
"Dumps statistics about page usage counts.\n",
B_KDEBUG_DONT_PARSE_ARGUMENTS);
#endif
#if VM_PAGE_ALLOCATION_TRACKING_AVAILABLE
add_debugger_command_etc("page_allocations_per_caller",
&dump_page_allocations_per_caller,
"Dump current page allocations summed up per caller",
"[ -d <caller> ] [ -r ]\n"
"The current allocations will by summed up by caller (their count)\n"
"printed in decreasing order by count.\n"
"If \"-d\" is given, each allocation for caller <caller> is printed\n"
"including the respective stack trace.\n"
"If \"-r\" is given, the allocation infos are reset after gathering\n"
"the information, so the next command invocation will only show the\n"
"allocations made after the reset.\n", 0);
add_debugger_command_etc("page_allocation_infos",
&dump_page_allocation_infos,
"Dump current page allocations",
"[ --stacktrace ] [ -p <page number> ] [ --team <team ID> ] "
"[ --thread <thread ID> ]\n"
"The current allocations filtered by optional values will be printed.\n"
"The optional \"-p\" page number filters for a specific page,\n"
"with \"--team\" and \"--thread\" allocations by specific teams\n"
"and/or threads can be filtered (these only work if a corresponding\n"
"tracing entry is still available).\n"
"If \"--stacktrace\" is given, then stack traces of the allocation\n"
"callers are printed, where available\n", 0);
#endif
return B_OK;
}
status_t
vm_page_init_post_thread(kernel_args *args)
{
new (&sFreePageCondition) ConditionVariable;
thread_id thread = spawn_kernel_thread(&page_scrubber, "page scrubber",
B_LOWEST_ACTIVE_PRIORITY, NULL);
resume_thread(thread);
sDefaultModifiedPageQueue.StartWriter("default");
sPageDaemonCondition.Init("page daemon");
thread = spawn_kernel_thread(&page_daemon, "page daemon",
B_NORMAL_PRIORITY, NULL);
resume_thread(thread);
return B_OK;
}
status_t
vm_mark_page_inuse(page_num_t page)
{
return vm_mark_page_range_inuse(page, 1);
}
status_t
vm_mark_page_range_inuse(page_num_t startPage, page_num_t length)
{
return mark_page_range_in_use(startPage, length, false);
}
void
vm_page_unreserve_pages(vm_page_reservation* reservation)
{
uint32 count = reservation->count;
reservation->count = 0;
if (count == 0)
return;
TA(UnreservePages(count));
unreserve_pages(count);
}
void
vm_page_reserve_pages(vm_page_reservation* reservation, uint32 count,
int priority)
{
reservation->count = count;
if (count == 0)
return;
TA(ReservePages(count));
reserve_pages(count, priority, false);
}
bool
vm_page_try_reserve_pages(vm_page_reservation* reservation, uint32 count,
int priority)
{
if (count == 0) {
reservation->count = count;
return true;
}
uint32 remaining = reserve_pages(count, priority, true);
if (remaining == 0) {
TA(ReservePages(count));
reservation->count = count;
return true;
}
unreserve_pages(count - remaining);
return false;
}
vm_page *
vm_page_allocate_page(vm_page_reservation* reservation, uint32 flags)
{
uint32 pageState = flags & VM_PAGE_ALLOC_STATE;
ASSERT(pageState != PAGE_STATE_FREE && pageState != PAGE_STATE_CLEAR);
ASSERT(pageState != PAGE_STATE_MODIFIED);
ASSERT(reservation->count > 0);
reservation->count--;
VMPageQueue* queue;
VMPageQueue* otherQueue;
if ((flags & VM_PAGE_ALLOC_CLEAR) != 0) {
queue = &sClearPageQueue;
otherQueue = &sFreePageQueue;
} else {
queue = &sFreePageQueue;
otherQueue = &sClearPageQueue;
}
ReadLocker locker(sFreePageQueuesLock);
vm_page* page = queue->RemoveHeadUnlocked();
if (page == NULL) {
page = otherQueue->RemoveHeadUnlocked();
if (page == NULL) {
locker.Unlock();
WriteLocker writeLocker(sFreePageQueuesLock);
page = queue->RemoveHead();
if (page == NULL)
otherQueue->RemoveHead();
if (page == NULL) {
panic("Had reserved page, but there is none!");
return NULL;
}
locker.Lock();
}
}
if (page->CacheRef() != NULL)
panic("supposed to be free page %p has cache @! page %p; cache _cache", page, page);
DEBUG_PAGE_ACCESS_START(page);
uint8 oldPageState = page->State();
ASSERT(oldPageState == PAGE_STATE_FREE || oldPageState == PAGE_STATE_CLEAR);
page->SetState(pageState);
page->busy = (flags & VM_PAGE_ALLOC_BUSY) != 0;
page->busy_io = false;
page->accessed = false;
page->modified = false;
page->usage_count = 0;
locker.Unlock();
if (pageState < PAGE_STATE_FIRST_UNQUEUED)
page_queue_for(page, pageState)->AppendUnlocked(page);
if ((flags & VM_PAGE_ALLOC_CLEAR) != 0 && oldPageState != PAGE_STATE_CLEAR)
clear_page(page);
#if VM_PAGE_ALLOCATION_TRACKING_AVAILABLE
page->allocation_tracking_info.Init(
TA(AllocatePage(page->physical_page_number)));
#else
TA(AllocatePage(page->physical_page_number));
#endif
return page;
}
static void
allocate_page_run_cleanup(VMPageQueue::PageList& freePages,
VMPageQueue::PageList& clearPages)
{
while (vm_page* page = freePages.RemoveTail()) {
page->busy = false;
page->SetState(PAGE_STATE_FREE);
DEBUG_PAGE_ACCESS_END(page);
sFreePageQueue.PrependUnlocked(page);
}
while (vm_page* page = clearPages.RemoveTail()) {
page->busy = false;
page->SetState(PAGE_STATE_CLEAR);
DEBUG_PAGE_ACCESS_END(page);
sClearPageQueue.PrependUnlocked(page);
}
sFreePageCondition.NotifyAll();
}
static page_num_t
allocate_page_run(page_num_t start, page_num_t length, uint32 flags,
WriteLocker& freeClearQueueLocker)
{
uint32 pageState = flags & VM_PAGE_ALLOC_STATE;
ASSERT(pageState != PAGE_STATE_FREE);
ASSERT(pageState != PAGE_STATE_CLEAR);
ASSERT(start + length <= sNumPages);
page_num_t cachedPages = 0;
VMPageQueue::PageList freePages;
VMPageQueue::PageList clearPages;
page_num_t i = 0;
for (; i < length; i++) {
bool pageAllocated = true;
bool noPage = false;
vm_page& page = sPages[start + i];
switch (page.State()) {
case PAGE_STATE_CLEAR:
DEBUG_PAGE_ACCESS_START(&page);
sClearPageQueue.Remove(&page);
clearPages.Add(&page);
break;
case PAGE_STATE_FREE:
DEBUG_PAGE_ACCESS_START(&page);
sFreePageQueue.Remove(&page);
freePages.Add(&page);
break;
case PAGE_STATE_CACHED:
cachedPages++;
pageAllocated = false;
break;
default:
noPage = true;
break;
}
if (noPage)
break;
if (pageAllocated) {
page.SetState(flags & VM_PAGE_ALLOC_STATE);
page.busy = (flags & VM_PAGE_ALLOC_BUSY) != 0;
page.busy_io = false;
page.accessed = false;
page.modified = false;
page.usage_count = 0;
}
}
if (i < length) {
allocate_page_run_cleanup(freePages, clearPages);
return i;
}
freeClearQueueLocker.Unlock();
if (cachedPages > 0) {
page_num_t freedCachedPages = 0;
page_num_t nextIndex = start;
vm_page* freePage = freePages.Head();
vm_page* clearPage = clearPages.Head();
while (cachedPages > 0) {
if (freePage != NULL && size_t(freePage - sPages) == nextIndex) {
freePage = freePages.GetNext(freePage);
nextIndex++;
continue;
}
if (clearPage != NULL && size_t(clearPage - sPages) == nextIndex) {
clearPage = clearPages.GetNext(clearPage);
nextIndex++;
continue;
}
vm_page& page = sPages[nextIndex];
if (!free_cached_page(&page, false)) {
break;
}
page.SetState(flags & VM_PAGE_ALLOC_STATE);
page.busy = (flags & VM_PAGE_ALLOC_BUSY) != 0;
page.busy_io = false;
page.accessed = false;
page.modified = false;
page.usage_count = 0;
freePages.InsertBefore(freePage, &page);
freedCachedPages++;
cachedPages--;
nextIndex++;
}
if (freedCachedPages > 0)
unreserve_pages(freedCachedPages);
if ((nextIndex - start) < length) {
freeClearQueueLocker.Lock();
allocate_page_run_cleanup(freePages, clearPages);
freeClearQueueLocker.Unlock();
return nextIndex - start;
}
}
if ((flags & VM_PAGE_ALLOC_CLEAR) != 0) {
for (VMPageQueue::PageList::Iterator it = freePages.GetIterator();
vm_page* page = it.Next();) {
clear_page(page);
}
}
if (pageState < PAGE_STATE_FIRST_UNQUEUED) {
freePages.TakeFrom(&clearPages);
page_queue_for(NULL, pageState)->AppendUnlocked(freePages, length);
}
#if VM_PAGE_ALLOCATION_TRACKING_AVAILABLE
AbstractTraceEntryWithStackTrace* traceEntry
= TA(AllocatePageRun(start, length));
for (page_num_t i = start; i < start + length; i++)
sPages[i].allocation_tracking_info.Init(traceEntry);
#else
TA(AllocatePageRun(start, length));
#endif
return length;
}
vm_page*
vm_page_allocate_page_run(uint32 flags, page_num_t length,
const physical_address_restrictions* restrictions, int priority)
{
page_num_t requestedStart
= std::max(restrictions->low_address / B_PAGE_SIZE, sPhysicalPageOffset)
- sPhysicalPageOffset;
page_num_t start = requestedStart;
page_num_t end;
if (restrictions->high_address > 0) {
end = std::max(restrictions->high_address / B_PAGE_SIZE,
sPhysicalPageOffset)
- sPhysicalPageOffset;
end = std::min(end, sNumPages);
} else
end = sNumPages;
page_num_t alignmentMask
= std::max(restrictions->alignment / B_PAGE_SIZE, (phys_addr_t)1) - 1;
ASSERT(((alignmentMask + 1) & alignmentMask) == 0);
uint32 boundaryMask = 0;
if (restrictions->boundary != 0) {
page_num_t boundary = restrictions->boundary / B_PAGE_SIZE;
ASSERT(((boundary - 1) & boundary) == 0);
ASSERT(boundary >= alignmentMask + 1);
ASSERT(boundary >= length);
boundaryMask = -boundary;
}
vm_page_reservation reservation;
vm_page_reserve_pages(&reservation, length, priority);
WriteLocker freeClearQueueLocker(sFreePageQueuesLock);
int32 freePages = sUnreservedFreePages;
bool useCached = (freePages > 0) && ((page_num_t)freePages > (length * 2));
for (;;) {
if (alignmentMask != 0 || boundaryMask != 0) {
page_num_t offsetStart = start + sPhysicalPageOffset;
if ((offsetStart & alignmentMask) != 0)
offsetStart = (offsetStart + alignmentMask) & ~alignmentMask;
if (boundaryMask != 0 && ((offsetStart ^ (offsetStart
+ length - 1)) & boundaryMask) != 0) {
offsetStart = (offsetStart + length - 1) & boundaryMask;
}
start = offsetStart - sPhysicalPageOffset;
}
if (start + length > end) {
if (!useCached) {
useCached = true;
start = requestedStart;
continue;
}
dprintf("vm_page_allocate_page_run(): Failed to allocate run of "
"length %" B_PRIuPHYSADDR " (%" B_PRIuPHYSADDR " %"
B_PRIuPHYSADDR ") in second iteration (align: %" B_PRIuPHYSADDR
" boundary: %" B_PRIuPHYSADDR ")!\n", length, requestedStart,
end, restrictions->alignment, restrictions->boundary);
freeClearQueueLocker.Unlock();
vm_page_unreserve_pages(&reservation);
return NULL;
}
bool foundRun = true;
page_num_t i;
for (i = 0; i < length; i++) {
uint32 pageState = sPages[start + i].State();
if (pageState != PAGE_STATE_FREE
&& pageState != PAGE_STATE_CLEAR
&& (pageState != PAGE_STATE_CACHED || !useCached)) {
foundRun = false;
break;
}
}
if (foundRun) {
i = allocate_page_run(start, length, flags, freeClearQueueLocker);
if (i == length) {
reservation.count = 0;
return &sPages[start];
}
freeClearQueueLocker.Lock();
}
start += i + 1;
}
}
vm_page *
vm_page_at_index(int32 index)
{
return &sPages[index];
}
vm_page *
vm_lookup_page(page_num_t pageNumber)
{
if (pageNumber < sPhysicalPageOffset)
return NULL;
pageNumber -= sPhysicalPageOffset;
if (pageNumber >= sNumPages)
return NULL;
return &sPages[pageNumber];
}
bool
vm_page_is_dummy(struct vm_page *page)
{
return page < sPages || page >= sPages + sNumPages;
}
void
vm_page_free_etc(VMCache* cache, vm_page* page,
vm_page_reservation* reservation)
{
PAGE_ASSERT(page, page->State() != PAGE_STATE_FREE
&& page->State() != PAGE_STATE_CLEAR);
PAGE_ASSERT(page, page->State() != PAGE_STATE_MODIFIED);
free_page(page, false);
if (reservation == NULL)
unreserve_pages(1);
else
reservation->count++;
}
void
vm_page_set_state(vm_page *page, int pageState)
{
PAGE_ASSERT(page, page->State() != PAGE_STATE_FREE
&& page->State() != PAGE_STATE_CLEAR);
set_page_state(page, pageState);
}
void
vm_page_requeue(struct vm_page *page, bool tail, VMPageQueue** _queue)
{
PAGE_ASSERT(page, page->Cache() != NULL);
page->Cache()->AssertLocked();
DEBUG_PAGE_ACCESS_CHECK(page);
if (page->State() == PAGE_STATE_FREE || page->State() == PAGE_STATE_CLEAR) {
panic("vm_page_requeue() called for free/clear page %p", page);
return;
}
VMPageQueue *queue = page_queue_for(page, page->State());
if (_queue != NULL)
*_queue = queue;
if (queue == NULL)
return;
queue->RequeueUnlocked(page, tail);
}
page_num_t
vm_page_num_pages(void)
{
return sNumPages - sNonExistingPages;
}
page_num_t
vm_page_num_free_pages(void)
{
int32 count = sUnreservedFreePages + sCachedPageQueue.Count();
return count > 0 ? count : 0;
}
page_num_t
vm_page_num_unused_pages(void)
{
int32 count = sUnreservedFreePages;
return count > 0 ? count : 0;
}
void
vm_page_get_stats(system_info *info)
{
page_num_t blockCachePages = block_cache_used_memory() / B_PAGE_SIZE;
info->block_cache_pages = blockCachePages;
page_num_t modifiedNonTemporaryPages
= (ModifiedPageQueue::GlobalModifiedCount() - sModifiedTemporaryPages);
info->max_pages = vm_page_num_pages();
info->cached_pages = sCachedPageQueue.Count() + modifiedNonTemporaryPages
+ blockCachePages;
uint32 subtractPages = info->cached_pages + sFreePageQueue.Count()
+ sClearPageQueue.Count();
info->used_pages = subtractPages > info->max_pages
? 0 : info->max_pages - subtractPages;
if (info->used_pages + info->cached_pages > info->max_pages) {
info->cached_pages = info->max_pages - info->used_pages;
}
info->page_faults = vm_num_page_faults();
info->ignored_pages = sIgnoredPages;
}
phys_addr_t
vm_page_max_address()
{
return ((phys_addr_t)sPhysicalPageOffset + sNumPages) * B_PAGE_SIZE - 1;
}
RANGE_MARKER_FUNCTION_END(vm_page)
