#ifndef _KERNEL_VM_VM_CACHE_H
#define _KERNEL_VM_VM_CACHE_H
#include <debug.h>
#include <kernel.h>
#include <util/DoublyLinkedList.h>
#include <vm/vm.h>
#include <vm/vm_types.h>
#include <vm/VMArea.h>
#include "kernel_debug_config.h"
struct kernel_args;
struct ObjectCache;
enum {
CACHE_TYPE_RAM = 1,
CACHE_TYPE_VNODE,
CACHE_TYPE_DEVICE,
CACHE_TYPE_NULL
};
enum {
PAGE_EVENT_NOT_BUSY = 0x01
};
extern ObjectCache* gCacheRefObjectCache;
extern ObjectCache* gAnonymousCacheObjectCache;
extern ObjectCache* gAnonymousNoSwapCacheObjectCache;
extern ObjectCache* gVnodeCacheObjectCache;
extern ObjectCache* gDeviceCacheObjectCache;
extern ObjectCache* gNullCacheObjectCache;
struct VMCachePagesTreeDefinition {
typedef page_num_t KeyType;
typedef vm_page NodeType;
static page_num_t GetKey(const NodeType* node)
{
return node->cache_offset;
}
static SplayTreeLink<NodeType>* GetLink(NodeType* node)
{
return &node->cache_link;
}
static int Compare(page_num_t key, const NodeType* node)
{
return key == node->cache_offset ? 0
: (key < node->cache_offset ? -1 : 1);
}
static NodeType** GetListLink(NodeType* node)
{
return &node->cache_next;
}
};
typedef IteratableSplayTree<VMCachePagesTreeDefinition> VMCachePagesTree;
struct VMCache : public DoublyLinkedListLinkImpl<VMCache> {
public:
typedef DoublyLinkedList<VMCache> ConsumerList;
public:
VMCache();
virtual ~VMCache();
status_t Init(const char* name, uint32 cacheType,
uint32 allocationFlags);
virtual void Delete();
inline bool Lock();
inline bool TryLock();
inline bool SwitchLock(mutex* from);
inline bool SwitchFromReadLock(rw_lock* from);
void Unlock(bool consumerLocked = false);
inline void AssertLocked();
inline void AcquireRefLocked();
inline void AcquireRef();
inline void ReleaseRefLocked();
inline void ReleaseRef();
inline void ReleaseRefAndUnlock(
bool consumerLocked = false);
int32 RefCount() const
{ return fRefCount; }
void* UserData() { return fUserData; }
void SetUserData(void* data) { fUserData = data; }
inline VMCacheRef* CacheRef() const { return fCacheRef; }
void WaitForPageEvents(vm_page* page, uint32 events,
bool relock);
void NotifyPageEvents(vm_page* page, uint32 events)
{ if (fPageEventWaiters != NULL)
_NotifyPageEvents(page, events); }
inline void MarkPageUnbusy(vm_page* page);
void FreeRemovedPage(vm_page* page);
vm_page* LookupPage(off_t offset);
void InsertPage(vm_page* page, off_t offset);
void RemovePage(vm_page* page);
void MovePage(vm_page* page, off_t offset);
void MovePage(vm_page* page);
void MoveAllPages(VMCache* fromCache);
inline page_num_t WiredPagesCount() const;
inline void IncrementWiredPagesCount();
inline void DecrementWiredPagesCount();
virtual int32 GuardSize() { return 0; }
void AddConsumer(VMCache* consumer);
void InsertAreaLocked(VMArea* area);
void RemoveArea(VMArea* area);
void TakeAreasFrom(VMCache* fromCache);
uint32 CountWritableAreas(VMArea* ignoreArea) const;
virtual status_t Resize(off_t newSize, int priority);
virtual status_t Rebase(off_t newBase, int priority);
virtual status_t Adopt(VMCache* source, off_t offset, off_t size,
off_t newOffset);
virtual ssize_t Discard(off_t offset, off_t size);
status_t FlushAndRemoveAllPages();
status_t SetMinimalCommitment(off_t commitment,
int priority);
virtual off_t Commitment() const;
virtual bool CanOvercommit();
virtual status_t Commit(off_t size, int priority);
virtual void TakeCommitmentFrom(VMCache* from, off_t commitment);
virtual bool StoreHasPage(off_t offset);
virtual status_t Read(off_t offset, const generic_io_vec *vecs,
size_t count, uint32 flags,
generic_size_t *_numBytes);
virtual status_t Write(off_t offset, const generic_io_vec *vecs,
size_t count, uint32 flags,
generic_size_t *_numBytes);
virtual status_t WriteAsync(off_t offset,
const generic_io_vec* vecs, size_t count,
generic_size_t numBytes, uint32 flags,
AsyncIOCallback* callback);
virtual bool CanWritePage(off_t offset);
status_t WriteModified();
virtual int32 MaxPagesPerWrite() const
{ return -1; }
virtual int32 MaxPagesPerAsyncWrite() const
{ return -1; }
virtual status_t Fault(struct VMAddressSpace *aspace,
off_t offset);
inline uint64 FaultCount() const
{ return fFaultCount; }
inline void IncrementFaultCount()
{ fFaultCount++; }
inline uint64 CopiedPagesCount() const
{ return fCopiedPagesCount; }
inline void IncrementCopiedPagesCount()
{ fCopiedPagesCount++; }
virtual void Merge(VMCache* source);
virtual status_t AcquireUnreferencedStoreRef();
virtual void AcquireStoreRef();
virtual void ReleaseStoreRef();
virtual bool DebugStoreHasPage(off_t offset);
vm_page* DebugLookupPage(off_t offset);
virtual void Dump(bool showPages) const;
protected:
virtual void DeleteObject() = 0;
public:
VMArea::CacheList areas;
ConsumerList consumers;
VMCachePagesTree pages;
VMCache* source;
off_t virtual_base;
off_t virtual_end;
uint32 page_count;
uint32 temporary : 1;
uint32 type : 6;
#if DEBUG_CACHE_LIST
VMCache* debug_previous;
VMCache* debug_next;
#endif
private:
struct PageEventWaiter;
friend struct VMCacheRef;
private:
void _NotifyPageEvents(vm_page* page, uint32 events);
inline bool _IsMergeable() const;
void _MergeWithOnlyConsumer();
void _RemoveConsumer(VMCache* consumer);
bool _FreePageRange(VMCachePagesTree::Iterator it,
page_num_t* toPage, page_num_t* freedPages);
private:
typedef DoublyLinkedQueue<vm_page, DoublyLinkedListCLink<vm_page,
SplayTreeLink<vm_page>, &vm_page::cache_link> > RemovedPagesQueue;
private:
int32 fRefCount;
mutex fLock;
PageEventWaiter* fPageEventWaiters;
void* fUserData;
VMCacheRef* fCacheRef;
RemovedPagesQueue fRemovedBusyPages;
page_num_t fWiredPagesCount;
uint64 fFaultCount;
uint64 fCopiedPagesCount;
};
#if DEBUG_CACHE_LIST
extern VMCache* gDebugCacheList;
#endif
class VMCacheFactory {
public:
static status_t CreateAnonymousCache(VMCache*& cache,
bool canOvercommit, int32 numPrecommittedPages,
int32 numGuardPages, bool swappable,
int priority);
static status_t CreateVnodeCache(VMCache*& cache,
struct vnode* vnode);
static status_t CreateDeviceCache(VMCache*& cache,
addr_t baseAddress);
static status_t CreateNullCache(int priority, VMCache*& cache);
};
bool
VMCache::Lock()
{
return mutex_lock(&fLock) == B_OK;
}
bool
VMCache::TryLock()
{
return mutex_trylock(&fLock) == B_OK;
}
bool
VMCache::SwitchLock(mutex* from)
{
return mutex_switch_lock(from, &fLock) == B_OK;
}
bool
VMCache::SwitchFromReadLock(rw_lock* from)
{
return mutex_switch_from_read_lock(from, &fLock) == B_OK;
}
void
VMCache::AssertLocked()
{
ASSERT_LOCKED_MUTEX(&fLock);
}
void
VMCache::AcquireRefLocked()
{
ASSERT_LOCKED_MUTEX(&fLock);
fRefCount++;
}
void
VMCache::AcquireRef()
{
Lock();
fRefCount++;
Unlock();
}
void
VMCache::ReleaseRefLocked()
{
ASSERT_LOCKED_MUTEX(&fLock);
fRefCount--;
}
void
VMCache::ReleaseRef()
{
Lock();
fRefCount--;
Unlock();
}
void
VMCache::ReleaseRefAndUnlock(bool consumerLocked)
{
ReleaseRefLocked();
Unlock(consumerLocked);
}
void
VMCache::MarkPageUnbusy(vm_page* page)
{
ASSERT(page->busy);
ASSERT(page->Cache() == this);
page->busy = false;
NotifyPageEvents(page, PAGE_EVENT_NOT_BUSY);
}
page_num_t
VMCache::WiredPagesCount() const
{
return fWiredPagesCount;
}
void
VMCache::IncrementWiredPagesCount()
{
ASSERT(fWiredPagesCount < page_count);
fWiredPagesCount++;
}
void
VMCache::DecrementWiredPagesCount()
{
ASSERT(fWiredPagesCount > 0);
fWiredPagesCount--;
}
inline void
vm_page::IncrementWiredCount()
{
if (fWiredCount++ == 0)
cache_ref->cache->IncrementWiredPagesCount();
}
inline void
vm_page::DecrementWiredCount()
{
ASSERT_PRINT(fWiredCount > 0, "page: %#" B_PRIx64, physical_page_number * B_PAGE_SIZE);
if (--fWiredCount == 0)
cache_ref->cache->DecrementWiredPagesCount();
}
#ifdef __cplusplus
extern "C" {
#endif
status_t vm_cache_init(struct kernel_args *args);
void vm_cache_init_post_heap();
struct VMCache *vm_cache_acquire_locked_page_cache(struct vm_page *page,
bool dontWait);
#ifdef __cplusplus
}
#endif
#endif
