diff --git a/Makefile b/Makefile index 2f684da..8cf1deb 100644 --- a/Makefile +++ b/Makefile @@ -665,7 +665,7 @@ KBUILD_LDFLAGS_MODULE += $(LDFLAGS_BUILD_ID) LDFLAGS_vmlinux += $(LDFLAGS_BUILD_ID) ifeq ($(CONFIG_STRIP_ASM_SYMS),y) -LDFLAGS_vmlinux += $(call ld-option, -X,) +LDFLAGS_vmlinux += $(call ld-option, -X,) endif # Default kernel image to build when no specific target is given. diff --git a/fs/drop_caches.c b/fs/drop_caches.c index c00e055..e5c9897 100644 --- a/fs/drop_caches.c +++ b/fs/drop_caches.c @@ -45,7 +45,7 @@ static void drop_slab(void) }; do { - nr_objects = shrink_slab(&shrink, 1000, 1000); + nr_objects = shrink_slab(&shrink, 1000, 1000); } while (nr_objects > 10); } diff --git a/fs/inode.c b/fs/inode.c index 601b724..6cb499d 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -278,8 +278,8 @@ void address_space_init_once(struct address_space *mapping) spin_lock_init(&mapping->private_lock); INIT_RAW_PRIO_TREE_ROOT(&mapping->i_mmap); INIT_LIST_HEAD(&mapping->i_mmap_nonlinear); - - pf_cache_init(mapping); + + } EXPORT_SYMBOL(address_space_init_once); @@ -358,7 +358,11 @@ void inode_sb_list_add(struct inode *inode) { spin_lock(&inode_sb_list_lock); list_add(&inode->i_sb_list, &inode->i_sb->s_inodes); +#ifdef XXX_PFC + pf_cache_init(&(inode->i_data)); //make unevictable and call reaper. +#endif spin_unlock(&inode_sb_list_lock); + } EXPORT_SYMBOL_GPL(inode_sb_list_add); @@ -367,6 +371,9 @@ static inline void inode_sb_list_del(struct inode *inode) if (!list_empty(&inode->i_sb_list)) { spin_lock(&inode_sb_list_lock); list_del_init(&inode->i_sb_list); +#ifdef XXX_PFC + pf_cache_destroy(inode); +#endif spin_unlock(&inode_sb_list_lock); } } @@ -456,7 +463,7 @@ static void evict(struct inode *inode) BUG_ON(!(inode->i_state & I_FREEING)); BUG_ON(!list_empty(&inode->i_lru)); - + // pf_cache_destroy(inode); //calls reaper if (!list_empty(&inode->i_wb_list)) inode_wb_list_del(inode); @@ -481,7 +488,9 @@ static void evict(struct inode *inode) BUG_ON(inode->i_state != (I_FREEING | I_CLEAR)); spin_unlock(&inode->i_lock); + // destroy_inode(inode); + } /* diff --git a/fs/proc/base.c b/fs/proc/base.c index 1fc1dca..2afc7d9 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -83,6 +83,7 @@ #include #include #include +#include #ifdef CONFIG_HARDWALL #include #endif @@ -1818,13 +1819,25 @@ out: return ~0U; } -#define PROC_FDINFO_MAX 64 +#define PROC_FDINFO_MAX 128 +/* Dump info about the file. Add the file hit rates and all that? */ static int proc_fd_info(struct inode *inode, struct path *path, char *info) { struct task_struct *task = get_proc_task(inode); struct files_struct *files = NULL; struct file *file; + struct CAR_state* C ; + struct address_space* m = &(inode->i_data) ; + if((inode->i_data).CAR_state) + C = (inode->i_data).CAR_state ; + + int ev ; +#ifdef XXX_PFC + ev = C->EVICTIONS[0] ; +#else + ev =0; +#endif int fd = proc_fd(inode); if (task) { @@ -1851,12 +1864,12 @@ static int proc_fd_info(struct inode *inode, struct path *path, char *info) *path = file->f_path; path_get(&file->f_path); } - if (info) + if (info) { + snprintf(info, PROC_FDINFO_MAX, - "pos:\t%lli\n" - "flags:\t0%o\n", - (long long) file->f_pos, - f_flags); +"pos:\t%lli\n" "flags:\t0%o\n" "size: %lu\n" "Hits: %lu\n" "Misses: %lu\n" "Evictions: %d\n" , + (long long) file->f_pos, f_flags, inode->i_data.nrpages, m->mapping_hits, m->mapping_misses,ev); + } spin_unlock(&files->file_lock); put_files_struct(files); return 0; diff --git a/fs/super.c b/fs/super.c index afd0f1a..38905fe 100644 --- a/fs/super.c +++ b/fs/super.c @@ -85,6 +85,7 @@ static int prune_super(struct shrinker *shrink, struct shrink_control *sc) * prune the dcache first as the icache is pinned by it, then * prune the icache, followed by the filesystem specific caches */ + printk("IN PRUNE_SUPER called by shrinker somehow\n"); prune_dcache_sb(sb, dentries); prune_icache_sb(sb, inodes); diff --git a/include/linux/fs.h b/include/linux/fs.h index 76465ca..89a6495 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -632,27 +632,45 @@ int pagecache_write_end(struct file *, struct address_space *mapping, loff_t pos, unsigned len, unsigned copied, struct page *page, void *fsdata); -struct pf_cache ; +struct pf_cache { + unsigned long curr_size ; + unsigned long max_size ; + pgoff_t contents[1024] ; + int curr_index ; + int eviction_pointer ; +}; struct backing_dev_info; + +struct CAR_state ; + struct address_space { - struct inode *host; /* owner: inode, block_device */ - struct radix_tree_root page_tree; /* radix tree of all pages */ - spinlock_t tree_lock; /* and lock protecting it */ - unsigned int i_mmap_writable;/* count VM_SHARED mappings */ - struct prio_tree_root i_mmap; /* tree of private and shared mappings */ - struct list_head i_mmap_nonlinear;/*list VM_NONLINEAR mappings */ - struct mutex i_mmap_mutex; /* protect tree, count, list */ - /* Protected by tree_lock together with the radix tree */ - unsigned long nrpages; /* number of total pages */ - pgoff_t writeback_index;/* writeback starts here */ - const struct address_space_operations *a_ops; /* methods */ - unsigned long flags; /* error bits/gfp mask */ - struct backing_dev_info *backing_dev_info; /* device readahead, etc */ - spinlock_t private_lock; /* for use by the address_space */ - struct list_head private_list; /* ditto */ - struct address_space *assoc_mapping; /* ditto */ - struct pf_cache* cache ; + struct inode *host; /* owner: inode, block_device */ + struct radix_tree_root page_tree; /* radix tree of all pages */ + spinlock_t tree_lock; /* and lock protecting it */ + unsigned int i_mmap_writable;/* count VM_SHARED mappings */ + struct prio_tree_root i_mmap; /* tree of private and shared mappings */ + struct list_head i_mmap_nonlinear;/*list VM_NONLINEAR mappings */ + struct mutex i_mmap_mutex; /* protect tree, count, list */ + /* Protected by tree_lock together with the radix tree */ + unsigned long nrpages; /* number of total pages */ + pgoff_t writeback_index;/* writeback starts here */ + const struct address_space_operations *a_ops; /* methods */ + unsigned long flags; /* error bits/gfp mask */ + struct backing_dev_info *backing_dev_info; /* device readahead, etc */ + spinlock_t private_lock; /* for use by the address_space */ + struct list_head private_list; /* ditto */ + struct address_space *assoc_mapping; /* ditto */ + struct CAR_state* CAR_state ; + struct FIFO_state* FIFO_state ; + struct pf_cache pf_cache ; + unsigned int CLOCK_flags ; /*Counter for the clock? */ + unsigned long ref_time ; /* Counter to determine LRU inode*/ + unsigned long mapping_hits ; + unsigned long mapping_misses ; + int eviction_in_progress ; + struct list_head reaper_list ; + } __attribute__((aligned(sizeof(long)))); /* * On most architectures that alignment is already the case; but diff --git a/include/linux/per-file-cache.h b/include/linux/per-file-cache.h new file mode 100644 index 0000000..796c171 --- /dev/null +++ b/include/linux/per-file-cache.h @@ -0,0 +1,185 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* for BUG_ON(!in_atomic()) only */ +#include +#include + +/************ HIGH LEVEL PER FILE CACHE INTERFACES **************************/ +int pf_cache_init(struct address_space* mapping) ; +int pf_cache_destroy(struct inode*); +int pf_cache_add(struct page* page,struct address_space* mapping, pgoff_t offset) ; + + +int pf_cache_nuke(struct address_space* mapping) ; + + +int pf_cache_access(struct page* page,struct address_space* mapping, pgoff_t offset) ; +int pf_cache_hit(struct page* page,struct address_space* mapping, pgoff_t offset) ; +int pf_cache_miss(struct page* page,struct address_space* mapping, pgoff_t offset) ; + +int pf_remove_page(struct address_space *mapping, struct page *page) ; + +int pf_cache_delete(struct page* page,struct address_space* mapping,pgoff_t offset) ; + +int pf_cache_evict(struct address_space* mapping) ; + +int pf_cache_evict_n(struct address_space* mapping, int n) ; + +int pf_cache_delete(struct page* page,struct address_space* mapping, pgoff_t offset) ; + +int pf_cache_expand(struct address_space* mapping, int n) ; + + +int pf_cache_stats(struct address_space* mapping) ; + +unsigned long pf_cache_cur_size(struct address_space* mapping) ; + +int pf_cache_history(struct address_space* mapping) ; + +int pf_cache_set_algorithm(struct address_space* mapping) ; + +int pf_cache_shrink(struct address_space* mapping, int to_shrink); + +struct page* pf_offset2page(struct address_space* mapping, pgoff_t offset); + +void pferr_mapped(void); + +void pferr_wback(void); + +void pferr_locked(void); + +void pferr_dirty(void); + +void pferr_private(void); + +void pferr_freeze_refs(void); +void pferr_freed(void); + +void bump_pf_hits(void); +void bump_pf_misses(void); +void bump_pf_RA(void); + + +extern int INODE_CLOCK_MAX ; +extern int INODE_CLOCK_MIN ; + +int inc_reaper_inode_clock(struct address_space* mapping) ; +int age_reaper_inode_clock(struct address_space* mapping) ; + +/********************* FIFO INTERFACES *****************************************/ + +int FIFO_init(struct address_space* mapping); +int FIFO_insert(pgoff_t offset, struct page* page,struct address_space* mapping); + +/*********************** CAR INTERFACES ***************************************/ + +struct pf_cache_object { + pgoff_t offset ; /*primary key */ + struct page* page ; /*No additional reference for this*/ + int referenced ; + struct list_head list ; +}; + +enum CAR_filetype { + PFT_small, + PFT_new, + PFT_huge, + PFT_sequential, + PFT_random, + PFT_dontneed, + PFT_willneed +}; + +#define NUM_CAR_LISTS 4 +#define MU_MAX 1 +#define NUM_CAR_BUCKETS 10 + + +/*This is the global state that is used by everyone. We are behind a + lock of somekind whenver manipulating this? */ +struct CAR_state { + struct address_space* parent_mapping ; + spinlock_t carlock ; + unsigned long C ; /* Maximum size of the cache */ + unsigned long curr_size ; /* Current size <= C */ + int slack ; /* Num pages above max-size which can be used. Allocation is C+slack? */ + unsigned long p ; /* The ARC L1/L2 control parameter.Varies from 1,C */ + unsigned long dir_size ; /* Max value is 2C, but can be less/more for the Z list.*/ + unsigned long size[NUM_CAR_LISTS] ; + struct list_head D[NUM_CAR_LISTS] ; /*Directory. Uses enum TB_list to identify T1,T2,B1,B2 */ + unsigned long H[NUM_CAR_BUCKETS] ; /* Hits */ + unsigned long B[NUM_CAR_BUCKETS] ; /* Block count per bucket. */ + enum CAR_filetype CAR_filetype ; + int MU[MU_MAX] ; /* Marginal utility=>hits */ + int dAccess ; + int MISSES[MU_MAX] ; + int dMisses ; + int EVICTIONS[MU_MAX] ; + int dEvictions ; + int recent_evictions ; /* More pages to request */ + int RA_success ; + int dRA_success ; +}; + + + +int CAR(pgoff_t obj , struct page* page, struct address_space* mapping); + +struct CAR_state* init_CAR(struct address_space* mapping) ; + +int shrink_CAR(struct CAR_state* , int num) ; + +int shrink_new_CAR(struct CAR_state* CAR_state, int num ); + +void CARLOCK(struct CAR_state* CAR_state); + +void CARUNLOCK(struct CAR_state* CAR_state); + +int CARTRYLOCK(struct CAR_state* CAR_state) ; + +int setCARType(struct file* file, enum CAR_filetype type) ; + +/******************* REAPER INTERFACE *******************************/ +enum REAP_event { + reap_new_file, /*New addr_space */ + reap_periodic, /*Update LRU lists */ + reap_file_grow, /* File needs more space */ + reap_overflow, /* Crossed F */ + reap_vmscan /* Called by vmscan to grow/shrink */ +}; + +//unsigned long pf_total_file_pages; + +int call_reaper(enum REAP_event event, void* arg); +int add_to_reaper(struct CAR_state* CAR_state); +int remove_from_reaper(struct address_space* mapping); +void inc_pf_hard_alloc(void); +int dec_reap_hard_alloc(void) ; +void inc_pf_soft_alloc(int n) ; +int reaper_anon_inform(long scanned, long rotated) ; + +/**********************************************************************/ + +#define XXX_PFC + + diff --git a/mm/Kconfig b/mm/Kconfig index 011b110..9135e9c 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -240,6 +240,7 @@ config KSM until a program has madvised that an area is MADV_MERGEABLE, and root has set /sys/kernel/mm/ksm/run to 1 (if CONFIG_SYSFS is set). + config DEFAULT_MMAP_MIN_ADDR int "Low address space to protect from user allocation" depends on MMU diff --git a/mm/Makefile b/mm/Makefile index 449132e..c6bb4fb 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -13,7 +13,7 @@ obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ readahead.o swap.o truncate.o vmscan.o shmem.o \ prio_tree.o util.o mmzone.o vmstat.o backing-dev.o \ page_isolation.o mm_init.o mmu_context.o percpu.o \ - per-file-cache.o \ + pf_cart.o pfifo.o reaper.o per-file-cache.o\ $(mmu-y) obj-y += init-mm.o diff --git a/mm/fadvise.c b/mm/fadvise.c index 8d723c9..e42cd30 100644 --- a/mm/fadvise.c +++ b/mm/fadvise.c @@ -17,6 +17,7 @@ #include #include #include +#include /*For car filetype */ #include @@ -85,19 +86,21 @@ SYSCALL_DEFINE(fadvise64_64)(int fd, loff_t offset, loff_t len, int advice) spin_lock(&file->f_lock); file->f_mode |= FMODE_RANDOM; spin_unlock(&file->f_lock); + setCARType(file,PFT_random) ; break; case POSIX_FADV_SEQUENTIAL: file->f_ra.ra_pages = bdi->ra_pages * 2; spin_lock(&file->f_lock); file->f_mode &= ~FMODE_RANDOM; spin_unlock(&file->f_lock); + setCARType(file,PFT_sequential) ; break; case POSIX_FADV_WILLNEED: if (!mapping->a_ops->readpage) { ret = -EINVAL; break; } - + setCARType(file,PFT_willneed) ; /* First and last PARTIAL page! */ start_index = offset >> PAGE_CACHE_SHIFT; end_index = endbyte >> PAGE_CACHE_SHIFT; @@ -116,6 +119,7 @@ SYSCALL_DEFINE(fadvise64_64)(int fd, loff_t offset, loff_t len, int advice) case POSIX_FADV_NOREUSE: break; case POSIX_FADV_DONTNEED: + setCARType(file,PFT_dontneed) ; if (!bdi_write_congested(mapping->backing_dev_info)) filemap_flush(mapping); diff --git a/mm/filemap.c b/mm/filemap.c index 76cd506..f839a11 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -147,6 +147,7 @@ void __delete_from_page_cache(struct page *page) dec_zone_page_state(page, NR_FILE_DIRTY); dec_bdi_stat(mapping->backing_dev_info, BDI_RECLAIMABLE); } + } /** @@ -467,8 +468,10 @@ int add_to_page_cache_locked(struct page *page, struct address_space *mapping, if (likely(!error)) { mapping->nrpages++; __inc_zone_page_state(page, NR_FILE_PAGES); + spin_unlock_irq(&mapping->tree_lock); - pf_cache_add(page, mapping, offset) ; /*Outside the spinlock. */ + pf_cache_add(page, mapping, offset) ; /*Outside the spinlock because we also evict here */ + pf_cache_miss(page,mapping,offset) ; /* Accounting only */ } else { page->mapping = NULL; /* Leave page->index set: truncation relies upon it */ @@ -490,8 +493,13 @@ int add_to_page_cache_lru(struct page *page, struct address_space *mapping, int ret; ret = add_to_page_cache(page, mapping, offset, gfp_mask); - /* if (ret == 0) */ - /* lru_cache_add_file(page); */ +#ifdef XXX_PFC + ; +#else + if (ret == 0) + lru_cache_add_file(page); +#endif + return ret; } EXPORT_SYMBOL_GPL(add_to_page_cache_lru); @@ -710,11 +718,14 @@ repeat: page_cache_release(page); goto repeat; } - pf_cache_access(page, mapping, offset) ; + //pf_cache_access(page, mapping, offset) ; } out: rcu_read_unlock(); + if(page) + pf_cache_hit(page, mapping, offset); + return page; } EXPORT_SYMBOL(find_get_page); @@ -1832,6 +1843,7 @@ repeat: page = ERR_PTR(err); } } + return page; } diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 485be89..5836afb 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -571,10 +571,11 @@ static inline int free_pages_check(struct page *page) (page->mapping != NULL) | (atomic_read(&page->_count) != 0) | (page->flags & PAGE_FLAGS_CHECK_AT_FREE) | - (mem_cgroup_bad_page_check(page)))) { + (mem_cgroup_bad_page_check(page)))) + { bad_page(page); return 1; - } + } if (page->flags & PAGE_FLAGS_CHECK_AT_PREP) page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; return 0; diff --git a/mm/per-file-cache.c b/mm/per-file-cache.c new file mode 100644 index 0000000..13b33db --- /dev/null +++ b/mm/per-file-cache.c @@ -0,0 +1,494 @@ + +#include +#include +#include +#include "internal.h" +#include + +#define pf_dbg printk +#define pf_err printk +unsigned long pf_total_evicted = 0; +extern unsigned long pf_total_file_pages; +static void update_histogram_hits(pgoff_t offset, struct CAR_state *CAR_state); +extern struct s_rstat *rstat; + +int INODE_CLOCK_MAX = 4; +int INODE_CLOCK_MIN = 0; + +int pf_cache_hit(struct page *page, struct address_space *mapping, pgoff_t offset) +{ + /* do_generic_file_read clears the readahead flag via + page_cache_async_readahed if the page is flagged as readahed. + Read-ahead "hits" are not actual cache hits. Moreover, readahead + would turn ALL sequential accesses to land-up on the T2 CAR list, + since a page would get added to the T1 list during the readahead, + and then upon a successful prefetch, would be marked as active and + land up in T2. */ +#ifdef XXX_PFC + if (!mapping->CAR_state) + return 0; + if (!PageReadahead(page)) + SetPageActive(page); + else { + mapping->CAR_state->RA_success++; + mapping->CAR_state->dRA_success++ ; + bump_pf_RA(); + } + mapping->CAR_state->MU[0]++; + bump_pf_hits(); + mapping->CAR_state->dAccess++ ; + update_histogram_hits(offset, mapping->CAR_state); + inc_reaper_inode_clock(mapping); +#else + if (PageReadahead(page)) { + bump_pf_RA(); + } + bump_pf_hits(); + mapping->mapping_hits++ ; +#endif +} + +int inc_reaper_inode_clock(struct address_space *mapping) +{ + mapping->CLOCK_flags++; + mapping->CLOCK_flags = max(INODE_CLOCK_MAX, mapping->CLOCK_flags); + return INODE_CLOCK_MAX - mapping->CLOCK_flags; +} + +/* Return 0 if the clock counter reaches 0 */ +int age_reaper_inode_clock(struct address_space *mapping) +{ + if(mapping->CLOCK_flags != 0) + mapping->CLOCK_flags--; + + return mapping->CLOCK_flags; +} + +int pf_cache_miss(struct page *page, struct address_space *mapping, pgoff_t offset) +{ + bump_pf_misses(); + mapping->mapping_misses++ ; + +#ifdef XXX_PFC + ClearPageActive(page); +#endif + +} + +/*Update the address-space access-time and CLOCK flags? */ +int pf_cache_access(struct page *page, struct address_space *mapping, pgoff_t offset) +{ + +} + +int is_car_ok = 0; + +int pf_cache_init(struct address_space *mapping) +{ + /** allocate the access history, spinlock, lru unevictable etc */ + + struct pf_cache *cache = &(mapping->pf_cache); + struct CAR_state *CAR_state; + cache->max_size = 1024; + cache->curr_size = 0; + cache->curr_index = 0; + cache->eviction_pointer = 0; + mapping_set_unevictable(mapping); + + CAR_state = init_CAR(mapping); + if (!CAR_state) { + return 0; + } + + add_to_reaper(CAR_state); /* So that we can evict */ + call_reaper(reap_new_file, CAR_state); /* Update size in CAR_state, slack if needed */ + // FIFO_init(mapping); +} + +/* Inode is being freed. Delete address-space from CAR, and reaper */ +int pf_cache_destroy(struct inode *inode) +{ + remove_from_reaper(&(inode->i_data)); + return 0; +} + +/** Delete the offset from the access history or whatever */ +int pf_cache_delete(struct page *page, struct address_space *mapping, pgoff_t offset) +{ + ClearPageActive(page); + dec_reap_hard_alloc(); +} + +pgoff_t pf_page2offset(struct page *page) +{ + return page->index; +} + +/* Reduce the cache size by num pages. Return number of pages evicted */ +int pf_cache_shrink(struct address_space *mapping, int num) +{ + printk("Shrinkging cache by %d \n", num); + return shrink_CAR(mapping->CAR_state, num); +} + +/* No reference is taken for this page */ +struct page *pf_offset2page(struct address_space *mapping, pgoff_t offset) +{ + void **pagep; + struct page *page; + + rcu_read_lock(); + repeat: + page = NULL; + pagep = radix_tree_lookup_slot(&mapping->page_tree, offset); + if (pagep) { + page = radix_tree_deref_slot(pagep); + if (unlikely(!page)) { + printk("Could not deref slot \n"); + goto out; + } + if (radix_tree_exception(page)) { + if (radix_tree_deref_retry(page)) + goto repeat; + printk("radix tree eexception \n"); + goto out; + } + if (unlikely(page != *pagep)) + goto repeat; + } + out: + rcu_read_unlock(); + return page; +} + +enum page_references { + PAGEREF_RECLAIM, + PAGEREF_RECLAIM_CLEAN, + PAGEREF_KEEP, + PAGEREF_ACTIVATE, +}; + +static enum page_references my_page_check_references(struct page *page) +{ + int referenced_ptes, referenced_page; + unsigned long vm_flags; + + referenced_ptes = page_referenced(page, 1, NULL, &vm_flags); + + referenced_page = TestClearPageReferenced(page); + + /* Lumpy reclaim - ignore references */ + + if (referenced_ptes) { + if (PageAnon(page)) + return PAGEREF_ACTIVATE; + /* + * All mapped pages start out with page table + * references from the instantiating fault, so we need + * to look twice if a mapped file page is used more + * than once. + * + * Mark it and spare it for another trip around the + * inactive list. Another page table reference will + * lead to its activation. + * + * Note: the mark is set for activated pages as well + * so that recently deactivated but used pages are + * quickly recovered. + */ + SetPageReferenced(page); + + if (referenced_page) + return PAGEREF_ACTIVATE; + + return PAGEREF_KEEP; + } + + /* Reclaim if clean, defer dirty pages to writeback */ + if (referenced_page && !PageSwapBacked(page)) + return PAGEREF_RECLAIM_CLEAN; + + return PAGEREF_RECLAIM; +} + +static int may_write_to_queue(struct backing_dev_info *bdi) +{ + if (current->flags & PF_SWAPWRITE) + return 1; + if (!bdi_write_congested(bdi)) + return 1; + if (bdi == current->backing_dev_info) + return 1; + + return 0; +} + +static int write_dirty(struct address_space *mapping, struct page *page) +{ + return 0; + + /* if (mapping->a_ops->writepage == NULL) */ + /* return 0 ; */ + /* if (!may_write_to_queue(mapping->backing_dev_info)) */ + /* return 0; */ + + /* if (clear_page_dirty_for_io(page)) { */ + /* int res; */ + /* struct writeback_control wbc = { */ + /* .sync_mode = WB_SYNC_NONE, */ + /* .nr_to_write = SWAP_CLUSTER_MAX, */ + /* .range_start = 0, */ + /* .range_end = LLONG_MAX, */ + /* .for_reclaim = 1, */ + /* }; */ + + /* SetPageReclaim(page); */ + /* res = mapping->a_ops->writepage(page, &wbc); */ + /* if (res < 0) */ + /* handle_write_error(mapping, page, res); */ + /* if (res == AOP_WRITEPAGE_ACTIVATE) { */ + /* ClearPageReclaim(page); */ + /* return 0; */ + /* } */ + /* if (!PageWriteback(page)) { */ + /* /\* synchronous write or broken a_ops? *\/ */ + /* ClearPageReclaim(page); */ + /* } */ + /* return 1; */ + /* } */ + /* return 0 ; */ +} + +/** This is a replacment for remove_mapping from vmscan.c. Return 1 if removed from the radix tree, 0 otherwise. */ +int pf_nuke_page(struct address_space *mapping, struct page *page) +{ + pgoff_t offset = page->index; + BUG_ON(!PageLocked(page)); + + // TestClearPageReferenced(page); + //ClearPageReferenced(page); + ClearPageActive(page); + spin_lock_irq(&mapping->tree_lock); + + /*Originally, there are 2 references for an unmapped cache page. The radix tree and the LRU list. + Since the LRU list doesnt exist, there is only one reference here here? */ + if (!page_freeze_refs(page, 1)) { + printk("cannot freeze refs %d \n", page->_count); + pferr_freeze_refs(); + goto cannot_free; + } + + if (unlikely(PageDirty(page))) { + if (!write_dirty(mapping, page)) { + pferr_dirty(); + page_unfreeze_refs(page, 1); + goto cannot_free; + } /*Else,successfully wrote, so continue reclaim */ + } + + void (*freepage) (struct page *); + freepage = mapping->a_ops->freepage; + + __delete_from_page_cache(page); + + spin_unlock_irq(&mapping->tree_lock); + + if (freepage != NULL) + freepage(page); + + // printk("Successfully evicted!! %lu. Total %lu \n",offset,pf_total_evicted++); + + page_unfreeze_refs(page, 1); + return 1; + + cannot_free: + spin_unlock_irq(&mapping->tree_lock); + return 0; +} + +unsigned long pf_total_freed = 0; + +static int list_experiment(pgoff_t offset); + +/* Puts on the freelist */ +int pf_release_page(struct page *page) +{ + if (!put_page_testzero(page)) { //decrements count + printk("References > 0\n"); + return 0; + } + ClearPageUnevictable(page); /* My addition, since CAR sets/unsets this active, although only an inactive page is ever freed. */ + unsigned long flags; + struct zone *zone = page_zone(page); + if (PageLRU(page)) { + spin_lock_irqsave(&zone->lru_lock, flags); + VM_BUG_ON(!PageLRU(page)); + __ClearPageLRU(page); + del_page_from_lru(zone, page); + spin_unlock_irqrestore(&zone->lru_lock, flags); + } + // __mod_zone_page_state(zone, NR_FILE , -1); + // __dec_zone_page_state(page, NR_FILE_PAGES); + // dec_zone_page_state(page, NR_FILE_PAGES); + + /* Finally free! */ + free_hot_cold_page(page, 0); + pferr_freed(); + /*printk("FREED! %lu\n",pf_total_freed++); Although good news, this leads to printk storms */ + + return 1; +} + +/* Return 1 if freed page, else 0. Modified shrink_page_list from vmscan.c */ +/* This is the top-level remoal function, which calls + nuke-page(removes mapping) and free_page(which actually puts the + page in the free page list and actually makes the page free */ +int pf_remove_page(struct address_space *mapping, struct page *page) +{ + int references; + //get rid of all locks. CAR. <- masterlock <- inode sb list locks + if (!trylock_page(page)) { + // printk("could not lock \n"); + pferr_locked(); + return 0; + } + if (page_mapped(page)) { + if (try_to_unmap(page, TTU_UNMAP) != SWAP_SUCCESS) { + pferr_mapped(); + goto keep; + } /* Else, we successfully unmapped the page */ + } + + if (PageWriteback(page)) { + // printk("wback\n"); + pferr_wback(); + goto keep; + // wait_on_page_writeback(page); + } + if (PageDirty(page)) { + pferr_dirty(); + /*NUKED Page is dirty, try to write it out here */ + goto keep; + } + if (page_has_private(page)) { + if (!try_to_release_page(page, GFP_KERNEL)) { + pferr_private(); + goto keep; + } + } + /* if(!page_is_file_cache(page)) { */ + /* goto keep; */ + /* } */ + references = my_page_check_references(page); + + /* switch (references) { */ + /* case PAGEREF_ACTIVATE: */ + /* goto activate_locked; */ + /* case PAGEREF_KEEP: */ + /* goto keep; */ + /* case PAGEREF_RECLAIM: */ + /* case PAGEREF_RECLAIM_CLEAN: */ + /* ; /\* try to reclaim the page below *\/ */ + /* } */ + + if (!pf_nuke_page(mapping, page)) + goto keep; + success: + __clear_page_locked(page); + unlock_page(page); + + if (pf_release_page(page)) { + //dec_reap_hard_alloc() ; + return 1; + } else + return 0; + + //__clear_page_locked(page); + + keep: + // printk("Keeping\n"); + unlock_page(page); + return 0; +} + +/* Increase size or evict. Return 0 if full and cant evict.*/ +static int handle_cache_full(struct address_space *mapping) +{ + struct pf_cache *cache = &(mapping->pf_cache); + + if (cache->eviction_pointer > 1024 - 100) { + cache->eviction_pointer = 0; + } + // printk("clock hand at %d on page %lu \n",cache->eviction_pointer, cache->contents[cache->eviction_pointer]) ; + + int i = 0; + // for (i=0;i<100;i++) { + pgoff_t victim = cache->contents[cache->eviction_pointer + i]; + struct page *page = pf_offset2page(mapping, victim); + cache->eviction_pointer; + if (page != NULL) + pf_remove_page(mapping, page); + // } + cache->eviction_pointer++; + return 1; +} + + +/* Add page to the list of pages we keep which are in the cache. Offsets only? + Assume we have buffer space. */ +static int update_occupancy(struct page *page, struct address_space *mapping, + pgoff_t offset) +{ + struct pf_cache *cache = &(mapping->pf_cache); + if (cache->curr_index > 1024) { + cache->curr_index = 0; + } + cache->contents[cache->curr_index++] = offset; + + return 1; +} + +static void update_histogram_blocks(pgoff_t offset, struct CAR_state *CAR_state) +{ + int bucket = offset % NUM_CAR_BUCKETS; + CAR_state->B[bucket]++; +} + +static void update_histogram_hits(pgoff_t offset, struct CAR_state *CAR_state) +{ + int bucket = offset % NUM_CAR_BUCKETS; + CAR_state->H[bucket]++; +} + +/** Add page to the right cache. Called by add_to_page_cache . + * Careful, this is called while holding tree_lock spinlock + * DONT SLEEP OR BLOCK + */ +int pf_cache_add(struct page *page, struct address_space *mapping, pgoff_t offset) +{ +#ifdef XXX_PFC + CAR(offset, page, mapping); + mapping->CAR_state->MISSES[0]++; + inc_pf_hard_alloc(); + update_histogram_blocks(offset, mapping->CAR_state); + return 1; +#endif +} + +/** Remove a page from the mapping. Called by us/eviction thread/kswapd? */ +int pf_cache_evict_page(struct address_space *mapping, int offset) +{ + pf_dbg("evict::offset:%lu\n", offset); +} + +/* Returns the page to evict */ +pgoff_t pf_cache_victim(struct address_space *mapping) +{ + return 0; +} + +unsigned long pf_cache_cur_size(struct address_space *mapping) +{ + unsigned long C = mapping->CAR_state->curr_size; + return C; +} diff --git a/mm/pf_cart.c b/mm/pf_cart.c new file mode 100644 index 0000000..c6f992d --- /dev/null +++ b/mm/pf_cart.c @@ -0,0 +1,711 @@ +/** + * This file contains the implementation of the CART Algorithm as + * described in [1]. CAR is a CLOCK implementation of the Adaptive + * Replacement Cache (ARC)[2]. There is no need to update any + * cache-state on cache-hits, avoiding lock contention in the + * read-side. + * References: + * 1 : CAR: Clock with Adapative Replacement. Bansal and Modha + * 2 : ARC: Adaptive Replacement Cache. Megiddo and Modha + * + */ + +#include +#include +#include +#include +#include +#include + +int pf_total_cached_car = 0; +int CAR_EVICT_THRESH = 1000; + +/* integer identifiers for the T/B lists */ +static enum TB_list { + T_1, + T_2, + B_1, + B_2 +}; + +#define T1_size CAR_state->size[T_1] +#define T2_size CAR_state->size[T_2] +#define B1_size CAR_state->size[B_1] +#define B2_size CAR_state->size[B_2] + +/************************** FORWARD DECLARATIONS *************************************/ + +static int search_shadow_lists(struct CAR_state *CAR_state, pgoff_t offset); +int car_add_to_tail(struct CAR_state *CAR_state, enum TB_list type, + struct pf_cache_object *obj); +static struct pf_cache_object *move_hand(struct CAR_state *CAR_state, + enum TB_list type); +static int pf_free(struct pf_cache_object *obj); +int evict_car_page(struct CAR_state *CAR_state, struct pf_cache_object *); +static int cache_full(struct CAR_state *CAR_state); +static int discard_LRU(struct CAR_state *CAR_state, enum TB_list type); +int advance_clock_hand(struct CAR_state *CAR_state); +static int isReferenced(struct pf_cache_object *); +static int clearReference(struct pf_cache_object *); +static int remove_from_list(struct CAR_state *CAR_state, enum TB_list type, + struct pf_cache_object *x); +static int search_remove_offset(struct CAR_state *CAR_state, + enum TB_list type, pgoff_t offset); +static struct pf_cache_object *alloc_pf_object(pgoff_t); +static void increase_p(struct CAR_state *CAR_state); +static void decrease_p(struct CAR_state *CAR_state); +static int cart_slab_init(void); + +/**************************** GLOBAL VARIABLES ************************************/ + +static int first_car_run = 1; +static struct kmem_cache *slab_cache_pfobj = NULL; /* This is a global slab cache used by ALL the files, only created ONCE */ + +/*******************************************************************************/ + +/* Allocate the CAR_State structure and initialize it. SLab cache also + created on the first invocation of this*/ +struct CAR_state *init_CAR(struct address_space *mapping) +{ + int i = 0; + mapping->CAR_state = kmalloc(sizeof(struct CAR_state), GFP_ATOMIC); + if (mapping->CAR_state == NULL) { + printk("CAR STATE COULD NOT BE ALLOCATED!!!\n"); + return 0; + } + struct CAR_state *CAR_state = mapping->CAR_state; //local reference for the rest of the function. + + CAR_state->parent_mapping = mapping; + spin_lock_init(&CAR_state->carlock); + CAR_state->C = 3000; + CAR_state->p = 2000; + if (mapping != CAR_state->parent_mapping) { + printk("mappings are different \n"); + return 0; + } + for (i = 0; i < NUM_CAR_LISTS; i++) { + INIT_LIST_HEAD(&(CAR_state->D[i])); + CAR_state->size[i] = 0; + CAR_state->MU[i] = 0; + } + + for (i = 0; i < NUM_CAR_BUCKETS; i++) { + CAR_state->H[i] = 0; + CAR_state->B[i] = 1; /* Sidestep divide by zero check everytime */ + } + if (first_car_run) { + /* Initialize a single slab for *all* pf_cache_objects for all the + files, instead of one slab-cache per file. */ + if (!cart_slab_init()) + return 0; + first_car_run = 0; + } + + return CAR_state; +} + +static int cart_slab_init(void) +{ + struct kmem_cache *temp; + temp = kmem_cache_create("pf_objects", sizeof(struct pf_cache_object), 0, 0, NULL); + if (temp == NULL) { + return 0; + } + slab_cache_pfobj = temp; + return 1; +} + +/* Macros T1_size are defined above. expand to simple CAR_state->size[0/1/2/3] */ +static int cache_full(struct CAR_state *CAR_state) +{ + if (T1_size + T2_size > CAR_state->C) + return 1; + return 0; +} + +int +car_add_to_tail(struct CAR_state *CAR_state, enum TB_list type, + struct pf_cache_object *obj) +{ + struct list_head *thehead = &(CAR_state->D[type]); + list_add_tail(&(obj->list), thehead); + CAR_state->size[type]++; + struct pf_cache_object *cursor; + + /* list_for_each_entry(cursor, thehead,list) { */ + /* printk("%lu, ",cursor->offset); */ + /* } */ + return 1; +} + +/********************************************************************************/ +/********************************************************************************/ +/********************************************************************************/ + +/* The actual CAR algorithm. Called only on a miss!. Assume a CAR_lock + is held throughout? Lots of list updates etc. +*/ + +/* No case for when a hit occours, since all we do is update the + active bit in the page flags after a successful radix-tree + lookup! +*/ + +int CAR(pgoff_t offset, struct page *page, struct address_space *mapping) +{ + struct CAR_state *CAR_state = mapping->CAR_state; + if (mapping->CAR_state == NULL) { + init_CAR(mapping); + } + unsigned long flags; + struct pf_cache_object *new_entry; + new_entry = alloc_pf_object(offset); + + CAR_state->MISSES[0]++; + CARLOCK(CAR_state); + + int in_shadow = search_shadow_lists(CAR_state, offset); /*Returns 0,1,2 whether hit is in None,B1,B2 */ + unsigned long C = CAR_state->C; + + pf_total_cached_car++; + + if (cache_full(CAR_state)) { + advance_clock_hand(CAR_state); /*Make room */ + CAR_state->EVICTIONS[0]++; + if (in_shadow == 0 && T1_size + B1_size > C) { + //printk("T1-B1 overflow. Discarding lru of b1 \n"); + discard_LRU(CAR_state, B_1); /*head */ + } + /* Cache dir full and not in shadow */ + else if (T1_size + B1_size + T2_size + B2_size > 2 * C && + in_shadow == 0) { + discard_LRU(CAR_state, B_2); + } + } + + /*The cache is full, and the page isnt there in the directory + * either, so we have discarded a shadow entry after advancing the + * clock hand */ + CAR_state->dMisses++ ; + if (new_entry == NULL) { + printk("aborting because of lack of slab memory\n"); + goto abort; + } + new_entry->page = page; + clearReference(new_entry); + + /*Cache-dir miss. Add to T_1 */ + if (in_shadow == 0) { + car_add_to_tail(CAR_state, T_1, new_entry); + } + /* B_1 hit. Move to T2. T2 because we already hit obj once earlier. */ + else if (in_shadow == B_1) { + // printk("B1 hit, adding to T2\n"); + search_remove_offset(CAR_state, B_1, offset); + car_add_to_tail(CAR_state, T_2, new_entry); + increase_p(CAR_state); + } + /* in B2 */ + else if (in_shadow == B_2) { + // printk("found in B2 \n"); + search_remove_offset(CAR_state, B_2, offset); + car_add_to_tail(CAR_state, T_2, new_entry); + decrease_p(CAR_state); + } + + abort: + CARUNLOCK(CAR_state); + int extra_space; + if (CAR_state->EVICTIONS[0] > CAR_EVICT_THRESH) { + extra_space = call_reaper(reap_file_grow, CAR_state); + CAR_state->EVICTIONS[1] = CAR_state->EVICTIONS[0]; + CAR_state->EVICTIONS[0] = 0; + CAR_state->dMisses=0 ; + CAR_state->dAccess=0 ; + CAR_state->dRA_success=0 ; + } + return 1; +} + +int try_to_evict_one(struct CAR_state *CAR_state, enum TB_list type) +{ + int i = 0; + struct list_head *head = &(CAR_state->D[type]); + struct pf_cache_object *obj; + list_for_each_entry(obj, head, list) { + i++; + if (evict_car_page(CAR_state, obj)) { + return 1; + } + } + printk("I scanned %d pages, nothing could be evicted \n", i); + return 0; +} + +/** Called to make some room in the cache + * Move unreferenced pages to their respective Shadow lists. + * Referenced pages are moved to T_2. + * Repeat until some page is moved to a shadow-list. + */ +int advance_clock_hand(struct CAR_state *CAR_state) +{ + struct pf_cache_object *hand; + unsigned long p = CAR_state->p; + // printk("advancing clock. T1size:%lu T2size:%lu B1size:%lu B2size:%lu p:%lu \n",T1_size, T2_size, B1_size,B2_size, p); + // return try_to_evict_one(T_1); + + int found = 0; + int tries = 0; /* Pages that were tried to be evicted, but couldn't be */ + do { + if (T1_size > p) { + hand = move_hand(CAR_state, T_1); /* Removes and returns head */ + if (!hand) + return 0; + if (!isReferenced(hand)) { + if (evict_car_page(CAR_state, hand)) { + car_add_to_tail(CAR_state, B_1, hand); + found = 1; + return 1; + } /*Else, page could not be evicted. Must put it back on the T1 list. */ + else { + found = 1; //original algorithm. + return 1; + car_add_to_tail(CAR_state, T_1, hand); /*reinsert */ + found = 0; + tries++; + if (tries >= T1_size) { + printk + ("entire list circled, nothing could be evicted!\n"); + return 0; + } + } + } /* Page is referenced. Move it to T_2 */ + else { + clearReference(hand); + car_add_to_tail(CAR_state, T_2, hand); + found = 0; + } + } else { /* T_2 has overflowed */ + hand = move_hand(CAR_state, T_2); + if (!hand) + return 0; + if (!isReferenced(hand)) { + if (evict_car_page(CAR_state, hand)) { + car_add_to_tail(CAR_state, B_2, hand); + found = 1; + return 1; + } /*Else, page could not be evicted. Must put it back on the T1 list. */ + else { + found = 1; //original algorithm + return 1; + car_add_to_tail(CAR_state, T_2, hand); /*reinsert */ + found = 0; + tries++; + if (tries >= T2_size) { + printk + ("entire list circled, nothing could be evicted!\n"); + return 0; + } + } + } /* Page is referenced. Give it a second chance on T_2 */ + else { + clearReference(hand); + car_add_to_tail(CAR_state, T_2, hand); + found = 0; + } + } /*End of 2 cases : hand advances on T1 or T2 */ + } + while (found == 0); + return 1; +} /* End of function */ + +/*****************************************************************************/ + +static struct pf_cache_object *alloc_pf_object(pgoff_t offset) +{ + struct pf_cache_object *ret = kmem_cache_alloc(slab_cache_pfobj, GFP_KERNEL); + if (ret == NULL) + return NULL; + ret->offset = offset; + INIT_LIST_HEAD(&(ret->list)); + return ret; +} + +static int pf_free(struct pf_cache_object *obj) +{ + kmem_cache_free(slab_cache_pfobj, obj); + //kfree(obj); +} + +/* Tries to Evicts the page which was on the clock-hand. Returns 0 if + we cant evict. Actually frees the page. */ +int evict_car_page(struct CAR_state *CAR_state, struct pf_cache_object *obj) +{ + int try = 0; + struct page *page; + + page = pf_offset2page(CAR_state->parent_mapping, obj->offset); + + if (page != NULL) { + return pf_remove_page(CAR_state->parent_mapping, page); + } + printk("page null \n"); + return 0; +} + +static int hit_rate_affected(struct CAR_state *CAR_state) +{ + return 0; +} + +int CAR_evict_one(struct CAR_state *CAR_state, int T1) +{ + struct pf_cache_object *hand; + unsigned long p = CAR_state->p; + + int found = 0; + int tries = 0; /* Pages that were tried to be evicted, but couldn't be */ + if (T1_size == 0 && T2_size == 0) { + // printk("CAR empty %d\n", CAR_state->parent_mapping->host->i_ino); + return 0; + } + while (!found) { + if (T1 && T1_size > 0) { /*Evict from T_1 */ + hand = move_hand(CAR_state, T_1); /* Removes and returns head */ + if (!hand) + return 0; + if (1) { + // if(!isReferenced(hand)) { + if (evict_car_page(CAR_state, hand)) { + car_add_to_tail(CAR_state, B_1, hand); + found = 1; + return 1; + } /*Else, page could not be evicted. Must put it back on the T1 list. */ + else { + car_add_to_tail(CAR_state, T_1, hand); /*reinsert */ + found = 0; + tries++; + if (tries >= T1_size) { + // printk("entire T1 list circled, nothing could be evicted!\n"); + return 0; + } + } + } /* Page is referenced. Move it to T_2 */ + else { + clearReference(hand); + car_add_to_tail(CAR_state, T_2, hand); + found = 0; + } + } else if (T2_size > 0) { + hand = move_hand(CAR_state, T_2); + if (!hand) + return 0; + if (1) { + // if(!isReferenced(hand)) { + if (evict_car_page(CAR_state, hand)) { + car_add_to_tail(CAR_state, B_2, hand); + found = 1; + return 1; + } /*Else, page could not be evicted. Must put it back on the T1 list. */ + else { + car_add_to_tail(CAR_state, T_2, hand); /*reinsert */ + found = 0; + tries++; + if (tries >= T2_size) { + //printk("entire T2 list circled, nothing could be evicted!\n") ; + return 0; + } + } + } /* Page is referenced. Give it a second chance on T_2 */ + else { + clearReference(hand); + car_add_to_tail(CAR_state, T_2, hand); + found = 0; + } + } /*End of 2 cases : hand advances on T1 or T2 */ + else + return 0; + } /* End of while */ + return 0; +} /* End of function */ + +/* Shrink the cache by num. Called by reaper. Be selfish, dont shrink + if hit rates would be badly affected. +*/ +int shrink_CAR(struct CAR_state *CAR_state, int num) +{ + int to_evict = num; + int num_evicted = 0; + unsigned long flags; + //CARLOCK(CAR_state); + if (!CARTRYLOCK(CAR_state)) + return -1; + + if (to_evict > CAR_state->C) { + to_evict = CAR_state->C; + } + int T1 = 1; + + while (to_evict) { + if (CAR_evict_one(CAR_state, T1)) { + to_evict--; + num_evicted++; + } else { + if (T1 == 0) + goto out; + else + T1 = 0; + } + } + out: + CARUNLOCK(CAR_state); + if (num != 0 && num_evicted == 0 && T1_size + T2_size != 0) { + ; // printk("BOOO. failed to evict %d pages out of %d \n",num, T1_size+T2_size); + } + /* if(num-to_evict!=0) */ + /* printk("Shrunk CAR %x BY %d \n", CAR_state->parent_mapping, num-to_evict); */ + return num_evicted; +} + +/***************************************************************/ +int drain_CAR(struct CAR_state *CAR_state) +{ + int num = T1_size + T2_size; + int T1; + while (num) { + if (CAR_evict_one(CAR_state, T1)) { + num--; + } else { + if (T1 == 0) + return num; + else + T1 = 0; + } + } + return num; +} + +int shrink_new_CAR(struct CAR_state *CAR_state, int num) +{ + CAR_state->C = num; + CAR_state->p = num / 2; + return 1; +} + +/************************** LIST FUNCTIONS ***************************************/ + +/* Unused */ +int check_sanity(struct CAR_state *CAR_state, enum TB_list type) +{ + struct list_head *tail = &(CAR_state->D[type].prev); + struct list_head *curr = &(CAR_state->D[type].next); + struct list_head *next = curr->next; + + printk("curr %lu , tail %lu, next %lu \n", + (list_entry(curr, struct pf_cache_object, list))->offset, + (list_entry(tail, struct pf_cache_object, list))->offset, + (list_entry(next, struct pf_cache_object, list))->offset); + + /* Check if struct page still in the tree etc */ + struct pf_cache_object *hand = + list_entry(curr, struct pf_cache_object, list); + struct page *treepage = + pf_offset2page(CAR_state->parent_mapping, hand->offset); + struct page *storedpage = hand->page; + if (!treepage) { + printk("tree page %lu is NULL.HOW \n", hand->offset); + return 0; + } + if (treepage != storedpage) { + printk("Page reference bad \n"); + printk("treepage->index %lu, storedpage->index %lu ", + treepage->index, storedpage->index); + } + return 0; +} + +/* Called only when moving a page to its shadow list.Returns the + object removed. */ +static struct pf_cache_object *move_hand(struct CAR_state *CAR_state, + enum TB_list type) +{ + struct list_head *mhead = &(CAR_state->D[type]); + struct list_head *head = mhead->next; /* first page */ + struct pf_cache_object *obj; + if (head == mhead) /*no pages present */ + return NULL; + obj = list_entry(head, struct pf_cache_object, list); + // check_sanity(CAR_state, type) ; + list_del(head); + CAR_state->size[type]--; + return obj; +} + +/* Discard a page in the shadow list */ +static int discard_LRU(struct CAR_state *CAR_state, enum TB_list type) +{ + struct list_head *head = &(CAR_state->D[type]); + struct list_head *hnext = head->next; /*Actual object */ + struct pf_cache_object *obj; + if (hnext == head) { /* list is empty, dont do anything! */ + // {printk("empty lru list to discard %d \n",type); + return 0; + } + obj = list_entry(hnext, struct pf_cache_object, list); + + list_del(hnext); + pf_free(obj); /* list entry? */ + CAR_state->size[type]--; + return 1; +} + +/* Return which shadow list @offset is present. 0 if none. */ +static int search_shadow_lists(struct CAR_state *CAR_state, pgoff_t offset) +{ + struct pf_cache_object *cursor; + struct list_head *head; + int type; + type = 2; /* B1=2, B2=3 */ + for (type = 2; type < NUM_CAR_LISTS; type++) { + head = &(CAR_state->D[type]); + list_for_each_entry(cursor, head, list) { + if (cursor && cursor->offset == offset) + return type; + } + } + return 0; +} + +/* Search for the object with matching offset. If found, remove it + from the list. IS THIS SAFE?*/ +static int search_remove_offset(struct CAR_state *CAR_state, enum TB_list type, + pgoff_t offset) +{ + struct pf_cache_object *cursor; + struct pf_cache_object *todel; + struct list_head *head; + + head = &(CAR_state->D[type]); + list_for_each_entry(cursor, head, list) { + if (cursor && cursor->offset == offset) { + todel = cursor; + goto del; + } + } + return 0; + del: + list_del(&(todel->list)); + return 1; +} + +/**********************************************************************************/ + +int car_access(struct page *page, struct address_space *mapping, pgoff_t offset) +{ + struct CAR_state *CAR_state = mapping->CAR_state; + if (!CAR_state) + return 0; +} + +static int isReferenced(struct pf_cache_object *obj) +{ + return PageActive(obj->page); + // return obj->referenced ; +} + +static int clearReference(struct pf_cache_object *obj) +{ + ClearPageActive(obj->page); + TestClearPageActive(obj->page); + obj->referenced = 0; + return 0; +} + +static void increase_p(struct CAR_state *CAR_state) +{ + unsigned long p = CAR_state->p; + int B_ratio; + if (!B1_size) + B_ratio = 0; + else + B_ratio = B2_size / B1_size; + if (B_ratio) + p = p + B_ratio; + else + p = p + 1; + if (p + 1 > CAR_state->C) + p = CAR_state->C - 1; + CAR_state->p = p; +} + +static void decrease_p(struct CAR_state *CAR_state) +{ + unsigned long p = CAR_state->p; + int B_ratio; + if (!B2_size) + B_ratio = 0; + else + B_ratio = B1_size / B2_size; + if (B_ratio) + p = p - B_ratio; + else + p = p - 1; + if (p < 2) + p = 1; + CAR_state->p = p; +} + +/********************************************************************************/ + +static int calc_hit_imbalance(struct CAR_state *CAR_state) +{ + int hit_ratio; + unsigned long total_hits; + unsigned long total_pages; + int i; + int ratios[NUM_CAR_BUCKETS] = { 0 }; + int ideal; + int cosine_simil; + + for (i = 0; i < NUM_CAR_BUCKETS; i++) { + total_hits += CAR_state->H[i]; + total_pages += CAR_state->B[i]; + } + total_pages -= 9; /* Initialized to 1 */ + hit_ratio = total_hits / total_pages; + printk("Hit ratio is %d. Other ratios are: \n", hit_ratio); + +} + +void CARLOCK(struct CAR_state *CAR_state) +{ + spin_lock(&CAR_state->carlock); +} + +int CARTRYLOCK(struct CAR_state *CAR_state) +{ + return spin_trylock(&CAR_state->carlock); +} + +void CARUNLOCK(struct CAR_state *CAR_state) +{ + spin_unlock(&CAR_state->carlock); +} + +/********************************************************************************/ + +/* Called from fadvise */ +int setCARType(struct file *file, enum CAR_filetype type) +{ + struct address_space *mapping = file->f_mapping; + if (!mapping) + return 0; + struct CAR_state *CAR_state = mapping->CAR_state; + if (!CAR_state) + return 0; + CAR_state->CAR_filetype = type; + + return (1 + type); +} + +/********************************************************************************/ diff --git a/mm/pfifo.c b/mm/pfifo.c new file mode 100644 index 0000000..07f3478 --- /dev/null +++ b/mm/pfifo.c @@ -0,0 +1,87 @@ +#include + +struct FIFO_state { + unsigned long size ; + struct address_space* mapping ; + struct list_head list ; +}; + + +static struct kmem_cache* slab; +static int first_post = 1 ; + +int FIFO_init(struct address_space* mapping) +{ + mapping->FIFO_state = kmalloc(sizeof(struct FIFO_state), GFP_KERNEL); + struct FIFO_state* FIFO_state = mapping->FIFO_state; + + if(first_post) { + slab = kmem_cache_create("pf_objects_fifo", sizeof(struct pf_cache_object), 0, 0, NULL); + first_post = 0 ; + } + if(FIFO_state == NULL) printk("CANOT INIT FIFO STATE \n"); + INIT_LIST_HEAD(&(FIFO_state->list)); + FIFO_state->mapping = mapping ; + FIFO_state->size = 0 ; + return 1 ; +} + + +int FIFO_remove(struct FIFO_state* FIFO_state, int max) +{ + if(max == 2) return 1 ; + struct list_head* lru = (FIFO_state->list).next ; + struct pf_cache_object* obj = list_entry( lru ,struct pf_cache_object,list) ; + printk("victim %lu , ", obj->offset) ; + struct page* page = pf_offset2page(FIFO_state->mapping, obj->offset); + if(page == NULL) { + printk("OMG NULL PAGE \n"); + } + if(page!=NULL && pf_remove_page(FIFO_state->mapping ,page)) { + list_del(lru) ; + FIFO_state->size-- ; + return 1 ; + } + else { + struct page* origpage = obj->page ; + if(origpage == NULL) { printk("WTF we didnt store a page?\n"); + return 0 ; } + printk(" origpage->mapping:%x,FIFO_state->mapping:%x,origpage->index:%lu,obj->offset:%lu\n",origpage->mapping,FIFO_state->mapping,origpage->index,obj->offset); + // printk("orig inode %lu . fifo inode %lu \n",FIFO_state->mapping->host->i_ino, origpage->mapping->host->i_ino); + if(origpage->mapping && origpage->index == obj->offset) { + printk("orig inode %lu \n", origpage->mapping->host->i_blocks); + pf_remove_page(origpage->mapping, origpage); + return 1 ; + } + if(origpage->mapping == FIFO_state->mapping && origpage->index == obj->offset) { + printk("PAGE HAS SAME INDEX PPL!\n"); + if(pf_remove_page(FIFO_state->mapping, origpage)) + return 1 ; + } + list_del(lru) ; + list_add_tail(lru, &(FIFO_state->list)); + // FIFO_remove(max-1); + } +} + + +int FIFO_insert(pgoff_t offset, struct page* page, struct address_space* mapping) +{ + struct FIFO_state* FIFO_state = mapping->FIFO_state ; + struct pf_cache_object* obj = kmem_cache_alloc(slab,GFP_KERNEL); + if(FIFO_state == NULL) { FIFO_init(mapping); printk("fifo null"); + FIFO_state = mapping->FIFO_state ;} + if(obj == NULL) printk("obj null"); + obj->offset = offset ; + obj->page = page ; + struct address_space* pagemapping = page->mapping ; + struct address_space* fifomapping = FIFO_state->mapping ; + list_add_tail( &(obj->list) , &(FIFO_state->list)); + FIFO_state->size++ ; + printk("Added offset %lu. Size %lu \n", offset, FIFO_state->size); + if(FIFO_state->size > 4000) { + FIFO_remove(FIFO_state, FIFO_state->size); + } + return 1 ; +} + diff --git a/mm/reaper.c b/mm/reaper.c new file mode 100644 index 0000000..aee3c61 --- /dev/null +++ b/mm/reaper.c @@ -0,0 +1,1135 @@ +/** + * Reaper thread, which frees pages belonging to files(address + * spaces). Reaping is done under memory pressure, periodically. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* for inode_has_buffers */ + +/******************************************************/ +#define rpdbg printk + +struct reaper_scan_cursor { + struct address_space *current_addr_space; + unsigned long seqnr; +}; + +/* The list with all the address_spaces. To be kept in sync. */ +static struct r_list { + struct list_head reaper_list; + int num_mappings; +}; + +static struct r_list *r_list = NULL; + +/**************** Global variables ***************/ + +static struct reaper_scan_cursor reaper_scan_cursor; + +static DEFINE_MUTEX(reaper_thread_mutex); +static DECLARE_WAIT_QUEUE_HEAD(reaper_thread_wait); +static DEFINE_SPINLOCK(reaper_list_lock); + +static int reaper_thread_inodes_to_scan = 100; +static int reaper_thread_sleep_millisecs = 100; +static int magic_file_grow = 100; + +static unsigned long reap_full_file(struct address_space *mapping); + +static int do_reaping(struct CAR_state *CAR_state, int K); + +struct s_rstat { + int F_SIZE; + long hits; + long misses; + long RA; /* Read ahead successes */ + int d_soft; + int d_hard; + int hard_alloc; + int soft_alloc; + int reap_target; + int total_culled; + int total_reaped; + int num_victims; + int eviction_failures; /* No pages evicted from a file */ + int total_pages_evicted; + int futile_list_searches; + int files; + int total_file_pages; + int ctr_mapped; + int ctr_wback; + int ctr_locked; + int ctr_dirty; + int ctr_private; + int ctr_freeze_refs; + int ctr_freed; + long old_nr_swap_pages; + int list_scans; + int second_chances; + int max_evicted_prev_scan; + int carlock_failures; + int f_grow_aggressive; + int anon_pressure ; + int dHits ; + int dMisses; + int dRA ; +}; + +struct s_rstat *rstat = NULL; + +struct filereaperr { + int ctr_mapped; + int ctr_wback; + int ctr_locked; + int ctr_dirty; + int ctr_private; + int ctr_freeze_refs; + int ctr_freed; +}; + +struct filereaperr *filereaperr = NULL; + +static int pages_to_reap = 0; + +/****************************************************************************/ +/** For reaper list protection */ +static unsigned long lock_flags; + +static void master_lock() +{ + spin_lock_irqsave(&reaper_list_lock, lock_flags); +} + +static void master_unlock() +{ + spin_unlock_irqrestore(&reaper_list_lock, lock_flags); +} + +/****************************************************************************/ + +void inc_pf_hard_alloc(void) +{ + rstat->hard_alloc++; + +} + +void inc_pf_soft_alloc(int n) +{ + rstat->soft_alloc += n; +} + +/***************** DEBUGGING AND TRACING ***************************/ + +static struct dentry *d_reaper; + +struct dentry *reaper_init_dentry(void) +{ + static int once; + + if (d_reaper) + return d_reaper; + + if (!debugfs_initialized()) + return NULL; + + d_reaper = debugfs_create_dir("reaper", NULL); + + if (!d_reaper && !once) { + once = 1; + pr_warning("Could not create debugfs directory 'reaper'\n"); + return NULL; + } + return d_reaper; +} + +#define MYPRINTLU(VAL) r = sprintf(buf, "%s : %lu \n", #VAL , rstat->VAL); \ + buf += r; \ + len += r; \ + + +static ssize_t rstats_read(struct file *file, char __user *ubuf, size_t cnt, loff_t *ppos) +{ + char *file_buf; + char *buf; /* Must kmalloc this */ + int len = 0; /*len accumulated read so far */ + int r; /* Read so far */ + + unsigned long foo = 12340; + + file_buf = kmalloc(800, GFP_KERNEL); + if (!file_buf) + return -ENOMEM; + buf = file_buf; + + MYPRINTLU(F_SIZE); + MYPRINTLU(hits); + MYPRINTLU(misses); + MYPRINTLU(RA); + MYPRINTLU(d_soft); + MYPRINTLU(d_hard); + MYPRINTLU(hard_alloc); + MYPRINTLU(soft_alloc); + MYPRINTLU(reap_target); + MYPRINTLU(total_culled); + MYPRINTLU(total_reaped); + MYPRINTLU(num_victims); + MYPRINTLU(eviction_failures); + MYPRINTLU(total_pages_evicted); + MYPRINTLU(futile_list_searches); + MYPRINTLU(files); + MYPRINTLU(total_file_pages); + MYPRINTLU(ctr_mapped); + MYPRINTLU(ctr_wback); + MYPRINTLU(ctr_locked); + MYPRINTLU(ctr_dirty); + MYPRINTLU(ctr_private); + MYPRINTLU(ctr_freeze_refs); + MYPRINTLU(ctr_freed); + MYPRINTLU(old_nr_swap_pages); + MYPRINTLU(list_scans); + MYPRINTLU(second_chances); + MYPRINTLU(max_evicted_prev_scan); + MYPRINTLU(carlock_failures); + MYPRINTLU(anon_pressure); + + len = simple_read_from_buffer(ubuf, cnt, ppos, file_buf, len); + kfree(file_buf); + return len; +} + +static const struct file_operations show_rstats_fops = { + .read = rstats_read, +}; + +struct dentry *debugfs_reaper_file_create(void) +{ + char *name = "rstats"; + mode_t mode = 0644; + struct dentry *parent = d_reaper; + void *data = NULL; + struct file_operations *fops = &show_rstats_fops; + struct dentry *rstat_file; + rstat_file = debugfs_create_file(name, mode, parent, data, fops); + return rstat_file; +} + + + +void init_debufs_stuff(void) +{ + reaper_init_dentry(); //creates the directory entry. + printk("REAPER DIRECTORY INITIALIZED \n"); + debugfs_reaper_file_create(); //creates the file + printk("REAPER FILE CREATED \n"); +} + +/*************** LIST MANAGEMENT ************************************/ + +static void init_reaper_magic(void) +{ + rstat->F_SIZE = 30000; /* 120 MB */ +} + +int init_master_list(void) +{ + rstat = kmalloc(sizeof(struct s_rstat), GFP_KERNEL); + init_reaper_magic(); + r_list = kmalloc(sizeof(struct r_list), GFP_KERNEL); + filereaperr = kmalloc(sizeof(struct filereaperr), GFP_KERNEL); + INIT_LIST_HEAD(&(r_list->reaper_list)); + + printk("initialized reaper list %lu \n", rstat->F_SIZE); + return 1; +} + +static int add_to_master_list(struct address_space *mapping) +{ + master_lock(); + if (!mapping) + goto out; + + if (mapping->host) { + if (!r_list) /* File IO via VFS starts before the thread init */ + init_master_list(); + + if (mapping->host->i_bdev) ; + + list_add_tail(&(mapping->reaper_list), &(r_list->reaper_list)); + } + out: + master_unlock(); + return 1; +} + +/* Remove from list, called during inode-sb-del. + * + * master lock gets released when carlock is taken during + * sync_do_reaping. We mark the mapping as in-use before releasing + * masterlock. During the removal(here), we spin on this (without + * actually taking the spinlock) BEFORE grabbing master lock and + * deleting from the sb list. NOTE that when this function is called, + * inode SBlist lock is already taken, so we cannot/willnot sleep, so + * the spinning while loop trick is safe from preemption and other + * races. + */ +int remove_from_reaper(struct address_space *mapping) +{ + while (mapping->eviction_in_progress) ; + + master_lock(); + list_del(&(mapping->reaper_list)); + master_unlock(); + + rstat->files--; + return 1; +} + +static int file_cache_full(void) +{ + if (rstat->hard_alloc > rstat->F_SIZE) { + return 1; + } + return 0; +} + +/****************************************************************************/ + +static int new_soft_crunch = 2; +static int new_hard_crunch = 4; + +/* Requested pages will be small. Allocate quickly without any + evictions. What if there are 100s of such calls during a reap scan + and we run out of space completely? +*/ +static int make_space_for_new_file(struct CAR_state *CAR_state) +{ + int freed; + int to_evict; + int EARLY_BOOT_SKIP ; + /* No reaping for files needed for early boot, which are actually + buffers anyway. */ + if (rstat->files < EARLY_BOOT_SKIP) + return 1; + + if (rstat->hard_alloc > rstat->F_SIZE) { + /**** Reaping already due + Some extra space for insurance */ + to_evict = rstat->hard_alloc - rstat->F_SIZE + (CAR_state->C) / 10; + freed = do_reaping(CAR_state, to_evict); + shrink_new_CAR(CAR_state, CAR_state->C / 2); + } + rstat->soft_alloc += CAR_state->C; + return 1; +} + +/* CAR_state is the file on who needs more space, so that we dont + evict from that file. NULL means this is a global request from + vmscan or something */ + +/* The problem with the per-file approach is that inter-file recency */ +/* and stack-distance is not easy to compare. A last-access time-stamp */ +/* gives a strict recency ordering, but not the reuse-distance. MU(n) is */ +/* defined as the number of HITS *after* adding block n. If a cache is */ +/* full, MU(n)==HITS. Otherwise, MU(n)=1 when a_i reaches n. Hopefully */ +/* the file sees a few hits by the time the reaper comes around. Anyway, */ +/* Marginal utility is the a moving average. The problem is thus of */ +/* efficiently ordering files by increasing utility. O(n2) is */ +/* prohibitively large, and read-side muss be lockless, so no */ +/* move-to-front. Given this, the next question is how many pages to evict */ +/* for a given file. MU=0 are prime candidates, but cant cull too many */ +/* pages because it might be unfair. Evicting single pages from cold */ +/* files is also illogical. Maybe increase the penalty for cold files? */ + +static int victim_reap_pages(struct address_space *mapping, int max) +{ + int to_reap; + struct CAR_state *CAR_state = mapping->CAR_state; + unsigned long hits = CAR_state->MU[0] + 1; + unsigned long total = mapping->nrpages; + unsigned long util_inv = total / hits; /* Inverse of the utility */ + // int tax = util_inv-1/util_inv ; + //update MU. + int clock_age = mapping->CLOCK_flags; /* 0 TO MAX. 4 is HOTTEST */ + int recl = (total + 1) / 2; + recl = recl / (clock_age + 1); +/*Lock overhead might be too much to give so many chances, better reap + * some number of pages now, even though the file is HOT */ + if (recl > max) + recl = max; + return recl; +} + +static void age_MU(struct address_space *mapping) +{ + struct CAR_state *CAR_state = mapping->CAR_state; + CAR_state->MU[1] = CAR_state->MU[1] + CAR_state->MU[0] / 2; + CAR_state->MU[0] = 0; //reset +} + +static unsigned long reap_full_file(struct address_space *mapping) +{ + //list_del(mapping->reaper_list, head); + // struct inode* inode = mapping->inode ; + unsigned long culled = invalidate_mapping_pages(mapping, 0, -1); + rstat->total_culled += culled; + return culled; +} + +static int test_numpages(void) +{ + struct list_head *head = &(r_list->reaper_list); + + struct address_space *curr; + + int reclaimed = 0; + + int r_i, s_i; /* reap pages from file i */ + int num_victims; + unsigned long culled, actual; + int i; + list_for_each_entry(curr, head, reaper_list) { + if (!curr->nrpages) + continue; + if (!curr->host) + continue; + if (curr->assoc_mapping) + continue; + rpdbg("Victm %d size %lu \n", curr->host->i_ino, curr->nrpages); + culled += curr->nrpages; + actual += curr->CAR_state->size[0] + curr->CAR_state->size[1]; + } + printk("pages under reaper %lu. Actual CAR pages %lu \n", culled, + actual); + return 1; +} + + +void dump_filereaperr(void) +{ + printk(" ctr_mapped:%d", filereaperr->ctr_mapped); + printk(" ctr_wback:%d", filereaperr->ctr_wback); + printk(" ctr_locked:%d", filereaperr->ctr_locked); + printk(" ctr_dirty:%d", filereaperr->ctr_dirty); + printk(" ctr_private:%d", filereaperr->ctr_private); + printk(" ctr_freeze_refs:%d", filereaperr->ctr_freeze_refs); + printk(" ctr_freed:%d\n", filereaperr->ctr_freed); +} + + +void clear_filereaperr(void) +{ + filereaperr->ctr_mapped = 0; + filereaperr->ctr_wback = 0; + filereaperr->ctr_locked = 0; + filereaperr->ctr_dirty = 0; + filereaperr->ctr_private = 0; + filereaperr->ctr_freeze_refs = 0; + filereaperr->ctr_freed = 0; +} + + +/* Managed to evict all the pages, so give the files a bit more clock + lifetime. Not required when the clock grace period is small(2) . */ +int reaper_success(void) +{ + return 1; + if (INODE_CLOCK_MIN > 0) + --INODE_CLOCK_MIN; + return 1; +} + +/* Reduce the file clock grace period because we failed to evict the + required number of pages.*/ +int reaper_failure(void) +{ + return 1; + INODE_CLOCK_MIN += 2; + if (INODE_CLOCK_MIN > INODE_CLOCK_MAX) + INODE_CLOCK_MIN = INODE_CLOCK_MAX; + return INODE_CLOCK_MIN; +} + +static int fi_do_fifo = 0; +int reaper_pressure; + +/* Function which actually does the page reclaim / eviction */ + +int sync_do_reaping(struct CAR_state *CAR_state, int K) +{ + struct list_head *rlist_head = &(r_list->reaper_list); + struct list_head *cursor = rlist_head->next; + struct list_head *ncur; //safe removal ? + struct address_space *curr; + + int target = K; + int reclaimed = 0; + // master_lock(); + int r_i, s_i; /* reap pages from file i */ + int num_victims; + unsigned long culled; + int i = 0; + int max_ev = 0; + master_lock(); + int actual_total_pages; + if (cursor == rlist_head) { + printk("EMPTY LIST \n"); + return 0; + } + struct inode *inode; + int r; + while (1) { + curr = + list_first_entry(cursor, struct address_space, reaper_list); + if (!curr) + break; + i++; + + list_del(&(curr->reaper_list)); + list_add_tail(&(curr->reaper_list), rlist_head); + + if (!curr->nrpages) + goto advance_cursor; + //if(!curr->host) goto advance_cursor ; + //if(curr->assoc_mapping) goto advance_cursor ; + + if (curr->CAR_state == CAR_state) { + /* CAR_state may be NULL. Also, its not nice to evict from a + file which requested the reaping in the first place */ + goto advance_cursor; + } + actual_total_pages += curr->nrpages; + + if (age_reaper_inode_clock(curr) > INODE_CLOCK_MIN) { /* Give more chances */ + if (!fi_do_fifo) { + rstat->second_chances++; + continue; + } + } + + /* This is the critical part. We mark the mapping as busy to + prevent against concurrent deletions for the mapping. The reaper + list 'master lock' is relinquished to prevent deadlocks, since + shrink_CAR causes IPI's on TLB shootdowns (arising from + unmapping mapped pages). + CPU1 : MLOCK--->CARLOCK-------------------->IPI + ^ | + CPU2: |------((MLOCK))spin with IRQ-off<---< + + The reason that MasterLOCK must be with IRQ disabled is to + prevent reaper from getting pre-empted during the reaper process + to prevent these kinds of scenarios: + + ________________________ + | | + CPU1 : MLOCK---->CARLOCK--->||PRE-EMPTED.|| CARLOCK + ^______________________ + | + CPU2 : ADD_TO_MASTER_LIST---->MLOCK + + This results in both the CPUs spinning on two different locks. + */ + /* MAX */ + r_i = victim_reap_pages(curr, target - reclaimed); + + curr->eviction_in_progress = 1; + master_unlock(); + + s_i = shrink_CAR(curr->CAR_state, r_i); + + master_lock(); + curr->eviction_in_progress = 0; + + if (s_i == 0) { + rstat->eviction_failures++; + //dump_filereaperr() ; /* Too many printks causing some kind of a deadlock?*/ + } else if (s_i < 0) + rstat->carlock_failures++; + else + reclaimed += s_i; + clear_filereaperr(); + age_MU(curr); /* Marginal Utility==Hits. Shift the array to the right */ + if (reclaimed >= target) { + reaper_success(); + fi_do_fifo = 0; + break; + } + if (s_i > max_ev) { + max_ev = s_i; + } + if (i > rstat->files) { + rstat->futile_list_searches++; + fi_do_fifo = 1; + reaper_failure(); /* Futile searches. reduce the clock min and try again */ + break; + } + advance_cursor: + continue; + } + master_unlock(); + + out: + rstat->total_reaped += reclaimed; + rstat->total_pages_evicted += reclaimed; /* Redundant, for sysfs interface */ + rstat->num_victims += i; + rstat->list_scans++; + + rstat->total_file_pages = actual_total_pages; + rstat->max_evicted_prev_scan = max_ev; + rstat->dHits=0; + rstat->dMisses=0; + rstat->dRA=0 ; + return reclaimed; +} + +int dec_reap_hard_alloc(void) +{ + rstat->hard_alloc--; +} + +/* The function called to free some pages. Because this is called from + places like CAR_init and CAR, it is better not to block by + synchronously evicting pages. Instead, simply add the required + number of pages for the reaper-thread to actually evict at a later + time. */ +int do_reaping(struct CAR_state *CAR_state, int K) +{ + pages_to_reap += K; + return K; +} + +static unsigned int switch_expand_file = 1; +static int CAR_MISS_DELAY = 10 ; + +/** Files dont get to ask how much they want to grow by. We allocate + * that. Non-blocking. + */ +static int expand_file(struct CAR_state *CAR_state) +{ + if (!switch_expand_file) /* Sysctl controllable knob to allow file expansion */ + return 0; + int to_grow = magic_file_grow; /* is a 100 */ + int pages_freed; + int FREE_BONUS ; + int dAccess = CAR_state->dAccess ; + int dMisses = CAR_state->dMisses ; + int dRA = CAR_state->dRA_success ; + int MU_i = dAccess/(dMisses+dRA+1) ; + int global_MU_i = (rstat->dHits+rstat->dMisses)/(rstat->dMisses+rstat->dRA) ; + enum CAR_filetype type = CAR_state->CAR_filetype; + if (type == PFT_small || type == PFT_dontneed) { + return 0; /* No pages for you. */ + } + int MU_inv = dAccess/(dMisses+1) ; + to_grow = (CAR_MISS_DELAY/2)*(dAccess+dMisses)/(dMisses+1) ; + /* This function is called only after a certain number of misses from a file. Say,10. Thus worst case slowdown is 10. If empty space exists, grab it? */ + if (file_cache_full()) { + pages_freed = do_reaping(CAR_state, to_grow); + CAR_state->C += pages_freed; + } /* Else, we have enough space. Allocate the request. */ + else { + rstat->soft_alloc += to_grow; + CAR_state->C += to_grow; + } + return pages_freed; +} + +/***************************************************/ + +int reaper_anon_inform(long scanned, long rotated) +{ + int ap ; + int anon_prio = 100 ; /* swapiness value */ + ap = (anon_prio + 1) * (scanned + 1); + ap /= rotated + 1; + rstat->anon_pressure = ap ; + return ap ; +} + +/********** DEPRECATED FUNCTIONS **********************/ +static int alloc_pages_from_file_cache(int C) +{ + inc_pf_soft_alloc(C); + return C; +} + +/* UNUSED */ +static int reap_pages(int num) +{ + int target = num; + struct list_head *head = &(r_list->reaper_list); + struct list_head *first = head->next; /* Actual mapping */ + //struct address_space* victim = list_entry(first, struct address_space, reaper_list) ; +} + +static int reap_fast(int num) +{ + return num; +} + +/* The system is deseprately short of memory. Evict num pages _now_ */ +static int reap_urgent(int num) +{ + +} + +/**************** PERIODIC SCANNING **********************************/ + +static int reaper_test = 0; /* Manual forced reaping */ +static int sysctl_increase_F = 1; + +static int decrease_F(long decrease, long old) +{ + rstat->F_SIZE -= (decrease - old); + return 1; +} + +/* Reduce F_size depending on swap activity. */ +static int shrink_F() +{ + if(!sysctl_increase_F) + return 0 ; + if (nr_swap_pages > rstat->old_nr_swap_pages) { + decrease_F(nr_swap_pages, rstat->old_nr_swap_pages); + return 1; + } + if(rstat->anon_pressure < 2) { + decrease_F(100,10); + } + return 0; +} + +static int fgrow_clock = 0; + +static int min_free = 10000; + +/* Increase F_size if there is enough free memory available. */ +static int increase_F(void) +{ + if (!sysctl_increase_F) + return 0; + if (!file_cache_full()) { /*Dont think of expanding when there's still some space */ + return 0; + } + if (fgrow_clock++ < 100) + return 0; + + fgrow_clock = 0; + int alpha = rstat->f_grow_aggressive; /*On scale from 0 to 9,default 0 */ + int num_free = global_page_state(NR_FREE_PAGES); + if (num_free < min_free) /* Keep atleast 40 MB free */ + return 0; + int to_grow = (num_free - min_free) / (20 - alpha); + if (to_grow > min_free) + to_grow = min_free; + rstat->F_SIZE += to_grow; + return to_grow; +} + +/* The body of the thread which is scheduled to run periodically. Reap + pages synchronously */ +static int reaper_do_scan(int to_scan) +{ + int success; + reaper_scan_cursor.seqnr++; +#ifdef XXX_PFC + + if (reaper_test != 0) { + sync_do_reaping(NULL, reaper_test); + reaper_test = 0; + } + if (pages_to_reap > 0) { /* incremented by do_reaping */ + success = sync_do_reaping(NULL, pages_to_reap); + pages_to_reap -= success; /* May be negative, but so what? */ + } + if (!shrink_F()) + increase_F(); + + // printk("In reaper do scan \n"); + return 1; +#endif + +} + +/****************** PUBLIC INTERFACE IMPLEMENTATION ********************/ + +/* Add a new address_space to the reaper's list, can also shrink other + * caches if required! + */ +int add_to_reaper(struct CAR_state *CAR_state) +{ + unsigned long C = CAR_state->C; /* C is small. Lets say 10 pages */ + struct address_space *mapping = CAR_state->parent_mapping; + + add_to_master_list(mapping); + rstat->files++; + return 1; //No changes +} + +/* This is the main entry point, the master function. + * Dispatches by event. Must be non-blocking/spinning + */ +int call_reaper(enum REAP_event event, void *arg) +{ + if (event == reap_new_file) { + // rpdbg("new-file %d. Size now %lu\n", rstat->files, rstat->hard_alloc); + make_space_for_new_file((struct CAR_state *)arg); + } + if (event == reap_periodic) { + + } + if (event == reap_file_grow) { + expand_file((struct CAR_state *)arg); + } + if (event == reap_overflow) { + + } + return 1; +} + +/***********************************************************************/ + +void bump_pf_hits(void) +{ + rstat->hits++; + rstat->dHits++ ; +} + +void bump_pf_misses(void) +{ + rstat->misses++; + rstat->dMisses++ ; +} + +void bump_pf_RA(void) +{ + rstat->RA++ ; + rstat->dRA++ ; +} + +void pferr_mapped(void) +{ + rstat->ctr_mapped++; + filereaperr->ctr_mapped++; +} + +void pferr_wback(void) +{ + rstat->ctr_wback++; + filereaperr->ctr_wback++; + +} + +void pferr_locked(void) +{ + rstat->ctr_locked++; + filereaperr->ctr_locked++; +} + +void pferr_dirty(void) +{ + rstat->ctr_dirty++; + filereaperr->ctr_dirty++; +} + +void pferr_private(void) +{ + rstat->ctr_private++; + filereaperr->ctr_private++; +} + +void pferr_freeze_refs(void) +{ + rstat->ctr_freeze_refs++; + filereaperr->ctr_freeze_refs++; +} + +void pferr_freed(void) +{ + rstat->ctr_freed++; + filereaperr->ctr_freed++; +} + +void dump_pf_eviction_stats(void) +{ + printk("------------------------------------------------\n"); + printk(" rstat->ctr_mapped %lu \n", rstat->ctr_mapped); + printk(" rstat->ctr_wback %lu \n", rstat->ctr_wback); + printk(" rstat->ctr_locked %lu \n", rstat->ctr_locked); + printk(" rstat->ctr_dirty %lu \n", rstat->ctr_dirty); + printk(" rstat->ctr_private %lu \n", rstat->ctr_private); + printk(" rstat->ctr_freeze_refs %lu \n", rstat->ctr_freeze_refs); + printk(" rstat->ctr_freed %lu \n", rstat->ctr_freed); +} + +/****************** THREAD MANAGEMENT **********************************/ + +static int reaper_should_run(void) +{ + return 1; +} + +static int reaper_scan_thread(void *nothing) +{ + set_freezable(); + set_user_nice(current, 0); + while (!kthread_should_stop()) { + mutex_lock(&reaper_thread_mutex); + if (reaper_should_run()) + reaper_do_scan(reaper_thread_inodes_to_scan); + mutex_unlock(&reaper_thread_mutex); + try_to_freeze(); + if (reaper_should_run()) { + schedule_timeout_interruptible(msecs_to_jiffies + (reaper_thread_sleep_millisecs)); + } else { + wait_event_freezable(reaper_thread_wait, + reaper_should_run() || kthread_should_stop()); + } + } + return 0; +} + +/******* Macros to create attributes (files) . This section from ksm.c *****/ + +#define REAPER_ATTR_RO(_name) \ + static struct kobj_attribute _name##_attr = __ATTR_RO(_name) +#define REAPER_ATTR(_name) \ + static struct kobj_attribute _name##_attr = \ + __ATTR(_name, 0644, _name##_show, _name##_store) + +static ssize_t full_scans_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%lu\n", reaper_scan_cursor.seqnr); +} + +REAPER_ATTR_RO(full_scans); + +static ssize_t num_mapping_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%lu\n", rstat->files); +} + +REAPER_ATTR_RO(num_mapping); + +static ssize_t file_pages_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%lu\n", rstat->hard_alloc); +} + +REAPER_ATTR_RO(file_pages); + +static ssize_t F_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%lu\n", rstat->F_SIZE); +} + +static ssize_t F_store(struct kobject *kobj, + struct kobj_attribute *attr, char *buf, size_t count) +{ + int err; + unsigned long val; + err = strict_strtoul(buf, 10, &val); + if (val != 0) { + rstat->F_SIZE = val; + } + return count; +} + +REAPER_ATTR(F); + +static ssize_t reap_run_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%lu\n", reaper_test); +} + +static ssize_t reap_run_store(struct kobject *kobj, + struct kobj_attribute *attr, char *buf, size_t count) +{ + int err; + unsigned long val; + err = strict_strtoul(buf, 10, &val); + if (val != 0) { + reaper_test = val; + } + return count; +} + +REAPER_ATTR(reap_run); + +static ssize_t total_numpages_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%lu\n", reaper_test); +} + +static ssize_t total_numpages_store(struct kobject *kobj, + struct kobj_attribute *attr, char *buf, size_t count) +{ + int err; + unsigned long val; + err = strict_strtoul(buf, 10, &val); + if (val) + test_numpages(); + return count; +} + +REAPER_ATTR(total_numpages); + +static ssize_t total_culled_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%lu\n", rstat->total_culled); +} + +REAPER_ATTR_RO(total_culled); + +static ssize_t total_reaped_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%lu\n", rstat->total_reaped); +} + +REAPER_ATTR_RO(total_reaped); + +static ssize_t pf_eviction_stats_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + unsigned long pf_eviction_stats = 0; + return sprintf(buf, "%lu\n", pf_eviction_stats); +} + +static ssize_t pf_eviction_stats_store(struct kobject *kobj, + struct kobj_attribute *attr, char *buf, size_t count) +{ + int err; + unsigned long val; + err = strict_strtoul(buf, 10, &val); + if (val) { + dump_pf_eviction_stats(); + } + return count; +} + +REAPER_ATTR(pf_eviction_stats); + +static ssize_t expand_file_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + unsigned long pf_eviction_stats = 0; + return sprintf(buf, "%lu\n", switch_expand_file); +} + +static ssize_t expand_file_store(struct kobject *kobj, + struct kobj_attribute *attr, char *buf, size_t count) +{ + int err; + unsigned long val; + err = strict_strtoul(buf, 10, &val); + switch_expand_file = val; + return count; +} + +REAPER_ATTR(expand_file); + +static ssize_t increase_F_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%lu\n", sysctl_increase_F); +} + +static ssize_t increase_F_store(struct kobject *kobj, + struct kobj_attribute *attr, char *buf, size_t count) +{ + int err; + unsigned long val; + err = strict_strtoul(buf, 10, &val); + sysctl_increase_F = val; + return count; +} + +REAPER_ATTR(increase_F); + +static ssize_t hits_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%lu\n", rstat->hits); +} +REAPER_ATTR_RO(hits); + +static ssize_t misses_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%lu\n", rstat->misses); +} +REAPER_ATTR_RO(misses); + +static ssize_t RA_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%lu\n", rstat->RA); +} +REAPER_ATTR_RO(RA); + + +/*Pointers to all the sysfs file attributes */ +static struct attribute *reaper_attrs[] = { + &full_scans_attr.attr, + &num_mapping_attr.attr, + &file_pages_attr.attr, + &F_attr.attr, + &reap_run_attr.attr, + &total_culled_attr.attr, + &total_reaped_attr.attr, + &total_numpages_attr.attr, + &pf_eviction_stats_attr.attr, + &expand_file_attr.attr, + &increase_F_attr.attr, + &hits_attr.attr, + &misses_attr.attr, + &RA_attr.attr, + NULL, +}; + +/* For the sysfs directory*/ +static struct attribute_group reaper_attr_group = { + .attrs = reaper_attrs, + .name = "reaper", +}; + +static int __init reaper_init(void) +{ + struct task_struct *reaper_thread; + int err; + + reaper_thread = kthread_run(reaper_scan_thread, NULL, "pf_reaper"); + if (IS_ERR(reaper_thread)) { + printk(KERN_ERR "Pf reaper: creating kthread failed\n"); + err = PTR_ERR(reaper_thread); + goto out; + } +#ifdef CONFIG_SYSFS + err = sysfs_create_group(mm_kobj, &reaper_attr_group); /* Creates directory */ + if (err) { + printk(KERN_ERR "pf reaper: register sysfs failed\n"); + kthread_stop(reaper_thread); + goto out; + } +#endif /* CONFIG_SYSFS */ + printk("Reaper thread initialized \n"); + rstat = kmalloc(sizeof(struct s_rstat), GFP_KERNEL); + if (!rstat) + goto out; + init_master_list(); + init_reaper_magic(); + init_debufs_stuff(); + return 0; + + out: + return err; +} + +module_init(reaper_init) diff --git a/mm/truncate.c b/mm/truncate.c index 632b15e..d428704 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -374,6 +374,7 @@ unsigned long invalidate_mapping_pages(struct address_space *mapping, cond_resched(); index++; } + // printk("invmap %lu from %x\n",count,mapping->host); return count; } EXPORT_SYMBOL(invalidate_mapping_pages); diff --git a/mm/vmscan.c b/mm/vmscan.c index cb33d9c..515ac38 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1,3 +1,4 @@ +//for git /* * linux/mm/vmscan.c * @@ -48,7 +49,7 @@ #include #include - +#include #include "internal.h" #define CREATE_TRACE_POINTS @@ -738,7 +739,7 @@ static noinline_for_stack void free_page_list(struct list_head *free_pages) list_for_each_entry_safe(page, tmp, free_pages, lru) { list_del(&page->lru); if (!pagevec_add(&freed_pvec, page)) { - __pagevec_free(&freed_pvec); + __pagevec_free(&freed_pvec); //calls free_hot_cold_pages pagevec_reinit(&freed_pvec); } } @@ -979,6 +980,7 @@ free_it: cull_mlocked: if (PageSwapCache(page)) try_to_free_swap(page); + unlock_page(page); putback_lru_page(page); reset_reclaim_mode(sc); @@ -1136,8 +1138,8 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan, mem_cgroup_rotate_lru_list(page, page_lru(page)); continue; - default: - BUG(); + //default: + //BUG(); } if (!order) @@ -1542,6 +1544,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct zone *zone, local_irq_disable(); if (current_is_kswapd()) __count_vm_events(KSWAPD_STEAL, nr_reclaimed); + __count_zone_vm_events(PGSTEAL, zone, nr_reclaimed); putback_lru_pages(zone, sc, nr_anon, nr_file, &page_list); @@ -1911,7 +1914,7 @@ static void get_scan_count(struct zone *zone, struct scan_control *sc, */ anon_prio = vmscan_swappiness(sc); file_prio = 200 - vmscan_swappiness(sc); - + /* * OK, so we have swap space and a fair amount of page cache * pages. We use the recently rotated / recently scanned @@ -1923,12 +1926,16 @@ static void get_scan_count(struct zone *zone, struct scan_control *sc, * * anon in [0], file in [1] */ +#ifdef XXX_PFC + reaper_anon_inform(reclaim_stat->recent_scanned[0], reclaim_stat->recent_rotated[0]); +#endif + spin_lock_irq(&zone->lru_lock); if (unlikely(reclaim_stat->recent_scanned[0] > anon / 4)) { reclaim_stat->recent_scanned[0] /= 2; reclaim_stat->recent_rotated[0] /= 2; } - + if (unlikely(reclaim_stat->recent_scanned[1] > file / 4)) { reclaim_stat->recent_scanned[1] /= 2; reclaim_stat->recent_rotated[1] /= 2;