Nullfs now uses pager.

4 files changed, 219 insertions, 104 deletions

diff --git a/src/kernel/user/elf.c b/src/kernel/user/elf.c
index a3dada7..ed9d3c4 100644
--- a/src/kernel/user/elf.c
+++ b/src/kernel/user/elf.c
@@ -42,9 +42,7 @@ proc_entry_t elf_load(fs_handle_t *f, process_t* process) {
 				size_t read_r = file_read(f, phdr.p_offset, phdr.p_filesz, (char*)phdr.p_vaddr);
 				if (read_r != phdr.p_filesz) goto error;
 
-				if (phdr.p_memsz > phdr.p_filesz) {
-					memset((char*)phdr.p_vaddr + phdr.p_filesz, 0, phdr.p_memsz - phdr.p_filesz);
-				}
+				// no need to zero out extra portion, paging code does that for us
 
 				if (!(phdr.p_flags & PF_W)) {
 					bool mchmap_ok = mchmap(process, (void*)phdr.p_vaddr,
diff --git a/src/kernel/user/nullfs.c b/src/kernel/user/nullfs.c
index 1f40d88..fa6a088 100644
--- a/src/kernel/user/nullfs.c
+++ b/src/kernel/user/nullfs.c
@@ -4,6 +4,7 @@
 #include <debug.h>
 
 #include <nullfs.h>
+#include <pager.h>
 
 // nullfs driver
 static bool nullfs_fs_make(fs_handle_t *source, const char* opts, fs_t *d);
@@ -64,8 +65,8 @@ static fs_node_ops_t nullfs_f_ops = {
 	.create = 0,
 	.delete = 0,
 	.move = 0,
-	.read = nullfs_f_read,
-	.write = nullfs_f_write,
+	.read = fs_read_from_pager,
+	.write = fs_write_to_pager,
 	.close = nullfs_f_close,
 	.readdir = 0,
 	.ioctl =0,
@@ -85,6 +86,7 @@ typedef struct nullfs_item {
 	char* name;
 	fs_node_ptr data;
 	fs_node_ops_t *ops;
+	pager_t *pager;
 
 	struct nullfs_item *next;
 } nullfs_item_t;
@@ -101,9 +103,8 @@ typedef struct {
 } nullfs_dir_t;
 
 typedef struct {
-	char* data;
-	size_t size;
-	bool own_data;
+	pager_t *pager;
+
 	int ok_modes;
 	
 	mutex_t lock;		// locked on open
@@ -161,7 +162,7 @@ void nullfs_fs_shutdown(fs_ptr fs) {
 	// TODO free all
 }
 
-bool nullfs_add_node(fs_t *fs, const char* name, fs_node_ptr data, fs_node_ops_t *ops) {
+bool nullfs_add_node(fs_t *fs, const char* name, fs_node_ptr data, fs_node_ops_t *ops, pager_t *pager) {
 	char file_name[DIR_MAX];
 
 	fs_node_t *n = fs_walk_path_except_last(fs->root, name, file_name);
@@ -179,6 +180,7 @@ bool nullfs_add_node(fs_t *fs, const char* name, fs_node_ptr data, fs_node_ops_t
 
 	i->data = data;
 	i->ops = ops;
+	i->pager = pager;
 
 	bool add_ok = hashtbl_add(d->items_idx, i->name, i);
 	if (!add_ok) goto error;
@@ -196,31 +198,30 @@ error:
 	return false;
 }
 
-bool nullfs_add_ram_file(fs_t *fs, const char* name, char* data, size_t init_sz, bool copy, int ok_modes) {
+bool nullfs_add_ram_file(fs_t *fs, const char* name, const char* data, size_t init_sz, int ok_modes) {
+	pager_t *p = 0;
+
 	nullfs_file_t *f = (nullfs_file_t*)malloc(sizeof(nullfs_file_t));
 	if (f == 0) return false;
+	
+	p = new_swap_pager(init_sz);
+	if (p == 0) goto error;
 
-	f->size = init_sz;
-	if (copy) {
-		f->data = malloc(init_sz);
-		memcpy(f->data, data, init_sz);
-		f->own_data = true;
-	} else {
-		f->data = data;
-		f->own_data = false;
-	}
-	f->ok_modes = ok_modes &
-		(FM_TRUNC | FM_READ | FM_WRITE);	// no MMAP support
+	f->ok_modes = ok_modes & (FM_MMAP | FM_TRUNC | FM_READ | FM_WRITE);
 	f->lock = MUTEX_UNLOCKED;
+	f->pager = p;
 
-	bool add_ok = nullfs_add_node(fs, name, f, &nullfs_f_ops);
-	if (!add_ok) {
-		if (f->own_data) free(f->data);
-		free(f);
-		return false;
-	}
+	bool add_ok = nullfs_add_node(fs, name, f, &nullfs_f_ops, p);
+	if (!add_ok) goto error;
+
+	pager_write(p, 0, init_sz, data);
 
 	return true;
+
+error:
+	if (p) delete_pager(p);
+	if (f) free(f);
+	return false;
 }
 
 //   -- Directory node --
@@ -262,6 +263,7 @@ bool nullfs_d_walk(fs_node_ptr n, const char* file, struct fs_node *node_d) {
 
 	node_d->ops = x->ops;
 	node_d->data = x->data;
+	node_d->pager = x->pager;
 
 	mutex_unlock(&d->lock);
 
@@ -290,7 +292,7 @@ bool nullfs_d_delete(fs_node_ptr n, const char* file) {
 		nullfs_file_t* f = (nullfs_file_t*)i->data;
 		if (!mutex_try_lock(&f->lock)) goto error;	// in use
 
-		if (f->own_data) free(f->data);
+		delete_pager(f->pager);
 		free(f);
 	} else {
 		goto error;		// special nodes (devices, ...) may not be deleted
@@ -332,13 +334,16 @@ bool nullfs_d_create(fs_node_ptr n, const char* file, int type) {
 	if (type == FT_REGULAR) {
 		mutex_lock(&d->lock);
 
+		pager_t *p = 0;
+
 		nullfs_file_t *f = (nullfs_file_t*)malloc(sizeof(nullfs_file_t));
 		if (f == 0) goto f_error;
 
+		p = new_swap_pager(0);
+		if (p == 0) goto f_error;
+
 		f->ok_modes = FM_READ | FM_WRITE | FM_TRUNC | FM_APPEND;
-		f->data = 0;
-		f->size = 0;
-		f->own_data = false;
+		f->pager = p;
 		f->lock = MUTEX_UNLOCKED;
 
 		i = (nullfs_item_t*)malloc(sizeof(nullfs_item_t));
@@ -349,6 +354,7 @@ bool nullfs_d_create(fs_node_ptr n, const char* file, int type) {
 
 		i->ops = &nullfs_f_ops;
 		i->data = f;
+		i->pager = p;
 
 		bool add_ok = hashtbl_add(d->items_idx, i->name, i);
 		if (!add_ok) goto f_error;
@@ -363,6 +369,7 @@ bool nullfs_d_create(fs_node_ptr n, const char* file, int type) {
 		if (i != 0 && i->name != 0) free(i->name);
 		if (i != 0) free(i);
 		if (f != 0) free(f);
+		if (p != 0) delete_pager(p);
 		mutex_unlock(&d->lock);
 		return false;
 	} else if (type == FT_DIR) {
@@ -459,10 +466,7 @@ bool nullfs_f_open(fs_node_ptr n, int mode) {
 		// truncate file
 		mutex_lock(&f->lock);
 
-		if (f->own_data) free(f->data);
-		f->size = 0;
-		f->own_data = false;
-		f->data = 0;
+		pager_resize(f->pager, 0);
 
 		mutex_unlock(&f->lock);
 	}
@@ -476,7 +480,7 @@ bool nullfs_f_stat(fs_node_ptr n, stat_t *st) {
 
 	st->type = FT_REGULAR;
 	st->access = f->ok_modes;
-	st->size = f->size;
+	st->size = f->pager->size;
 
 	mutex_unlock(&f->lock);
 	return true;
@@ -488,52 +492,6 @@ void nullfs_f_dispose(fs_node_ptr n) {
 
 //   -- File handle --
 
-static size_t nullfs_f_read(fs_handle_t *h, size_t offset, size_t len, char* buf) {
-	nullfs_file_t *f = (nullfs_file_t*)h->node->data;
-	mutex_lock(&f->lock);
-
-	size_t ret = 0;
-	
-	if (offset >= f->size) goto end_read;
-	if (offset + len > f->size) len = f->size - offset;
-
-	memcpy(buf, f->data + offset, len);
-	ret = len;
-
-end_read:
-	mutex_unlock(&f->lock);
-	return ret;
-}
-
-static size_t nullfs_f_write(fs_handle_t *h, size_t offset, size_t len, const char* buf) {
-	nullfs_file_t *f = (nullfs_file_t*)h->node->data;
-	mutex_lock(&f->lock);
-
-	size_t ret = 0;
-
-	if (offset + len > f->size) {
-		// resize buffer (zero out new portion)
-		void* new_buffer = malloc(offset + len);
-		if (new_buffer == 0) goto end_write;
-
-		memcpy(new_buffer, f->data, f->size);
-		if (offset > f->size)
-			memset(new_buffer + f->size, 0, offset - f->size);
-
-		if (f->own_data) free(f->data);
-		f->data = new_buffer;
-		f->own_data = true;
-		f->size = offset + len;
-	}
-
-	memcpy(f->data + offset, buf, len);
-	ret = len;
-
-end_write:
-	mutex_unlock(&f->lock);
-	return ret;
-}
-
 static void nullfs_f_close(fs_handle_t *h) {
 	// nothing to do
 }
diff --git a/src/kernel/user/pager.c b/src/kernel/user/pager.c
index c892599..f03e45b 100644
--- a/src/kernel/user/pager.c
+++ b/src/kernel/user/pager.c
@@ -15,12 +15,14 @@
 static void swap_page_in(pager_t *p, size_t offset, size_t len);
 static void swap_page_out(pager_t *p, size_t offset, size_t len);
 static void swap_page_release(pager_t *p, size_t offset, size_t len);
+static bool swap_resize(pager_t *p, size_t new_size);
 
 static pager_ops_t swap_pager_ops = {
 	.page_in = swap_page_in,
 	.page_commit = 0,
 	.page_out = swap_page_out,
 	.page_release = swap_page_release,
+	.resize = swap_resize,
 };
 
 pager_t *new_swap_pager(size_t size) {
@@ -44,17 +46,15 @@ error:
 
 void swap_page_in(pager_t *p, size_t offset, size_t len) {
 	ASSERT(PAGE_ALIGNED(offset));
-	size_t npages = PAGE_ALIGN_UP(len) / PAGE_SIZE;
 
-	void *region = region_alloc(PAGE_SIZE, "Page zero area", 0);
+	void *region = region_alloc(PAGE_SIZE, "Page zeroing area", 0);
 	if (region == 0) return;
 
-	for (size_t i = 0; i < npages; i++) {
-		size_t page = offset + (i * PAGE_SIZE);
+	for (size_t page = offset; page < offset + len; page += PAGE_SIZE) {
 		if (hashtbl_find(p->pages, (void*)page) == 0) {
 			uint32_t frame = frame_alloc(1);
 			if (frame != 0) {
-				pd_map_page(region, frame, true);
+				if (!pd_map_page(region, frame, true)) PANIC("TODO");;
 				memset(region, 0, PAGE_SIZE);
 				pd_unmap_page(region);
 			}
@@ -73,10 +73,8 @@ void swap_page_out(pager_t *p, size_t offset, size_t len) {
 
 void swap_page_release(pager_t *p, size_t offset, size_t len) {
 	ASSERT(PAGE_ALIGNED(offset));
-	size_t npages = PAGE_ALIGN_UP(len) / PAGE_SIZE;
 
-	for (size_t i = 0; i < npages; i++) {
-		size_t page = offset + (i * PAGE_SIZE);
+	for (size_t page = offset; page < offset + len; page += PAGE_SIZE) {
 		uint32_t frame = (uint32_t)hashtbl_find(p->pages, (void*)page) >> ENT_FRAME_SHIFT;
 		if (frame != 0) {
 			hashtbl_remove(p->pages, (void*)page);
@@ -85,6 +83,79 @@ void swap_page_release(pager_t *p, size_t offset, size_t len) {
 	}
 }
 
+bool swap_resize(pager_t *p, size_t new_size) {
+	// later : remove unused pages in swap file
+
+	swap_page_release(p, PAGE_ALIGN_UP(new_size), p->size - PAGE_ALIGN_UP(new_size));
+
+	size_t last_page = PAGE_ALIGN_DOWN(new_size);
+
+	if (!PAGE_ALIGNED(new_size) && hashtbl_find(p->pages, (void*)last_page) != 0) {
+		void *region = region_alloc(PAGE_SIZE, "Page zeroing area", 0);
+		if (!region) PANIC("TODO");
+
+		uint32_t frame = (uint32_t)hashtbl_find(p->pages, (void*)last_page) >> ENT_FRAME_SHIFT;
+		ASSERT(frame != 0);
+
+		if (!pd_map_page(region, frame, true)) PANIC("TODO");
+
+		size_t b0 = new_size - last_page;
+		memset(region + b0, 0, PAGE_SIZE - b0);
+
+		pd_unmap_page(region);
+
+		region_free(region);
+	}
+
+	return true;
+}
+
+// ============ //
+// DEVICE PAGER //
+// ============ //
+
+static void device_page_in(pager_t *p, size_t offset, size_t len);
+
+static pager_ops_t device_pager_ops = {
+	.page_in = device_page_in,
+	.page_commit = 0,
+	.page_out = 0,
+	.page_release = 0,
+	.resize = 0,
+};
+
+pager_t *new_device_page(size_t size, size_t phys_offset) {
+	ASSERT(PAGE_ALIGNED(phys_offset));
+
+	pager_t *p = (pager_t*)malloc(sizeof(pager_t));
+	if (p == 0) return 0;
+
+	p->pages = create_hashtbl(id_key_eq_fun, id_hash_fun, 0);
+	if (p->pages == 0) goto error;
+
+	p->ops = &device_pager_ops;
+	p->size = size;
+	p->lock = MUTEX_UNLOCKED;
+	p->maps = 0;
+	p->device_pager.phys_offset = phys_offset;
+
+	return p;
+
+error:
+	if (p) free(p);
+	return 0;
+}
+
+void device_page_in(pager_t *p, size_t offset, size_t len) {
+	ASSERT(PAGE_ALIGNED(offset));
+
+	for (size_t page = offset; page < offset + len; page += PAGE_SIZE) {
+		if (hashtbl_find(p->pages, (void*)page) == 0) {
+			hashtbl_add(p->pages, (void*)page, (void*)(p->device_pager.phys_offset + page));
+		}
+	}
+}
+
 // ======================= //
 // GENERIC PAGER FUNCTIONS //
 // ======================= //
@@ -92,23 +163,33 @@ void swap_page_release(pager_t *p, size_t offset, size_t len) {
 void delete_pager(pager_t *p) {
 	ASSERT(p->maps == 0);
 
-	pager_commit_dirty(p);
-
 	mutex_lock(&p->lock);
 
-	p->ops->page_release(p, 0, p->size);
+	if (p->ops->page_commit) p->ops->page_commit(p, 0, p->size);
+	if (p->ops->page_release) p->ops->page_release(p, 0, p->size);
+
 	delete_hashtbl(p->pages);
 	free(p);
 }
 
+bool pager_resize(pager_t *p, size_t newsize) {
+	if (!p->ops->resize) return false;
+
+	mutex_lock(&p->lock);
+
+	bool ret = p->ops->resize(p, newsize);
+
+	mutex_unlock(&p->lock);
+	
+	return ret;
+}
+
 void pager_access(pager_t *p, size_t offset, size_t len, bool accessed, bool dirty) {
 	mutex_lock(&p->lock);
 
 	ASSERT(PAGE_ALIGNED(offset));
-	size_t npages = PAGE_ALIGN_UP(len) / PAGE_SIZE;
 
-	for (size_t i = 0; i < npages; i++) {
-		size_t page = offset + (i * PAGE_SIZE);
+	for (size_t page = offset; page < offset + len; page += PAGE_SIZE) {
 		uint32_t v = (uint32_t)hashtbl_find(p->pages, (void*)page);
 		if (v != 0) {
 			hashtbl_change(p->pages, (void*)page,
@@ -124,14 +205,7 @@ void pager_commit_dirty(pager_t *p) {
 
 	mutex_lock(&p->lock);
 
-	for (size_t i = 0; i < p->size; i += PAGE_SIZE) {
-		uint32_t v = (uint32_t)hashtbl_find(p->pages, (void*)i);
-		if (v & PAGE_DIRTY) {
-			p->ops->page_commit(p, i, PAGE_SIZE);
-			hashtbl_change(p->pages, (void*)i, (void*)(v & ~PAGE_DIRTY));
-		}
-
-	}
+	p->ops->page_commit(p, 0, p->size);
 
 	mutex_unlock(&p->lock);
 }
@@ -153,6 +227,59 @@ int pager_get_frame(pager_t *p, size_t offset) {
 	return ret;
 }
 
+static size_t pager_do_rw(pager_t *p, size_t offset, size_t len, char* buf, bool write) {
+	size_t ret = 0;
+	void* region = 0;
+
+	mutex_lock(&p->lock);
+
+	if (offset >= p->size) return 0;
+	if (offset + len > p->size) len = p->size - offset;
+
+	size_t first_page = PAGE_ALIGN_DOWN(offset);
+	size_t region_len = offset + len - first_page;
+
+	region = region_alloc(PAGE_ALIGN_UP(region_len), "Temporary pager read/write zone", 0);
+	if (region == 0) goto end_read;
+
+	p->ops->page_in(p, first_page, region_len);
+	for (size_t i = first_page; i < first_page + region_len; i += PAGE_SIZE) {
+		uint32_t frame = (uint32_t)hashtbl_find(p->pages, (void*)i) >> ENT_FRAME_SHIFT;
+		if (frame == 0) goto end_read;
+		if (!pd_map_page(region + i - first_page, frame, write)) goto end_read;
+	}
+
+	if (write) {
+		memcpy(region + offset - first_page, buf, len);
+		for (size_t page = first_page; page < first_page+ region_len; page += PAGE_SIZE) {
+			uint32_t v = (uint32_t)hashtbl_find(p->pages, (void*)page);
+			ASSERT(v != 0);
+			hashtbl_change(p->pages, (void*)page, (void*)(v | PAGE_ACCESSED));
+		}
+	} else {
+		memcpy(buf, region + offset - first_page, len);
+	}
+	ret = len;
+
+end_read:
+	if (region) {
+		for (void* x = region; x < region + region_len; x += PAGE_SIZE) {
+			if (pd_get_frame(x) != 0) pd_unmap_page(x);
+		}
+		region_free(region);
+	}
+	mutex_unlock(&p->lock);
+	return ret;
+}
+
+size_t pager_read(pager_t *p, size_t offset, size_t len, char* buf) {
+	return pager_do_rw(p, offset, len, buf, false);
+}
+
+size_t pager_write(pager_t *p, size_t offset, size_t len, const char* buf) {
+	return pager_do_rw(p, offset, len, (char*)buf, true);
+}
+
 void pager_add_map(pager_t *p, user_region_t *reg) {
 	mutex_lock(&p->lock);
 
@@ -180,4 +307,18 @@ void pager_rm_map(pager_t *p, user_region_t *reg) {
 	mutex_unlock(&p->lock);
 }
 
+//  ---- For use by FS drivers
+
+size_t fs_read_from_pager(fs_handle_t *f, size_t offset, size_t len, char* buf) {
+	ASSERT(f->node->pager != 0);
+
+	return pager_read(f->node->pager, offset, len, buf);
+}
+size_t fs_write_to_pager(fs_handle_t *f, size_t offset, size_t len, const char* buf) {
+	ASSERT(f->node->pager != 0);
+	if (offset + len > f->node->pager->size) pager_resize(f->node->pager, offset + len);
+
+	return pager_write(f->node->pager, offset, len, buf);
+}
+
 /* vim: set ts=4 sw=4 tw=0 noet :*/
diff --git a/src/kernel/user/process.c b/src/kernel/user/process.c
index 1b3013b..7433517 100644
--- a/src/kernel/user/process.c
+++ b/src/kernel/user/process.c
@@ -639,6 +639,24 @@ bool munmap(process_t *proc, void* addr) {
 	return true;
 }
 
+void dbg_dump_proc_memmap(process_t *proc) {
+	//WARNING not thread safe
+
+	dbg_printf("/ Region map for process %d\n", proc->pid);
+
+	for (user_region_t *it = proc->regions; it != 0; it = it->next_in_proc) {
+		dbg_printf("| 0x%p - 0x%p : (0x%p) ", it->addr, it->addr + it->size, it->pager);
+		if (it->file != 0) {
+			dbg_printf("mmap of 0x%p", it->file);
+		} else {
+			dbg_printf("private");
+		}
+		dbg_printf(", %d pages\n", hashtbl_count(it->pager->pages));
+	}
+	
+	dbg_printf("\\ ----\n");
+}
+
 // =============================== //
 // USER MEMORY PAGE FAULT HANDLERS //
 // =============================== //