Kernel: Improve "out of memory" error handling

It can now boot with fairly low memory while not crashing and instead
gracefully failing the operation. Userland still does not properly
handle errors.

7 files changed, 55 insertions, 40 deletions

diff --git a/kernel/cpu/idt.c b/kernel/cpu/idt.c
index b668c6e..ce21d08 100644
--- a/kernel/cpu/idt.c
+++ b/kernel/cpu/idt.c
@@ -53,6 +53,10 @@ void general_protection_fault(reg_t *regs) {
   klog("General Protetion Fault", 0x1);
   kprintf(" Error Code: %x\n", regs->error_code);
   kprintf("Instruction Pointer: %x\n", regs->eip);
+  if (current_task) {
+    kprintf("PID: %x\n", current_task->pid);
+    kprintf("Name: %s\n", current_task->program_name);
+  }
   dump_backtrace(12);
   halt();
   EOI(0xD - 8);
diff --git a/kernel/kmalloc.c b/kernel/kmalloc.c
index 84a16bd..124e5a5 100644
--- a/kernel/kmalloc.c
+++ b/kernel/kmalloc.c
@@ -59,7 +59,7 @@ int init_heap(void) {
 }
 
 int add_heap_memory(size_t min_desired) {
-  min_desired += sizeof(MallocHeader) + 0x2000;
+  min_desired += sizeof(MallocHeader);
   size_t allocation_size = max(min_desired, NEW_ALLOC_SIZE);
   allocation_size += delta_page(allocation_size);
   void *p;
@@ -157,7 +157,6 @@ void *int_kmalloc(size_t s) {
   if (!free_entry) {
     if (!add_heap_memory(s)) {
       klog("Ran out of memory.", LOG_ERROR);
-      assert(0);
       return NULL;
     }
     return kmalloc(s);
diff --git a/kernel/lib/list.c b/kernel/lib/list.c
index c663230..e4d4b23 100644
--- a/kernel/lib/list.c
+++ b/kernel/lib/list.c
@@ -2,15 +2,13 @@
 #include <kmalloc.h>
 #include <lib/list.h>
 
-int list_init(struct list *list) {
-  // TODO: Make it dynamic
+void list_init(struct list *list) {
   list->capacity = 5;
+  list->tail_index = -1;
   list->entries = kmalloc(sizeof(void *) * list->capacity);
   if (!list->entries) {
-    return 0;
+    list->capacity = 0;
   }
-  list->tail_index = -1;
-  return 1;
 }
 
 void list_reset(struct list *list) {
@@ -24,15 +22,24 @@ int list_clone(struct list *in, struct list *out) {
     if (!list_get(in, i, &output)) {
       break;
     }
-    list_add(out, output, NULL);
+    if (!list_add(out, output, NULL)) {
+      list_free(out);
+      return 0;
+    }
   }
   return 1;
 }
 
 int list_add(struct list *list, void *entry, int *index) {
   if (list->tail_index + 1 >= list->capacity) {
-    list->capacity += 25;
-    list->entries = krealloc(list->entries, sizeof(void *) * list->capacity);
+    int new_capacity = list->capacity + 25;
+    void *new_allocation =
+        krealloc(list->entries, sizeof(void *) * new_capacity);
+    if (!new_allocation) {
+      return 0;
+    }
+    list->entries = new_allocation;
+    list->capacity = new_capacity;
   }
   list->tail_index++;
   list->entries[list->tail_index] = entry;
@@ -60,5 +67,8 @@ int list_get(const struct list *list, int index, void **out) {
 }
 
 void list_free(struct list *list) {
+  list->capacity = 0;
+  list->tail_index = -1;
   kfree(list->entries);
+  list->entries = NULL;
 }
diff --git a/kernel/lib/list.h b/kernel/lib/list.h
index b1f0127..2d1fa0f 100644
--- a/kernel/lib/list.h
+++ b/kernel/lib/list.h
@@ -6,7 +6,7 @@ struct list {
   int tail_index;
 };
 
-int list_init(struct list *list);
+void list_init(struct list *list);
 void list_reset(struct list *list);
 void list_free(struct list *list);
 int list_clone(struct list *in, struct list *out);
diff --git a/kernel/sched/scheduler.c b/kernel/sched/scheduler.c
index f56ff56..0eff530 100644
--- a/kernel/sched/scheduler.c
+++ b/kernel/sched/scheduler.c
@@ -100,12 +100,19 @@ void set_signal_handler(int sig, void (*handler)(int)) {
 void insert_eip_on_stack(u32 cr3, u32 address, u32 value);
 
 process_t *create_process(process_t *p, u32 esp, u32 eip) {
-  process_t *r;
-  r = kcalloc(1, sizeof(process_t));
+  process_t *r = kcalloc(1, sizeof(process_t));
+  if (!r) {
+    return NULL;
+  }
+  r->tcb = kcalloc(1, sizeof(struct TCB));
+  if (!r->tcb) {
+    kfree(r);
+    return NULL;
+  }
   r->pid = next_pid;
   next_pid++;
   if (!p) {
-    assert(1 == next_pid);
+    assert(0 == r->pid);
     strlcpy(r->program_name, "[kernel]", sizeof(current_task->program_name));
   } else {
     strlcpy(r->program_name, "[Not yet named]",
@@ -113,21 +120,11 @@ process_t *create_process(process_t *p, u32 esp, u32 eip) {
   }
 
   if (p) {
-    r->cr3 = clone_directory(p->cr3);
-    if (!r->cr3) {
+    if (!list_clone(&p->file_descriptors, &r->file_descriptors)) {
+      kfree(r->tcb);
       kfree(r);
       return NULL;
     }
-  } else {
-    r->cr3 = get_active_pagedirectory();
-  }
-  r->parent = p;
-
-  r->tcb = kcalloc(1, sizeof(struct TCB));
-  r->tcb->CR3 = r->cr3->physical_address;
-
-  if (p) {
-    list_clone(&p->file_descriptors, &r->file_descriptors);
     for (int i = 0;; i++) {
       vfs_fd_t *out;
       if (!list_get(&p->file_descriptors, i, (void **)&out)) {
@@ -140,6 +137,21 @@ process_t *create_process(process_t *p, u32 esp, u32 eip) {
   } else {
     list_init(&r->file_descriptors);
   }
+
+  if (p) {
+    r->cr3 = clone_directory(p->cr3);
+    if (!r->cr3) {
+      kfree(r->tcb);
+      kfree(r);
+      return NULL;
+    }
+  } else {
+    r->cr3 = get_active_pagedirectory();
+  }
+  r->parent = p;
+
+  r->tcb->CR3 = r->cr3->physical_address;
+
   list_init(&r->read_list);
   list_init(&r->write_list);
   list_init(&r->disconnect_list);
@@ -149,18 +161,10 @@ process_t *create_process(process_t *p, u32 esp, u32 eip) {
 
   list_init(&r->event_queue);
 
-  // Temporarily switch to the page directory to be able to place the
-  // "entry_instruction_pointer" onto the stack.
-
   if (esp) {
     esp -= 4;
     insert_eip_on_stack(r->cr3->physical_address, esp, eip);
-    esp -= 4;
-    insert_eip_on_stack(r->cr3->physical_address, esp, 0x1337);
-    esp -= 4;
-    insert_eip_on_stack(r->cr3->physical_address, esp, 0x1488);
-    esp -= 4;
-    esp -= 4;
+    esp -= 16;
     insert_eip_on_stack(r->cr3->physical_address, esp, 0xF00DBABE);
     r->tcb->ESP = esp;
   }
diff --git a/userland/libgui/libgui.c b/userland/libgui/libgui.c
index f7a8079..07edbdd 100644
--- a/userland/libgui/libgui.c
+++ b/userland/libgui/libgui.c
@@ -308,10 +308,7 @@ GUI_Window *GUI_CreateWindow(uint32_t x, uint32_t y, uint32_t sx, uint32_t sy) {
   ftruncate(w->bitmap_fd, MAX_WINDOW_SIZE * sizeof(uint32_t));
   void *rc =
       mmap(NULL, MAX_WINDOW_SIZE * sizeof(uint32_t), 0, 0, w->bitmap_fd, 0);
-  if (!((int)rc >= 0)) {
-    printf("rc: %x\n", rc);
-    assert(0);
-  }
+  assert(rc != (void *)(-1));
   w->bitmap_ptr = rc;
 
   // Send the request to the windowserver
diff --git a/userland/windowserver/ws.c b/userland/windowserver/ws.c
index 6d01048..11efa59 100644
--- a/userland/windowserver/ws.c
+++ b/userland/windowserver/ws.c
@@ -103,6 +103,7 @@ void setup_display(DISPLAY *disp, const char *path, uint64_t size) {
       ;
   }
   disp->true_buffer = mmap(NULL, size, 0, 0, disp->vga_fd, 0);
+  assert(disp->true_buffer != (void *)(-1));
   disp->back_buffer = malloc(size + 0x1000);
   disp->window = window;
 
@@ -110,7 +111,7 @@ void setup_display(DISPLAY *disp, const char *path, uint64_t size) {
   assert(disp->wallpaper_fd >= 0);
   ftruncate(disp->wallpaper_fd, size);
   void *rc = mmap(NULL, size, 0, 0, disp->wallpaper_fd, 0);
-  assert(rc);
+  assert(rc != (void *)(-1));
   disp->wallpaper_buffer = rc;
   for (int i = 0; i < disp->size / disp->bpp; i++) {
     uint32_t *p = disp->wallpaper_buffer + i * sizeof(uint32_t);