Kernel: Fix undefined behavior and cleanup functions.

The int_syscall caused crashes for higher level of optimizations. To fix
this I rewrote the function in assembly as I deemed it to be simpler.

9 files changed, 52 insertions, 21 deletions

diff --git a/kernel/Makefile b/kernel/Makefile
index 7bdb91b..8f2cd3d 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -1,7 +1,8 @@
 CC="i686-sb-gcc"
 AS="i686-sb-as"
-OBJ = arch/i386/boot.o init/kernel.o cpu/gdt.o cpu/reload_gdt.o cpu/idt.o cpu/io.o libc/stdio/print.o drivers/keyboard.o log.o drivers/pit.o libc/string/memcpy.o libc/string/strlen.o libc/string/memcmp.o drivers/ata.o libc/string/memset.o cpu/syscall.o  read_eip.o libc/exit/assert.o process.o libc/string/strcpy.o arch/i386/mmu.o kmalloc.o fs/ext2.o fs/vfs.o fs/devfs.o cpu/spinlock.o random.o libc/string/strcmp.o crypto/ChaCha20/chacha20.o crypto/SHA1/sha1.o fs/tmpfs.o libc/string/isequal.o drivers/pst.o syscalls/ppoll.o syscalls/ftruncate.o kubsan.o syscalls/mmap.o drivers/serial.o syscalls/accept.o syscalls/bind.o syscalls/socket.o socket.o poll.o fs/fifo.o hashmap/hashmap.o fs/shm.o syscalls/shm.o elf.o ksbrk.o sched/scheduler.o syscalls/stat.o libc/string/copy.o libc/string/strncpy.o drivers/mouse.o libc/string/strlcpy.o libc/string/strcat.o drivers/vbe.o syscalls/msleep.o syscalls/uptime.o syscalls/mkdir.o drivers/pci.o drivers/rtl8139.o network/ethernet.o network/arp.o network/bytes.o network/ipv4.o network/udp.o syscalls/recvfrom.o math.o syscalls/sendto.o signal.o syscalls/kill.o syscalls/sigaction.o network/tcp.o drivers/ahci.o crypto/xoshiro256plusplus/xoshiro256plusplus.o syscalls/chdir.o syscalls/getcwd.o syscalls/isatty.o syscalls/randomfill.o syscalls/open.o syscalls/write.o syscalls/pwrite.o ipc.o syscalls/ipc.o syscalls/port.o syscalls/map_frames.o syscalls/virtual_to_physical.o syscalls/install_irq.o arch/i386/interrupts.o cpu/isr.o lib/stack.o lib/buffered_write.o lib/list.o cpu/arch_inst.o
-CFLAGS = -std=c99 -Ofast -fsanitize=vla-bound,shift-exponent,pointer-overflow,shift,signed-integer-overflow,bounds -ggdb -ffreestanding -Wall -Werror -mgeneral-regs-only -Wimplicit-fallthrough -I./libc/include/ -I. -Wno-pointer-sign -DKERNEL
+OBJ = arch/i386/boot.o init/kernel.o cpu/gdt.o cpu/reload_gdt.o cpu/idt.o cpu/io.o libc/stdio/print.o drivers/keyboard.o log.o drivers/pit.o libc/string/memcpy.o libc/string/strlen.o libc/string/memcmp.o drivers/ata.o libc/string/memset.o cpu/syscall.o  read_eip.o libc/exit/assert.o process.o libc/string/strcpy.o arch/i386/mmu.o kmalloc.o fs/ext2.o fs/vfs.o fs/devfs.o cpu/spinlock.o random.o libc/string/strcmp.o crypto/ChaCha20/chacha20.o crypto/SHA1/sha1.o fs/tmpfs.o libc/string/isequal.o drivers/pst.o syscalls/ppoll.o syscalls/ftruncate.o kubsan.o syscalls/mmap.o drivers/serial.o syscalls/accept.o syscalls/bind.o syscalls/socket.o socket.o poll.o fs/fifo.o hashmap/hashmap.o fs/shm.o syscalls/shm.o elf.o ksbrk.o sched/scheduler.o syscalls/stat.o libc/string/copy.o libc/string/strncpy.o drivers/mouse.o libc/string/strlcpy.o libc/string/strcat.o drivers/vbe.o syscalls/msleep.o syscalls/uptime.o syscalls/mkdir.o drivers/pci.o drivers/rtl8139.o network/ethernet.o network/arp.o network/bytes.o network/ipv4.o network/udp.o syscalls/recvfrom.o math.o syscalls/sendto.o signal.o syscalls/kill.o syscalls/sigaction.o network/tcp.o drivers/ahci.o crypto/xoshiro256plusplus/xoshiro256plusplus.o syscalls/chdir.o syscalls/getcwd.o syscalls/isatty.o syscalls/randomfill.o syscalls/open.o syscalls/write.o syscalls/pwrite.o ipc.o syscalls/ipc.o syscalls/port.o syscalls/map_frames.o syscalls/virtual_to_physical.o syscalls/install_irq.o arch/i386/interrupts.o cpu/isr.o lib/stack.o lib/buffered_write.o lib/list.o cpu/arch_inst.o cpu/int_syscall.o
+#CFLAGS = -std=c99 -Ofast -fsanitize=vla-bound,shift-exponent,pointer-overflow,shift,signed-integer-overflow,bounds -ggdb -ffreestanding -Wall -Werror -mgeneral-regs-only -Wimplicit-fallthrough -I./libc/include/ -I. -Wno-pointer-sign -DKERNEL
+CFLAGS = -std=c99 -Ofast -ggdb -ffreestanding -Wall -Werror -mgeneral-regs-only -Wimplicit-fallthrough -I./libc/include/ -I. -Wno-pointer-sign -DKERNEL
 #CFLAGS = -Ofast -fsanitize=vla-bound,shift-exponent,pointer-overflow,shift,signed-integer-overflow,bounds -ggdb -ffreestanding -Wall -Werror -mgeneral-regs-only -Wimplicit-fallthrough -I./libc/include/ -I. -Wno-pointer-sign -DKERNEL
 #CFLAGS = -O0 -ggdb -ffreestanding -Wall -Werror -mgeneral-regs-only -Wimplicit-fallthrough -I./libc/include/ -I. -Wno-pointer-sign -DKERNEL
 LDFLAGS=-flto -Ofast
diff --git a/kernel/cpu/idt.c b/kernel/cpu/idt.c
index ec7ca63..805eb20 100644
--- a/kernel/cpu/idt.c
+++ b/kernel/cpu/idt.c
@@ -249,7 +249,6 @@ void install_handler(void (*handler_function)(), u16 type_attribute, u8 entry) {
 }
 
 void idt_init(void) {
-  //  list_of_handlers = kcalloc(sizeof(void *), 128);
   memset(list_of_handlers, 0, sizeof(void *) * 256);
 
   install_handler(page_fault, INT_32_INTERRUPT_GATE(0x0), 0xE);
diff --git a/kernel/cpu/int_syscall.s b/kernel/cpu/int_syscall.s
new file mode 100644
index 0000000..b2c0de9
--- /dev/null
+++ b/kernel/cpu/int_syscall.s
@@ -0,0 +1,35 @@
+.intel_syntax noprefix
+.global int_syscall
+.extern syscall_functions
+int_syscall:
+push   ebp
+mov    ebp,esp
+push   edi
+push   esi
+push   ebx
+
+mov    edx,DWORD PTR [ebp+0x8] # reg_t*
+mov    eax,DWORD PTR [edx+0x20] # syscall number
+mov    eax,DWORD PTR [eax*4+syscall_functions] # function pointer
+
+mov    edi,DWORD PTR [edx+0x4]
+push edi
+mov    edi,DWORD PTR [edx+0x8]
+push edi
+mov    edi,DWORD PTR [edx+0x18]
+push edi
+mov    edi,DWORD PTR [edx+0x1c]
+push edi
+mov    edi,DWORD PTR [edx+0x14]
+push edi
+call   eax
+add    esp,0x20
+mov    edx,DWORD PTR [ebp+0x8] # reg_t*
+mov    DWORD PTR [edx+0x20],eax
+lea    esp,[ebp-0xc]
+
+pop    ebx
+pop    esi
+pop    edi
+pop    ebp
+ret
diff --git a/kernel/cpu/isr.s b/kernel/cpu/isr.s
index 1bb2fff..8032485 100644
--- a/kernel/cpu/isr.s
+++ b/kernel/cpu/isr.s
@@ -270,12 +270,12 @@ ISR_NOERRCODE 254
 ISR_NOERRCODE 255
 
 isr_common_stub:
-    pusha                    # Pushes edi,esi,ebp,esp,ebx,edx,ecx,eax
+    pusha            # Pushes edi,esi,ebp,esp,ebx,edx,ecx,eax
 
-    mov ax, ds               # Lower 16-bits of eax = ds.
-    push eax                 # save the data segment descriptor
+    mov ax, ds       # Lower 16-bits of eax = ds.
+    push eax         # save the data segment descriptor
 
-    mov ax, 0x10  # load the kernel data segment descriptor
+    mov ax, 0x10     # load the kernel data segment descriptor
     mov ds, ax
     mov es, ax
     mov fs, ax
@@ -285,13 +285,13 @@ isr_common_stub:
     call int_handler
     add esp, 4
 
-    pop ebx        # reload the original data segment descriptor
+    pop ebx          # reload the original data segment descriptor
     mov ds, bx
     mov es, bx
     mov fs, bx
     mov gs, bx
 
-    popa                     # Pops edi,esi,ebp...
-    add esp, 8     # Cleans up the pushed error code and pushed ISR number
+    popa             # Pops edi,esi,ebp...
+    add esp, 8       # Cleans up the pushed error code and pushed ISR number
     sti
-    iret           # pops 5 things at once: CS, EIP, EFLAGS, SS, and ESP
+    iret             # pops 5 things at once: CS, EIP, EFLAGS, SS, and ESP
diff --git a/kernel/cpu/syscall.c b/kernel/cpu/syscall.c
index e2ea7e3..193367e 100644
--- a/kernel/cpu/syscall.c
+++ b/kernel/cpu/syscall.c
@@ -182,18 +182,13 @@ int (*syscall_functions[])() = {
     (void(*))syscall_tmp_handle_packet,
 };
 
+void int_syscall(reg_t *r);
 void syscall_function_handler(u32 eax, u32 arg1, u32 arg2, u32 arg3, u32 arg4,
                               u32 arg5, u32 ebp, u32 esp) {
   assert(eax < sizeof(syscall_functions) / sizeof(syscall_functions[0]));
   syscall_functions[eax](arg1, arg2, arg3, arg4, arg5);
 }
 
-void int_syscall(reg_t *r) {
-  u32 syscall = r->eax;
-  assert(syscall < sizeof(syscall_functions) / sizeof(syscall_functions[0]));
-  r->eax = syscall_functions[syscall](r->ebx, r->ecx, r->edx, r->esi, r->edi);
-}
-
 void syscalls_init(void) {
   install_handler(int_syscall, INT_32_INTERRUPT_GATE(0x3), 0x80);
 }
diff --git a/kernel/drivers/mouse.c b/kernel/drivers/mouse.c
index ec7ce8c..af4f22e 100644
--- a/kernel/drivers/mouse.c
+++ b/kernel/drivers/mouse.c
@@ -142,4 +142,5 @@ void install_mouse(void) {
 
   install_handler(int_mouse, INT_32_INTERRUPT_GATE(0x3), 12 + 0x20);
   install_handler(what, INT_32_INTERRUPT_GATE(0x3), 0xe + 0x20);
+  install_handler(what, INT_32_INTERRUPT_GATE(0x3), 0xf + 0x20);
 }
diff --git a/kernel/drivers/pit.c b/kernel/drivers/pit.c
index 060fc54..30fd3ed 100644
--- a/kernel/drivers/pit.c
+++ b/kernel/drivers/pit.c
@@ -43,8 +43,8 @@ void set_pit_count(u16 _hertz) {
   outb(PIT_IO_CHANNEL_0, (divisor & 0xFF00) >> 8);
 }
 
-void int_clock(reg_t regs) {
-  outb(0x20, 0x20);
+void int_clock(reg_t *regs) {
+  EOI(0x20);
   pit_counter++;
   if (pit_counter * 1000 >= hertz) {
     pit_counter = 0;
diff --git a/kernel/ipc.c b/kernel/ipc.c
index 56ca79a..bce346a 100644
--- a/kernel/ipc.c
+++ b/kernel/ipc.c
@@ -58,7 +58,7 @@ int ipc_read(u8 *buffer, u32 length, u32 *sender_pid) {
         return 0;
       }
       get_current_task()->is_halted = 1;
-      enable_interrupts();
+      switch_task();
       continue;
     }
     break;
diff --git a/kernel/sched/scheduler.c b/kernel/sched/scheduler.c
index 614f621..e34e7a7 100644
--- a/kernel/sched/scheduler.c
+++ b/kernel/sched/scheduler.c
@@ -325,7 +325,7 @@ int fork(void) {
 }
 
 int isset_fdhalt(process_t *p) {
-  if(NULL == p) {
+  if (NULL == p) {
     p = get_current_task();
   }
   int blocked = 0;