Add extern funcs, parameter registers, fix asm_rm RSP bug

4 files changed, 119 insertions, 40 deletions

diff --git a/executor/executor.h b/executor/executor.h
index 5be9abc..979784b 100644
--- a/executor/executor.h
+++ b/executor/executor.h
@@ -25,25 +25,26 @@ CHECK_RESULT int amal_executor_instructions_start(amal_executor *self, u16 num_f
 CHECK_RESULT int amal_executor_instructions_end(amal_executor *self);
 
 CHECK_RESULT int amal_exec_nop(amal_executor *self);
-CHECK_RESULT int amal_exec_setz(amal_executor *self, u8 dst_reg);
-CHECK_RESULT int amal_exec_mov(amal_executor *self, u8 dst_reg, u8 src_reg);
-CHECK_RESULT int amal_exec_movi(amal_executor *self, u8 dst_reg, i64 imm);
-CHECK_RESULT int amal_exec_movd(amal_executor *self, u8 dst_reg, BufferView data);
-CHECK_RESULT int amal_exec_add(amal_executor *self, u8 dst_reg, u8 src_reg1, u8 src_reg2);
-CHECK_RESULT int amal_exec_sub(amal_executor *self, u8 dst_reg, u8 src_reg1, u8 src_reg2);
-CHECK_RESULT int amal_exec_imul(amal_executor *self, u8 dst_reg, u8 src_reg1, u8 src_reg2);
-CHECK_RESULT int amal_exec_mul(amal_executor *self, u8 dst_reg, u8 src_reg1, u8 src_reg2);
-CHECK_RESULT int amal_exec_idiv(amal_executor *self, u8 dst_reg, u8 src_reg1, u8 src_reg2);
-CHECK_RESULT int amal_exec_div(amal_executor *self, u8 dst_reg, u8 src_reg1, u8 src_reg2);
-CHECK_RESULT int amal_exec_push(amal_executor *self, u8 reg);
+CHECK_RESULT int amal_exec_setz(amal_executor *self, i8 dst_reg);
+CHECK_RESULT int amal_exec_mov(amal_executor *self, i8 dst_reg, i8 src_reg);
+CHECK_RESULT int amal_exec_movi(amal_executor *self, i8 dst_reg, i64 imm);
+CHECK_RESULT int amal_exec_movd(amal_executor *self, i8 dst_reg, BufferView data);
+CHECK_RESULT int amal_exec_add(amal_executor *self, i8 dst_reg, i8 src_reg1, i8 src_reg2);
+CHECK_RESULT int amal_exec_sub(amal_executor *self, i8 dst_reg, i8 src_reg1, i8 src_reg2);
+CHECK_RESULT int amal_exec_imul(amal_executor *self, i8 dst_reg, i8 src_reg1, i8 src_reg2);
+CHECK_RESULT int amal_exec_mul(amal_executor *self, i8 dst_reg, i8 src_reg1, i8 src_reg2);
+CHECK_RESULT int amal_exec_idiv(amal_executor *self, i8 dst_reg, i8 src_reg1, i8 src_reg2);
+CHECK_RESULT int amal_exec_div(amal_executor *self, i8 dst_reg, i8 src_reg1, i8 src_reg2);
+CHECK_RESULT int amal_exec_push(amal_executor *self, i8 reg);
 CHECK_RESULT int amal_exec_pushi(amal_executor *self, i64 imm);
 CHECK_RESULT int amal_exec_pushd(amal_executor *self, BufferView data);
-CHECK_RESULT int amal_exec_call(amal_executor *self, u16 func_index, u8 num_args);
-/*CHECK_RESULT int amal_exec_callr(u8 dst_reg, BufferView data);*/
-CHECK_RESULT int amal_exec_cmp(amal_executor *self, u8 dst_reg, u8 src_reg1, u8 src_reg2);
-CHECK_RESULT int amal_exec_jz(amal_executor *self, u8 dst_reg, i16 offset);
+CHECK_RESULT int amal_exec_call(amal_executor *self, u16 func_index, u8 num_args, i8 dst_reg);
+/*CHECK_RESULT int amal_exec_callr(i8 dst_reg, BufferView data);*/
+CHECK_RESULT int amal_exec_calle(amal_executor *self, void *func, u8 num_args, i8 dst_reg);
+CHECK_RESULT int amal_exec_cmp(amal_executor *self, i8 dst_reg, i8 src_reg1, i8 src_reg2);
+CHECK_RESULT int amal_exec_jz(amal_executor *self, i8 dst_reg, i16 offset);
 CHECK_RESULT int amal_exec_jmp(amal_executor *self, i16 offset);
-CHECK_RESULT int amal_exec_ret(amal_executor *self, u8 reg);
+CHECK_RESULT int amal_exec_ret(amal_executor *self, i8 reg);
 CHECK_RESULT int amal_exec_func_start(amal_executor *self, u16 num_regs);
 CHECK_RESULT int amal_exec_func_end(amal_executor *self);
 
diff --git a/executor/x86_64/asm.c b/executor/x86_64/asm.c
index f032538..606e539 100644
--- a/executor/x86_64/asm.c
+++ b/executor/x86_64/asm.c
@@ -21,6 +21,7 @@ static usize ins_start_offset = 0;
 static void asm_debug_str_append(const char *str) {
     const usize len = strlen(str);
     am_memcpy(asm_debug_str_buffer + asm_debug_str_buffer_index, str, len);
+    asm_debug_str_buffer[asm_debug_str_buffer_index + len + 1] = '\0';
     asm_debug_str_buffer_index += len;
 }
 
@@ -101,12 +102,14 @@ static const char* asm_ptr_to_string(AsmPtr *self) {
         asm_debug_str_append(" * ");
         asm_debug_str_append_num(1 << self->scale);
     }
-    if(self->disp < 0)
-        asm_debug_str_append(" - ");
-    else
-        asm_debug_str_append(" + ");
-    asm_debug_str_append_num(abs(self->disp));
-    asm_debug_str_append("]\0");
+    if(self->disp != 0) {
+        if(self->disp < 0)
+            asm_debug_str_append(" - ");
+        else
+            asm_debug_str_append(" + ");
+        asm_debug_str_append_num(abs(self->disp));
+    }
+    asm_debug_str_append("]");
     return buf;
 }
 #else
@@ -279,6 +282,9 @@ static void asm_rm(Asm *self, AsmPtr *mem, Reg64 reg) {
             disp_bytes = 4;
         }
 
+        /* Scale is not valid when index is RSP */
+        assert(mem->index != RSP || mem->scale == 0);
+
         #ifdef DEBUG
         if(mem->scale != 0 && mem->scale != 2 && mem->scale != 4 && mem->scale != 8) {
             amal_log_error("Invalid scale %d, expected 0, 2, 4, or 8", mem->scale);
@@ -307,7 +313,11 @@ static void asm_rm(Asm *self, AsmPtr *mem, Reg64 reg) {
             rm_byte = 0x80 + mem->base;
             disp_bytes = 4;
         }
+
         *self->code_it++ = (reg << 3) | rm_byte;
+        /* RSP requires SIB byte when displacement is not 0 */
+        if(mem->base == RSP)
+            *self->code_it++ = 0x24;
     }
 
     am_memcpy(self->code_it, &mem->disp, disp_bytes);
@@ -444,6 +454,16 @@ int asm_popr(Asm *self, Reg64 reg) {
     return 0;
 }
 
+int asm_callr(Asm *self, Reg64 reg) {
+    ins_start(self);
+    return_if_error(asm_ensure_capacity(self, 3));
+    *self->code_it++ = REX_W;
+    *self->code_it++ = 0xFF;
+    asm_rr(self, reg, 0x2);
+    ins_end(self, "call %s", reg64_to_str(reg));
+    return 0;
+}
+
 /*
     Note: This is sometimes called with @relative 0 (will print call -5), in which case it's most likely a dummy call until the relative position
     is later changed with @asm_override_call_rel32. TODO: Update the ins_end debug print to take that into account somehow
diff --git a/executor/x86_64/asm.h b/executor/x86_64/asm.h
index 7e5ac67..51f2d84 100644
--- a/executor/x86_64/asm.h
+++ b/executor/x86_64/asm.h
@@ -80,6 +80,7 @@ CHECK_RESULT int asm_idiv_rr(Asm *self, Reg64 src);
 
 CHECK_RESULT int asm_pushr(Asm *self, Reg64 reg);
 CHECK_RESULT int asm_popr(Asm *self, Reg64 reg);
+CHECK_RESULT int asm_callr(Asm *self, Reg64 reg);
 /*
     In x86 assembly, the @relative position starts from the next instruction.
     This offset shouldn't be calculated by the caller and is instead managed
diff --git a/executor/x86_64/executor.c b/executor/x86_64/executor.c
index b7aa91f..ebe848d 100644
--- a/executor/x86_64/executor.c
+++ b/executor/x86_64/executor.c
@@ -28,7 +28,16 @@ typedef struct {
     amal_executor_impl *impl; \
     impl = (amal_executor_impl*)self;
 
-#define get_register_stack_offset(reg) -(i32)(reg * (int)sizeof(usize) + (int)sizeof(usize))
+/*
+    @reg will be a positive value when accessing local variables, in which case the first
+    local variable is located at -sizeof(usize) and the next one is at -(2 * sizeof(usize)).
+    @reg will be a negative value starting at -1 when accessing parameters.
+    The first parameter is located at 3*sizeof(usize) and the next one is at 4*sizeof(usize).
+    Parameter starts at 3*sizeof(usize) because offset 0 is the return address, offset 1*sizeof(usize) is the
+    saved RBP and 2*sizeof(usize) is saved RBX.
+*/
+#define get_register_stack_offset(reg) \
+    (reg >= 0 ? (i32)(-reg * (int)sizeof(usize) - sizeof(usize)) : (i32)(-reg * (int)sizeof(usize) + 2*sizeof(usize)))
 
 static i64 abs_i64(i64 value) {
     return value >= 0 ? value : -value;
@@ -88,14 +97,14 @@ int amal_exec_nop(amal_executor *self) {
     return asm_nop(&impl->asm);
 }
 
-int amal_exec_setz(amal_executor *self, u8 dst_reg) {
+int amal_exec_setz(amal_executor *self, i8 dst_reg) {
     AsmPtr dst;
     IMPL
     asm_ptr_init_disp(&dst, RBP, get_register_stack_offset(dst_reg));
     return asm_mov_mi(&impl->asm, &dst, 0);
 }
 
-int amal_exec_mov(amal_executor *self, u8 dst_reg, u8 src_reg) {
+int amal_exec_mov(amal_executor *self, i8 dst_reg, i8 src_reg) {
     AsmPtr ptr;
     IMPL
 
@@ -106,7 +115,7 @@ int amal_exec_mov(amal_executor *self, u8 dst_reg, u8 src_reg) {
     return asm_mov_mr(&impl->asm, &ptr, RAX);
 }
 
-int amal_exec_movi(amal_executor *self, u8 dst_reg, i64 imm) {
+int amal_exec_movi(amal_executor *self, i8 dst_reg, i64 imm) {
     IMPL
     /* TODO: if @number is a float then use float instructions */
     if(abs_i64(imm) <= INT32_MAX) {
@@ -122,7 +131,7 @@ int amal_exec_movi(amal_executor *self, u8 dst_reg, i64 imm) {
     return 0;
 }
 
-int amal_exec_movd(amal_executor *self, u8 dst_reg, BufferView data) {
+int amal_exec_movd(amal_executor *self, i8 dst_reg, BufferView data) {
     AsmPtr dst;
     IMPL
 
@@ -131,7 +140,7 @@ int amal_exec_movd(amal_executor *self, u8 dst_reg, BufferView data) {
     return asm_mov_mr(&impl->asm, &dst, RAX);
 }
 
-int amal_exec_add(amal_executor *self, u8 dst_reg, u8 src_reg1, u8 src_reg2) {
+int amal_exec_add(amal_executor *self, i8 dst_reg, i8 src_reg1, i8 src_reg2) {
     AsmPtr dst;
     AsmPtr reg1;
     AsmPtr reg2;
@@ -147,7 +156,7 @@ int amal_exec_add(amal_executor *self, u8 dst_reg, u8 src_reg1, u8 src_reg2) {
     return asm_mov_mr(&impl->asm, &dst, RAX);
 }
 
-int amal_exec_sub(amal_executor *self, u8 dst_reg, u8 src_reg1, u8 src_reg2) {
+int amal_exec_sub(amal_executor *self, i8 dst_reg, i8 src_reg1, i8 src_reg2) {
     AsmPtr dst;
     AsmPtr reg1;
     AsmPtr reg2;
@@ -163,7 +172,7 @@ int amal_exec_sub(amal_executor *self, u8 dst_reg, u8 src_reg1, u8 src_reg2) {
     return asm_mov_mr(&impl->asm, &dst, RAX);
 }
 
-int amal_exec_imul(amal_executor *self, u8 dst_reg, u8 src_reg1, u8 src_reg2) {
+int amal_exec_imul(amal_executor *self, i8 dst_reg, i8 src_reg1, i8 src_reg2) {
     AsmPtr dst;
     AsmPtr reg1;
     AsmPtr reg2;
@@ -179,7 +188,7 @@ int amal_exec_imul(amal_executor *self, u8 dst_reg, u8 src_reg1, u8 src_reg2) {
     return asm_mov_mr(&impl->asm, &dst, RAX);
 }
 
-int amal_exec_mul(amal_executor *self, u8 dst_reg, u8 src_reg1, u8 src_reg2) {
+int amal_exec_mul(amal_executor *self, i8 dst_reg, i8 src_reg1, i8 src_reg2) {
     (void)self;
     (void)dst_reg;
     (void)src_reg1;
@@ -203,7 +212,7 @@ int amal_exec_mul(amal_executor *self, u8 dst_reg, u8 src_reg1, u8 src_reg2) {
     return 0;
 }
 
-int amal_exec_idiv(amal_executor *self, u8 dst_reg, u8 src_reg1, u8 src_reg2) {
+int amal_exec_idiv(amal_executor *self, i8 dst_reg, i8 src_reg1, i8 src_reg2) {
     AsmPtr dst;
     AsmPtr reg1;
     AsmPtr reg2;
@@ -216,11 +225,12 @@ int amal_exec_idiv(amal_executor *self, u8 dst_reg, u8 src_reg1, u8 src_reg2) {
     return_if_error(asm_mov_rm(&impl->asm, RAX, &reg1));
     return_if_error(asm_mov_rm(&impl->asm, RCX, &reg2));
     return_if_error(asm_cqo(&impl->asm));
+    /* TODO: Preserve RDX if needed, since it's also used as a parameter in system-v x86_64 abi */
     return_if_error(asm_idiv_rr(&impl->asm, RCX));
     return asm_mov_mr(&impl->asm, &dst, RAX);
 }
 
-int amal_exec_div(amal_executor *self, u8 dst_reg, u8 src_reg1, u8 src_reg2) {
+int amal_exec_div(amal_executor *self, i8 dst_reg, i8 src_reg1, i8 src_reg2) {
     (void)self;
     (void)dst_reg;
     (void)src_reg1;
@@ -230,7 +240,7 @@ int amal_exec_div(amal_executor *self, u8 dst_reg, u8 src_reg1, u8 src_reg2) {
     return 0;
 }
 
-int amal_exec_push(amal_executor *self, u8 reg) {
+int amal_exec_push(amal_executor *self, i8 reg) {
     AsmPtr reg_ptr;
     IMPL
 
@@ -255,7 +265,7 @@ int amal_exec_pushd(amal_executor *self, BufferView data) {
     return 0;
 }
 
-int amal_exec_call(amal_executor *self, u16 func_index, u8 num_args) {
+int amal_exec_call(amal_executor *self, u16 func_index, u8 num_args, i8 dst_reg) {
     isize asm_size;
     IMPL
     /* TODO: Preserve necessary registers before call? */
@@ -275,18 +285,65 @@ int amal_exec_call(amal_executor *self, u16 func_index, u8 num_args) {
         return_if_error(asm_call_rel32(&impl->asm, 0));
     }
 
+    {
+        AsmPtr dst;
+        asm_ptr_init_disp(&dst, RBP, get_register_stack_offset(dst_reg));
+        /* TODO: Make this work when result is not stored in RAX (multiple return results) */
+        return_if_error(asm_mov_mr(&impl->asm, &dst, RAX));
+    }
+    if(num_args > 0)
+        return asm_add_rm64_imm(&impl->asm, RSP, num_args * sizeof(isize));
+    return 0;
+}
+
+const Reg64 SYS_V_PARAM_REGS[] = { RDI, RSI, RDX, RCX };
+
+/*
+    TODO: Make argument passing work for different calling conventions and different ABI.
+    This currently assumes x86_64 system v abi.
+    System-V ABI parameters:
+     	RDI, RSI, RDX, RCX, R8, R9, XMM0–7.
+        The rest are passed in the stack.
+*/
+/* TODO: Make this work when function returns something else than a POD */
+int amal_exec_calle(amal_executor *self, void *func, u8 num_args, i8 dst_reg) {
+    AsmPtr dst;
+    IMPL
+
+    /* TODO: Support R and XMM registers so more than 5 arguments can be used for functions */
+    assert(num_args < 5);
+    {
+        /*
+            TODO: Do this directly in @PUSH instruction instead. For now we copy
+            the pushed data to the registers that need to be set for the specific abi for parameters
+        */
+        int i;
+        AsmPtr src;
+        asm_ptr_init_disp(&src, RSP, 0);
+        for(i = num_args - 1; i >= 0; --i) {
+            return_if_error(asm_mov_rm(&impl->asm, SYS_V_PARAM_REGS[i], &src));
+            src.disp += 0x8;
+        }
+    }
+
+    /* TODO: Preserve necessary registers before call? */
+    /* TODO: This assumes all arguments are isize */
+    return_if_error(asm_mov_ri(&impl->asm, RAX, (intptr_t)func));
+    return_if_error(asm_callr(&impl->asm, RAX));
+    asm_ptr_init_disp(&dst, RBP, get_register_stack_offset(dst_reg));
+    return_if_error(asm_mov_mr(&impl->asm, &dst, RAX));
     if(num_args > 0)
         return asm_add_rm64_imm(&impl->asm, RSP, num_args * sizeof(isize));
     return 0;
 }
 
 /*
-int amal_exec_callr(u8 dst_reg, BufferView data) {
+int amal_exec_callr(i8 dst_reg, BufferView data) {
 
 }
 */
 
-int amal_exec_cmp(amal_executor *self, u8 dst_reg, u8 src_reg1, u8 src_reg2) {
+int amal_exec_cmp(amal_executor *self, i8 dst_reg, i8 src_reg1, i8 src_reg2) {
     (void)self;
     (void)dst_reg;
     (void)src_reg1;
@@ -296,7 +353,7 @@ int amal_exec_cmp(amal_executor *self, u8 dst_reg, u8 src_reg1, u8 src_reg2) {
     return 0;
 }
 
-int amal_exec_jz(amal_executor *self, u8 dst_reg, i16 offset) {
+int amal_exec_jz(amal_executor *self, i8 dst_reg, i16 offset) {
     (void)self;
     (void)dst_reg;
     (void)offset;
@@ -313,12 +370,12 @@ int amal_exec_jmp(amal_executor *self, i16 offset) {
     return 0;
 }
 
-int amal_exec_ret(amal_executor *self, u8 reg) {
+int amal_exec_ret(amal_executor *self, i8 reg) {
     AsmPtr ret_reg;
     IMPL
 
     asm_ptr_init_disp(&ret_reg, RBP, get_register_stack_offset(reg));
-    /* Result is returned in RAX register. TODO: Make this work for larger data */
+    /* Result is returned in RAX register. TODO: Make this work when returning more than one result */
     return_if_error(asm_mov_rm(&impl->asm, RAX, &ret_reg));
     return amal_exec_func_end(self);
 }