diff --git a/src/lj_ccall.c b/src/lj_ccall.c index 1aae0d40..f6f2f369 100644 --- a/src/lj_ccall.c +++ b/src/lj_ccall.c @@ -577,6 +577,197 @@ goto done; \ } +#elif LJ_TARGET_RISCV64 +/* -- RISC-V lp64d calling conventions ------------------------------------ */ + +/* Forward declarations for functions used by macros below. */ +#define CCALL_MIX_MAXFIELDS 2 /* MIX state machine has exactly 2 slots. */ +typedef struct { + CTSize offset; + uint8_t esz; + uint8_t isfp; +} CCallMixField; + +typedef struct { + uint8_t nfields; + CCallMixField fields[CCALL_MIX_MAXFIELDS]; +} CCallMixFieldList; + +static CTSize ccall_place_mix_fields(CCallState *cc, CCallMixFieldList *list, + CTSize sz, uint8_t *src, + MSize *pnfpr, MSize *pngpr, int isva); +static void ccall_unplace_mix_fields(CCallState *cc, CCallMixFieldList *list, + uint8_t *dp); + +#define CCALL_HANDLE_STRUCTRET \ + /* Return by reference only if >16 bytes and not classifiable as MIX. */ \ + cc->retref = !ccall_classify_struct(cts, ctr, NULL, 0).ispod && sz > 16; \ + if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp; + +#define CCALL_HANDLE_STRUCTRET2 \ +{ \ + CCallMixFieldList mixlist = {0}; \ + CCallStructClass cl = ccall_classify_struct(cts, ctr, &mixlist, 0); \ + CCallStructMix mix = cl.mix; \ + switch (mix.val) { \ + case MIX_IX: case MIX_FX: \ + case MIX_FF: case MIX_IF: case MIX_FI: { \ + ccall_unplace_mix_fields(cc, &mixlist, (uint8_t *)dp); \ + break; \ + } \ + case MIX_UNINIT: \ + case MIX_FAILED: { \ + memcpy(dp, sp, ctr->size); \ + break; \ + } \ + default: lj_assertL(0, "Invalid ret mix %d", mix.val); \ + } \ +} + +#define CCALL_HANDLE_COMPLEXRET \ + /* Complex values are returned in 1 or 2 FPRs. */ \ + cc->retref = 0; + +#define CCALL_HANDLE_COMPLEXRET2 \ + if (ctr->size == 2*sizeof(float)) { /* Copy complex float from FPRs. */ \ + ((float *)dp)[0] = cc->fpr[0].f; \ + ((float *)dp)[1] = cc->fpr[1].f; \ + } else { /* Copy complex double from FPRs. */ \ + ((double *)dp)[0] = cc->fpr[0].d; \ + ((double *)dp)[1] = cc->fpr[1].d; \ + } + +#define CCALL_HANDLE_COMPLEXARG \ + /* Pass long double complex by reference. */ \ + if (sz == 2*sizeof(long double)) { \ + rp = cdataptr(lj_cdata_new(cts, did, sz)); \ + sz = CTSIZE_PTR; \ + } \ + /* Pass complex in two FPRs or two GPRs or on stack. */ \ + else if (sz == 2*sizeof(float)) { \ + mix = (CCallStructMix){ .val = MIX_FF }; \ + sz = CTSIZE_PTR; \ + } else /*if (sz == 2*sizeof(double))*/ { \ + mix = (CCallStructMix){ .val = MIX_FF }; \ + sz = 2*CTSIZE_PTR; \ + } + +#define CCALL_HANDLE_RET \ + if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \ + sp = (uint8_t *)&cc->fpr[0].f; + +#define CCALL_HANDLE_STRUCTARG \ + /* Pass structs of size >16 by reference if not classifiable as MIX. */ \ + CCallMixFieldList mixlist = {0}; \ + CCallStructClass cl = ccall_classify_struct(cts, d, &mixlist, 0); \ + uint8_t ispod = cl.ispod; \ + mix = cl.mix; \ + /* Check if MIX registers available; fall back to integer conv if not */ \ + if (ispod && mix.val != MIX_FAILED && mix.val != MIX_UNINIT) { \ + int need_fpr = 0, need_gpr = 0; \ + switch (mix.val) { \ + case MIX_FX: need_fpr = 1; break; \ + case MIX_FF: need_fpr = 2; break; \ + case MIX_FI: case MIX_IF: need_fpr = 1; need_gpr = 1; break; \ + default: lj_assertCTS(0, "Invalid MIX %d", mix.val); \ + } \ + if ((need_fpr && nfpr + need_fpr > CCALL_NARG_FPR) || \ + (need_gpr && ngpr + need_gpr > CCALL_NARG_GPR)) \ + mix.val = MIX_FAILED; /* Fall back to integer convention */ \ + } \ + /* Integer convention >2xXLEN: pass by reference */ \ + if (ispod && (mix.val == MIX_FAILED || mix.val == MIX_UNINIT) && sz > 16) \ + ispod = 0; /* Route to by-reference path below. */ \ + if (!ispod && sz > 16) { \ + rp = cdataptr(lj_cdata_new(cts, did, sz)); \ + sz = CTSIZE_PTR; \ + } else if (ispod) { \ + /* POD struct: convert to stack, then place fields in registers. */ \ + MSize al = (1u << ctype_align(ccall_struct_align(cts, d))) - 1; \ + MSize ns = (nsp + al) & ~al; \ + if (ns + sz > CCALL_SIZE_STACK) goto err_nyi; \ + uint8_t *src = (uint8_t *)cc->stack + ns; \ + lj_cconv_ct_tv(cts, d, src, o, CCF_ARG(narg)); \ + if (mix.val == MIX_FAILED || mix.val == MIX_UNINIT) { \ + /* Integer convention: place 8-byte chunks in GPRs, rest on stack. */ \ + int nchunks = (sz + 7) >> 3; \ + int chunks_avail = CCALL_NARG_GPR - (int)ngpr; \ + int i; \ + for (i = 0; i < chunks_avail && i < nchunks; i++) \ + cc->gpr[ngpr++] = ((GPRArg *)src)[i]; \ + if (i < nchunks) { \ + memmove(src, src + i * 8, sz - i * 8); \ + nsp = ns + (nchunks - i) * 8; \ + } \ + } else { \ + /* MIX convention: place fields from flat descriptor list. */ \ + CTSize tail = ccall_place_mix_fields(cc, &mixlist, sz, \ + src, &nfpr, &ngpr, isva); \ + if (tail < sz) { \ + memmove(src, src + tail, sz - tail); \ + nsp = ns + (sz - tail); \ + } \ + } \ + continue; \ + } + +#define CCALL_HANDLE_REGARG \ + if (!isva) { \ + if (isfp && nfpr + 1 <= CCALL_NARG_FPR) { \ + dp = &cc->fpr[nfpr]; \ + nfpr += 1; \ + goto done; \ + } \ + } \ + if (ngpr + n <= maxgpr) { \ + dp = &cc->gpr[ngpr]; \ + ngpr += n; \ + goto done; \ + } + +/* Place MIX struct fields from a flat descriptor list into registers. +** Returns byte offset of first unplaced field (sz if all placed). */ +static CTSize ccall_place_mix_fields(CCallState *cc, CCallMixFieldList *list, + CTSize sz, uint8_t *src, + MSize *pnfpr, MSize *pngpr, + int isva) +{ + for (int i = 0; i < list->nfields; i++) { + CCallMixField *f = &list->fields[i]; + uint64_t val; + if (f->isfp) { + val = (f->esz == 4) ? (*(uint32_t *)(src + f->offset) | U64x(ffffffff,00000000)) + : *(uint64_t *)(src + f->offset); + if (!isva && *pnfpr < CCALL_NARG_FPR) cc->fpr[(*pnfpr)++].u = val; + else if (*pngpr < CCALL_NARG_GPR) cc->gpr[(*pngpr)++] = val; + else return f->offset; + } else { + val = (f->esz < 8) ? (uint64_t)(int64_t)*(int32_t *)(src + f->offset) + : *(uint64_t *)(src + f->offset); + if (*pngpr < CCALL_NARG_GPR) cc->gpr[(*pngpr)++] = val; + else return f->offset; + } + } + return sz; /* all placed */ +} + +/* Reconstruct MIX struct fields from registers using flat descriptor list. */ +static void ccall_unplace_mix_fields(CCallState *cc, CCallMixFieldList *list, + uint8_t *dp) +{ + int gpi = 0, fpi = 0; + for (int i = 0; i < list->nfields; i++) { + CCallMixField *f = &list->fields[i]; + if (f->isfp) { + if (f->esz == 4) *(float *)(dp + f->offset) = cc->fpr[fpi++].f; + else *(double *)(dp + f->offset) = cc->fpr[fpi++].d; + } else { + if (f->esz < 8) *(int32_t *)(dp + f->offset) = (int32_t)cc->gpr[gpi++]; + else *(int64_t *)(dp + f->offset) = cc->gpr[gpi++]; + } + } +} + #else #error "Missing calling convention definitions for this architecture" #endif @@ -929,6 +1120,206 @@ static void ccall_copy_struct(CCallState *cc, CType *ctr, void *dp, void *sp, #endif +/* -- RISC-V ABI struct classification ---------------------------- */ + +#if LJ_TARGET_RISCV64 + +/* RISC-V 64 LP64D fp reg struct classification. */ +/* X: unknown/uninit, F: float, D: double, I: integer */ + +/* Flat descriptor for a single field element produced by classification. +** Place/unplace iterate this list instead of re-walking the ctype tree. */ +#define CCALL_MIX_MAXFIELDS 2 /* MIX state machine has exactly 2 slots. */ + +/* Add a field descriptor to the flat list (if space remains). */ +static LJ_AINLINE void ccall_add_mix_field(CCallMixFieldList *list, + CTSize offset, CTSize esz, int isfp) +{ + if (list && list->nfields < CCALL_MIX_MAXFIELDS) { + CCallMixField *f = &list->fields[list->nfields++]; + f->offset = offset; + f->esz = (uint8_t)esz; + f->isfp = (uint8_t)isfp; + } +} + +typedef enum eCCallStructMixElem { + MIX_ELEM_UNINIT = 0, + MIX_ELEM_FLOAT = 1, + MIX_ELEM_INT = 2, +} eCCallStructMixElem; +typedef enum eCCallStructMix { + MIX_UNINIT = 0, /* XX */ + MIX_FX = 1, /* F X — single float/double */ + MIX_IX = 2, /* I X — single int */ + MIX_FF = 5, /* F F — two floats/doubles */ + MIX_IF = 6, /* I F — int then float/double */ + MIX_FI = 9, /* F I — float/double then int */ + MIX_FAILED = 10 /* anything else */ +} eCCallStructMix; + +typedef union CCallStructMix { + eCCallStructMix val : 4; + struct { + eCCallStructMixElem e1 : 2; + eCCallStructMixElem e2 : 2; + }; +} CCallStructMix; + +typedef union CCallStructClass { + uint32_t val; + struct { + uint8_t ispod; + CCallStructMix mix; + }; +} CCallStructClass; + +static CCallStructClass ccall_classify_struct(CTState *cts, CType *ct, + CCallMixFieldList *list, CTSize base_ofs) +{ + CTSize sz = ct->size; + CCallStructMix mix = { .val = MIX_UNINIT }; + if (ct->info & CTF_UNION) mix.val = MIX_FAILED; + while (ct->sib && mix.val != MIX_FAILED) { + unsigned int m = 1; + CType *sct; + ct = ctype_get(cts, ct->sib); + { + CTSize fofs = ct->size; /* Field offset within this struct. */ + if (ctype_isfield(ct->info)) { + sct = ctype_rawchild(cts, ct); + if (ctype_isarray(sct->info) && !sct->size) continue; + while (ctype_isarray(sct->info)) { + CType *cct = ctype_rawchild(cts, sct); + m *= sct->size / cct->size; + sct = cct; + } + { + CTSize esz = sct->size; + if (ctype_isfp(sct->info)) { + unsigned int mi; + for (mi = 0; mi < m; mi++) { + /* Mix state trans: fp (float or double) + * mix XX -> mix = FX + * mix X[IF] -> mix = FAILED + * mix [IF]X -> mix = [IF]F + * mix [IF][IF] -> mix = FAILED + */ + eCCallStructMixElem ne = MIX_ELEM_FLOAT; + if (mix.val == MIX_UNINIT) { + mix = (CCallStructMix){ .e1 = ne, .e2 = MIX_ELEM_UNINIT }; + } else { + eCCallStructMixElem o1 = mix.e1, o2 = mix.e2, n2 = ne; + mix = (o2 != MIX_ELEM_UNINIT ? (CCallStructMix){ .val = MIX_FAILED } + : (CCallStructMix){ .e1 = o1, .e2 = n2 }); + } + ccall_add_mix_field(list, base_ofs + fofs + mi*esz, esz, 1); + } + } else if (ctype_iscomplex(sct->info)) { + unsigned int mi; + CType *esct = ctype_rawchild(cts, sct); + CTSize cesz = esct->size; + for (mi = 0; mi < m; mi++) { + /* Mix state trans: complex + * mix XX -> mix = FF + * mix other -> mix = FAILED + */ + eCCallStructMixElem ne = MIX_ELEM_FLOAT; + mix = (mix.val == MIX_UNINIT) ? (CCallStructMix){ .e1 = ne, .e2 = ne } + : (CCallStructMix){ .val = MIX_FAILED }; + ccall_add_mix_field(list, base_ofs + fofs + mi*cesz*2, cesz, 1); + ccall_add_mix_field(list, base_ofs + fofs + mi*cesz*2 + cesz, cesz, 1); + } + } else if (ctype_isinteger_or_bool(sct->info) || ctype_isenum(sct->info)) { + unsigned int mi; + for (mi = 0; mi < m; mi++) { + /* Mix state trans: int + * mix XX -> mix = IX + * mix X[IF] -> mix = FAILED + * mix [IF]X -> mix = [IF]I; this auto fails II + * mix [IF][IF] -> mix = FAILED + */ + if (mix.val == MIX_UNINIT) { + mix = (CCallStructMix){ .e1 = MIX_ELEM_INT, .e2 = MIX_ELEM_UNINIT }; + } else { + eCCallStructMixElem o1 = mix.e1, o2 = mix.e2, n2 = MIX_ELEM_INT; + mix = (o2 != MIX_ELEM_UNINIT ? (CCallStructMix){ .val = MIX_FAILED } + : (CCallStructMix){ .e1 = o1, .e2 = n2 }); + } + ccall_add_mix_field(list, base_ofs + fofs + mi*esz, esz, 0); + } + } else if (ctype_isstruct(sct->info)) { + goto substruct; + } else { + goto not_ag; + } + } + } else if (ctype_isbitfield(ct->info) && ctype_bitbsz(ct->info)) { + /* Non-zero bitfield: treat as integer element for MIX classification. */ + CTSize fofs = ct->size; + sct = ctype_rawchild(cts, ct); + { + /* Mix state trans: bitfield + * mix XX -> mix = IX + * mix X[IF] -> mix = FAILED + * mix [IF]X -> mix = [IF]I + * mix [IF][IF] -> mix = FAILED + */ + if (mix.val == MIX_UNINIT) { + mix = (CCallStructMix){ .e1 = MIX_ELEM_INT, .e2 = MIX_ELEM_UNINIT }; + } else { + eCCallStructMixElem o1 = mix.e1, o2 = mix.e2, n2 = MIX_ELEM_INT; + mix = (o2 != MIX_ELEM_UNINIT ? (CCallStructMix){ .val = MIX_FAILED } + : (CCallStructMix){ .e1 = o1, .e2 = n2 }); + } + ccall_add_mix_field(list, base_ofs + fofs, sct->size, 0); + } + } else if (ctype_isxattrib(ct->info, CTA_SUBTYPE)) { + sct = ctype_rawchild(cts, ct); + substruct: + if (sct->size > 0) { + CCallStructClass s = ccall_classify_struct(cts, sct, list, + base_ofs + fofs); + CCallStructMix smix = s.mix; + uint8_t spod = s.ispod; + if (smix.val == MIX_FAILED) mix.val = MIX_FAILED; + if (!spod) goto not_ag; + while (m--) { + /* Mix state transfer: substruct + * mix XX, smix any -> mix = smix + * mix X[IF], smix any -> mix = FAILED + * mix [IF]X, smix [XIF]X -> mix = [IF][XIF] + * smix other -> mix = FAILED + * mix [IF][IF], smix XX -> mix = mix + * smix other -> mix = FAILED; this keeps II fail + */ + if (mix.val == MIX_UNINIT) { + mix = smix; + } else { + eCCallStructMixElem o1 = mix.e1, o2 = mix.e2; + eCCallStructMixElem n1 = smix.e1, n2 = smix.e2; + if (o2 != MIX_ELEM_UNINIT) { + mix = (smix.val != MIX_UNINIT) ? (CCallStructMix){ .val = MIX_FAILED } + : mix; + } else { + mix = (n2 != MIX_ELEM_UNINIT) ? (CCallStructMix){ .val = MIX_FAILED } + : (CCallStructMix){ .e1 = o1, .e2 = n1 }; + } + } + } + } + } + } + } + if (MIX_UNINIT < mix.val && mix.val < MIX_FAILED && mix.e1 != MIX_ELEM_UNINIT) /* Mixed passing */ + return (CCallStructClass){ .ispod = 1, .mix = mix }; +not_ag: /* Not a float/double aggregate or int/fp mix pair aggregate */ + if (list) list->nfields = 0; /* Invalidate flat list on classification failure. */ + return (CCallStructClass){ .ispod = (sz <= 16), .mix = mix }; +} + +#endif + #ifndef ccall_struct_align /* Alignment of pass-by-value structs. */ #define ccall_struct_align(cts, ct) ((ct)->info & CTF_ALIGN) @@ -985,6 +1376,7 @@ static int ccall_set_args(lua_State *L, CTState *cts, CType *ct, #endif #endif + /* Clear unused regs to get some determinism in case of misdeclaration. */ memset(cc->gpr, 0, sizeof(cc->gpr)); #if CCALL_NUM_FPR @@ -1047,6 +1439,9 @@ static int ccall_set_args(lua_State *L, CTState *cts, CType *ct, CType *d; CTSize sz; MSize n, isfp = 0, isva = 0; +#if LJ_TARGET_RISCV64 + CCallStructMix mix = { .val = MIX_UNINIT }; +#endif void *dp, *rp = NULL; #if LJ_TARGET_X64 && !LJ_ABI_WIN int onstack = 0; @@ -1131,7 +1526,11 @@ static int ccall_set_args(lua_State *L, CTState *cts, CType *ct, if (isfp && d->size == sizeof(float)) ((float *)dp)[1] = ((float *)dp)[0]; /* Floats occupy high slot. */ #endif -#if LJ_TARGET_MIPS64 || (LJ_TARGET_ARM64 && LJ_BE) +#if LJ_TARGET_RISCV64 + if (isfp && d->size == sizeof(float)) + ((uint32_t *)dp)[1] = 0xffffffffu; /* Float NaN boxing */ +#endif +#if LJ_TARGET_MIPS64 || (LJ_TARGET_ARM64 && LJ_BE) || LJ_TARGET_RISCV64 if ((ctype_isinteger_or_bool(d->info) || ctype_isenum(d->info) #if LJ_TARGET_MIPS64 || (isfp && nsp == 0) diff --git a/src/lj_ccall.h b/src/lj_ccall.h index e7c2dc1f..4130983b 100644 --- a/src/lj_ccall.h +++ b/src/lj_ccall.h @@ -129,6 +129,23 @@ typedef union FPRArg { struct { LJ_ENDIAN_LOHI(float f; , float g;) }; } FPRArg; +#elif LJ_TARGET_RISCV64 + +#define CCALL_NARG_GPR 8 +#define CCALL_NARG_FPR 8 +#define CCALL_NRET_GPR 2 +#define CCALL_NRET_FPR 2 +#define CCALL_SPS_EXTRA 3 +#define CCALL_SPS_FREE 1 + +typedef intptr_t GPRArg; +typedef union FPRArg { + double d; + uint64_t u; + struct { LJ_ENDIAN_LOHI(float f; , float g;) }; + struct { LJ_ENDIAN_LOHI(uint32_t lo; , uint32_t hi;) }; +} FPRArg; + #else #error "Missing calling convention definitions for this architecture" #endif diff --git a/src/lj_ccallback.c b/src/lj_ccallback.c index a7979a5d..190cc03c 100644 --- a/src/lj_ccallback.c +++ b/src/lj_ccallback.c @@ -82,6 +82,10 @@ static MSize CALLBACK_OFS2SLOT(MSize ofs) #define CALLBACK_MCODE_HEAD 52 +#elif LJ_TARGET_RISCV64 + +#define CALLBACK_MCODE_HEAD 68 + #else /* Missing support for this architecture. */ @@ -260,6 +264,39 @@ static void *callback_mcode_init(global_State *g, uint32_t *page) } return p; } +#elif LJ_TARGET_RISCV64 +static void *callback_mcode_init(global_State *g, uint32_t *page) +{ + uint32_t *p = page; + uintptr_t target = (uintptr_t)(void *)lj_vm_ffi_callback; + uintptr_t ug = (uintptr_t)(void *)g; + uintptr_t target_hi = (target >> 32), target_lo = target & 0xffffffffULL; + uintptr_t ug_hi = (ug >> 32), ug_lo = ug & 0xffffffffULL; + MSize slot; + *p++ = RISCVI_LUI | RISCVF_D(RID_X6) | RISCVF_IMMU(RISCVF_HI(target_hi)); + *p++ = RISCVI_LUI | RISCVF_D(RID_X7) | RISCVF_IMMU(RISCVF_HI(ug_hi)); + *p++ = RISCVI_ADDI | RISCVF_D(RID_X6) | RISCVF_S1(RID_X6) | RISCVF_IMMI(RISCVF_LO(target_hi)); + *p++ = RISCVI_ADDI | RISCVF_D(RID_X7) | RISCVF_S1(RID_X7) | RISCVF_IMMI(RISCVF_LO(ug_hi)); + *p++ = RISCVI_SLLI | RISCVF_D(RID_X6) | RISCVF_S1(RID_X6) | RISCVF_SHAMT(11); + *p++ = RISCVI_SLLI | RISCVF_D(RID_X7) | RISCVF_S1(RID_X7) | RISCVF_SHAMT(11); + *p++ = RISCVI_ADDI | RISCVF_D(RID_X6) | RISCVF_S1(RID_X6) | RISCVF_IMMI(target_lo >> 21); + *p++ = RISCVI_ADDI | RISCVF_D(RID_X7) | RISCVF_S1(RID_X7) | RISCVF_IMMI(ug_lo >> 21); + *p++ = RISCVI_SLLI | RISCVF_D(RID_X6) | RISCVF_S1(RID_X6) | RISCVF_SHAMT(11); + *p++ = RISCVI_SLLI | RISCVF_D(RID_X7) | RISCVF_S1(RID_X7) | RISCVF_SHAMT(11); + *p++ = RISCVI_ADDI | RISCVF_D(RID_X6) | RISCVF_S1(RID_X6) | RISCVF_IMMI((target_lo >> 10) & 0x7ff); + *p++ = RISCVI_ADDI | RISCVF_D(RID_X7) | RISCVF_S1(RID_X7) | RISCVF_IMMI((ug_lo >> 10) & 0x7ff); + *p++ = RISCVI_SLLI | RISCVF_D(RID_X6) | RISCVF_S1(RID_X6) | RISCVF_SHAMT(10); + *p++ = RISCVI_SLLI | RISCVF_D(RID_X7) | RISCVF_S1(RID_X7) | RISCVF_SHAMT(10); + *p++ = RISCVI_ADDI | RISCVF_D(RID_X6) | RISCVF_S1(RID_X6) | RISCVF_IMMI(target_lo & 0x3ff); + *p++ = RISCVI_ADDI | RISCVF_D(RID_X7) | RISCVF_S1(RID_X7) | RISCVF_IMMI(ug_lo & 0x3ff); + *p++ = RISCVI_JALR | RISCVF_D(RID_X0) | RISCVF_S1(RID_X6) | RISCVF_IMMJ(0); + for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) { + *p++ = RISCVI_LUI | RISCVF_D(RID_X5) | RISCVF_IMMU(slot); + *p = RISCVI_JAL | RISCVF_IMMJ(((char *)page-(char *)p)); + p++; + } + return p; +} #else /* Missing support for this architecture. */ #define callback_mcode_init(g, p) (p) @@ -549,6 +586,31 @@ void lj_ccallback_mcode_free(CTState *cts) } #endif +#define CALLBACK_HANDLE_RET \ + if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \ + ((float *)dp)[1] = *(float *)dp; + +#elif LJ_TARGET_RISCV64 + +#define CALLBACK_HANDLE_REGARG \ + if (isfp) { \ + if (nfpr + n <= CCALL_NARG_FPR) { \ + sp = &cts->cb.fpr[nfpr]; \ + nfpr += n; \ + goto done; \ + } else if (ngpr + n <= maxgpr) { \ + sp = &cts->cb.gpr[ngpr]; \ + ngpr += n; \ + goto done; \ + } \ + } else { \ + if (ngpr + n <= maxgpr) { \ + sp = &cts->cb.gpr[ngpr]; \ + ngpr += n; \ + goto done; \ + } \ + } + #define CALLBACK_HANDLE_RET \ if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \ ((float *)dp)[1] = *(float *)dp; @@ -699,7 +761,7 @@ static void callback_conv_result(CTState *cts, lua_State *L, TValue *o) *(int32_t *)dp = ctr->size == 1 ? (int32_t)*(int8_t *)dp : (int32_t)*(int16_t *)dp; } -#if LJ_TARGET_MIPS64 || (LJ_TARGET_ARM64 && LJ_BE) +#if LJ_TARGET_MIPS64 || (LJ_TARGET_ARM64 && LJ_BE) || LJ_TARGET_RISCV64 /* Always sign-extend results to 64 bits. Even a soft-fp 'float'. */ if (ctr->size <= 4 && (LJ_ABI_SOFTFP || ctype_isinteger_or_bool(ctr->info))) diff --git a/src/vm_riscv64.dasc b/src/vm_riscv64.dasc index 4cf4d8e0..11588b94 100644 --- a/src/vm_riscv64.dasc +++ b/src/vm_riscv64.dasc @@ -811,14 +811,29 @@ static void build_subroutines(BuildCtx *ctx) | mv BASE, TMP2 // Restore caller BASE. | ld LFUNC:TMP1, FRAME_FUNC(TMP2) | ld PC, -24(RB) // Restore PC from [cont|PC]. + |.if FFI + | sltiu TMP3, TMP0, 2 + |.endif | cleartp LFUNC:TMP1 | add TMP2, RA, RD | ld TMP1, LFUNC:TMP1->pc | sd TISNIL, -8(TMP2) // Ensure one valid arg. + |.if FFI + | bnez TMP3, >1 + |.endif | // BASE = base, RA = resultptr, RB = meta base | ld KBASE, PC2PROTO(k)(TMP1) | jr TMP0 // Jump to continuation. | + |.if FFI + |1: + | addi TMP1, RB, -32 + | bxnez TMP0, ->cont_ffi_callback // cont = 1: return from FFI callback. + | // cont = 0: tailcall from C function. + | sub RC, TMP1, BASE + | j ->vm_call_tail + |.endif + | |->cont_cat: // RA = resultptr, RB = meta base | lw INS, -4(PC) | addi CARG2, RB, -32 @@ -1018,6 +1033,18 @@ static void build_subroutines(BuildCtx *ctx) | // Returns 0/1 or TValue * (metamethod). | j <3 | + |->vmeta_equal_cd: + |.if FFI + | addi PC, PC, -4 + | mv CARG1, L + | mv CARG2, INS + | sd BASE, L->base + | sd PC, SAVE_PC(sp) + | call_intern vmeta_equal_cd, lj_meta_equal_cd // (lua_State *L, BCIns op) + | // Returns 0/1 or TValue * (metamethod). + | j <3 + |.endif + | |->vmeta_istype: | addi PC, PC, -4 | sd BASE, L->base @@ -2278,6 +2305,130 @@ static void build_subroutines(BuildCtx *ctx) |.endif | |//----------------------------------------------------------------------- + |//-- FFI helper functions ----------------------------------------------- + |//----------------------------------------------------------------------- + | + |// Handler for callback functions. Callback slot number in x5, g in x7. + |->vm_ffi_callback: + |.if FFI + |.type CTSTATE, CTState, PC + | saveregs + | ld CTSTATE, GL:x7->ctype_state + | mv GL, x7 + | addxi DISPATCH, x7, GG_G2DISP + | srli x5, x5, 12 + | sw x5, CTSTATE->cb.slot + | sd CARG1, CTSTATE->cb.gpr[0] + | fsd FARG1, CTSTATE->cb.fpr[0] + | sd CARG2, CTSTATE->cb.gpr[1] + | fsd FARG2, CTSTATE->cb.fpr[1] + | sd CARG3, CTSTATE->cb.gpr[2] + | fsd FARG3, CTSTATE->cb.fpr[2] + | sd CARG4, CTSTATE->cb.gpr[3] + | fsd FARG4, CTSTATE->cb.fpr[3] + | sd CARG5, CTSTATE->cb.gpr[4] + | fsd FARG5, CTSTATE->cb.fpr[4] + | sd CARG6, CTSTATE->cb.gpr[5] + | fsd FARG6, CTSTATE->cb.fpr[5] + | sd CARG7, CTSTATE->cb.gpr[6] + | fsd FARG7, CTSTATE->cb.fpr[6] + | sd CARG8, CTSTATE->cb.gpr[7] + | fsd FARG8, CTSTATE->cb.fpr[7] + | addi TMP0, sp, CFRAME_SPACE + | sd TMP0, CTSTATE->cb.stack + | sd x0, SAVE_PC(sp) // Any value outside of bytecode is ok. + | mv CARG1, CTSTATE + | mv CARG2, sp + | call_intern vm_ffi_callback, lj_ccallback_enter // (CTState *cts, void *cf) + | // Returns lua_State *. + | ld BASE, L:CRET1->base + | ld RC, L:CRET1->top + | mv L, CRET1 + | lui TMP3, 0x43380 // TOBIT = Hiword of 2^52 + 2^51 (double). + | ld LFUNC:RB, FRAME_FUNC(BASE) + | li TISNIL, LJ_TNIL + | li TISNUM, LJ_TISNUM + | slli TMP3, TMP3, 32 + | li_vmstate INTERP + | subw RC, RC, BASE + | cleartp LFUNC:RB + | st_vmstate + | fmv.d.x TOBIT, TMP3 + | ins_callt + |.endif + | + |->cont_ffi_callback: // Return from FFI callback. + |.if FFI + | ld CTSTATE, GL->ctype_state + | sd BASE, L->base + | sd RB, L->top + | sd L, CTSTATE->L + | mv CARG1, CTSTATE + | mv CARG2, RA + | // (CTState *cts, TValue *o) + | call_intern cont_ffi_callback, lj_ccallback_leave + | fld FRET1, CTSTATE->cb.fpr[0] + | ld CRET1, CTSTATE->cb.gpr[0] + | fld FRET2, CTSTATE->cb.fpr[1] + | ld CRET2, CTSTATE->cb.gpr[1] + | j ->vm_leave_unw + |.endif + | + |->vm_ffi_call: // Call C function via FFI. + | // Caveat: needs special frame unwinding, see below. + |.if FFI + | .type CCSTATE, CCallState, CARG1 + | lw TMP1, CCSTATE->spadj + | lbu CARG2, CCSTATE->nsp + | mv TMP2, sp + | sub sp, sp, TMP1 + | sd ra, -8(TMP2) + | sd x18, -16(TMP2) + | sd CCSTATE, -24(TMP2) + | mv x18, TMP2 + | addi TMP1, CCSTATE, offsetof(CCallState, stack) + | mv TMP2, sp + | add TMP3, TMP1, CARG2 + | beqz CARG2, >2 + |1: + | ld TMP0, 0(TMP1) + | addi TMP1, TMP1, 8 + | sd TMP0, 0(TMP2) + | addi TMP2, TMP2, 8 + | bltu TMP1, TMP3, <1 + |2: + | fld FARG1, CCSTATE->fpr[0] + | fld FARG2, CCSTATE->fpr[1] + | fld FARG3, CCSTATE->fpr[2] + | fld FARG4, CCSTATE->fpr[3] + | fld FARG5, CCSTATE->fpr[4] + | fld FARG6, CCSTATE->fpr[5] + | fld FARG7, CCSTATE->fpr[6] + | fld FARG8, CCSTATE->fpr[7] + | ld CFUNCADDR, CCSTATE->func + | ld CARG2, CCSTATE->gpr[1] + | ld CARG3, CCSTATE->gpr[2] + | ld CARG4, CCSTATE->gpr[3] + | ld CARG5, CCSTATE->gpr[4] + | ld CARG6, CCSTATE->gpr[5] + | ld CARG7, CCSTATE->gpr[6] + | ld CARG8, CCSTATE->gpr[7] + | ld CARG1, CCSTATE->gpr[0] // Do this last, since CCSTATE is CARG1. + | jalr CFUNCADDR + | ld CCSTATE:TMP1, -24(x18) + | ld TMP0, -16(x18) + | ld ra, -8(x18) + | sd CRET1, CCSTATE:TMP1->gpr[0] + | sd CRET2, CCSTATE:TMP1->gpr[1] + | fsd FRET1, CCSTATE:TMP1->fpr[0] + | fsd FRET2, CCSTATE:TMP1->fpr[1] + | mv sp, x18 + | mv x18, TMP0 + | ret + |.endif + |// Note: vm_ffi_call must be the last function in this object file! + | + |//----------------------------------------------------------------------- } /* Generate the code for a single instruction. */ @@ -2402,6 +2553,15 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | beqz TMP0, ->BC_ISNEN_Z } |// Either or both types are not numbers. + |.if FFI + | // Check if RA or RD is a cdata. + | xori TMP0, CARG3, LJ_TCDATA + | xori TMP1, CARG4, LJ_TCDATA + | seqz TMP0, TMP0 + | seqz TMP1, TMP1 + | or TMP0, TMP0, TMP1 + | bxnez TMP0, ->vmeta_equal_cd + |.endif | lui TMP3, (-(BCBIAS_J*4 >> 12)) & 0xfffff // -BCBIAS_J*4 | decode_BC4b TMP2 | addw TMP2, TMP2, TMP3 // (jump-0x8000)<<2 @@ -2454,10 +2614,17 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | sub RD, KBASE, RD | lhu TMP2, -4+OFS_RD(PC) | ld CARG2, -8(RD) // KBASE-8-str_const*8 + |.if FFI + | gettp CARG3, CARG1 + | li TMP1, LJ_TCDATA + |.endif | li TMP0, LJ_TSTR | decode_BC4b TMP2 | settp CARG2, TMP0 | lui TMP3, (-(BCBIAS_J*4 >> 12)) & 0xfffff // -BCBIAS_J*4 + |.if FFI + | bxeq CARG3, TMP1, ->vmeta_equal_cd + |.endif | xor TMP0, CARG1, CARG2 // TMP2=0: A==D; TMP2!=0: A!=D | addw TMP2, TMP2, TMP3 if (vk) { @@ -2512,7 +2679,11 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | |4: // RA is not an integer. | addw TMP2, TMP2, TMP3 + |.if FFI + | bgeu CARG3, TISNUM, >7 + |.else | bgeu CARG3, TISNUM, <2 + |.endif | fmv.d.x FTMP0, CARG1 | fmv.d.x FTMP2, CARG2 | bne CARG4, TISNUM, >5 @@ -2525,11 +2696,26 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | j <1 | |6: // RA is an integer, RD is a number. + |.if FFI + | bgeu CARG4, TISNUM, >8 + |.else | bgeu CARG4, TISNUM, <2 + |.endif | fcvt.d.w FTMP0, CARG1 | fmv.d.x FTMP2, CARG2 | j <5 | + |.if FFI + |7: // RA not int, not number + | li TMP0, LJ_TCDATA + | bne CARG3, TMP0, <2 + | j ->vmeta_equal_cd + | + |8: // RD not int, not number + | li TMP0, LJ_TCDATA + | bne CARG4, TMP0, <2 + | j ->vmeta_equal_cd + |.endif break; case BC_ISEQP: case BC_ISNEP: @@ -2543,6 +2729,10 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | gettp TMP1, TMP1 | addi PC, PC, 4 | xor TMP0, TMP1, TMP0 // TMP0=0 A=D; TMP0!=0 A!=D + |.if FFI + | li TMP3, LJ_TCDATA + | bxeq TMP1, TMP3, ->vmeta_equal_cd + |.endif | decode_BC4b TMP2 | lui TMP3, (-(BCBIAS_J*4 >> 12)) & 0xfffff // -BCBIAS_J*4 | addw TMP2, TMP2, TMP3 // TMP2=(jump-0x8000)<<2 @@ -2942,6 +3132,16 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | ins_next break; case BC_KCDATA: + |.if FFI + | // RA = dst*8, RD = cdata_const*8 (~) + | sub TMP1, KBASE, RD + | ld TMP0, -8(TMP1) // KBASE-8-cdata_const*8 + | li TMP2, LJ_TCDATA + | add RA, BASE, RA + | settp TMP0, TMP2 + | sd TMP0, 0(RA) + | ins_next + |.endif break; case BC_KSHORT: | // RA = dst*8, RD = int16_literal*8