riscv(ffi): add call convention and support framework
This commit is contained in:
gns
parent
a5990be55a
commit
23d49c6973
401
src/lj_ccall.c
401
src/lj_ccall.c
@ -577,6 +577,197 @@
|
||||
goto done; \
|
||||
}
|
||||
|
||||
#elif LJ_TARGET_RISCV64
|
||||
/* -- RISC-V lp64d calling conventions ------------------------------------ */
|
||||
|
||||
/* Forward declarations for functions used by macros below. */
|
||||
#define CCALL_MIX_MAXFIELDS 2 /* MIX state machine has exactly 2 slots. */
|
||||
typedef struct {
|
||||
CTSize offset;
|
||||
uint8_t esz;
|
||||
uint8_t isfp;
|
||||
} CCallMixField;
|
||||
|
||||
typedef struct {
|
||||
uint8_t nfields;
|
||||
CCallMixField fields[CCALL_MIX_MAXFIELDS];
|
||||
} CCallMixFieldList;
|
||||
|
||||
static CTSize ccall_place_mix_fields(CCallState *cc, CCallMixFieldList *list,
|
||||
CTSize sz, uint8_t *src,
|
||||
MSize *pnfpr, MSize *pngpr, int isva);
|
||||
static void ccall_unplace_mix_fields(CCallState *cc, CCallMixFieldList *list,
|
||||
uint8_t *dp);
|
||||
|
||||
#define CCALL_HANDLE_STRUCTRET \
|
||||
/* Return by reference only if >16 bytes and not classifiable as MIX. */ \
|
||||
cc->retref = !ccall_classify_struct(cts, ctr, NULL, 0).ispod && sz > 16; \
|
||||
if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
|
||||
|
||||
#define CCALL_HANDLE_STRUCTRET2 \
|
||||
{ \
|
||||
CCallMixFieldList mixlist = {0}; \
|
||||
CCallStructClass cl = ccall_classify_struct(cts, ctr, &mixlist, 0); \
|
||||
CCallStructMix mix = cl.mix; \
|
||||
switch (mix.val) { \
|
||||
case MIX_IX: case MIX_FX: \
|
||||
case MIX_FF: case MIX_IF: case MIX_FI: { \
|
||||
ccall_unplace_mix_fields(cc, &mixlist, (uint8_t *)dp); \
|
||||
break; \
|
||||
} \
|
||||
case MIX_UNINIT: \
|
||||
case MIX_FAILED: { \
|
||||
memcpy(dp, sp, ctr->size); \
|
||||
break; \
|
||||
} \
|
||||
default: lj_assertL(0, "Invalid ret mix %d", mix.val); \
|
||||
} \
|
||||
}
|
||||
|
||||
#define CCALL_HANDLE_COMPLEXRET \
|
||||
/* Complex values are returned in 1 or 2 FPRs. */ \
|
||||
cc->retref = 0;
|
||||
|
||||
#define CCALL_HANDLE_COMPLEXRET2 \
|
||||
if (ctr->size == 2*sizeof(float)) { /* Copy complex float from FPRs. */ \
|
||||
((float *)dp)[0] = cc->fpr[0].f; \
|
||||
((float *)dp)[1] = cc->fpr[1].f; \
|
||||
} else { /* Copy complex double from FPRs. */ \
|
||||
((double *)dp)[0] = cc->fpr[0].d; \
|
||||
((double *)dp)[1] = cc->fpr[1].d; \
|
||||
}
|
||||
|
||||
#define CCALL_HANDLE_COMPLEXARG \
|
||||
/* Pass long double complex by reference. */ \
|
||||
if (sz == 2*sizeof(long double)) { \
|
||||
rp = cdataptr(lj_cdata_new(cts, did, sz)); \
|
||||
sz = CTSIZE_PTR; \
|
||||
} \
|
||||
/* Pass complex in two FPRs or two GPRs or on stack. */ \
|
||||
else if (sz == 2*sizeof(float)) { \
|
||||
mix = (CCallStructMix){ .val = MIX_FF }; \
|
||||
sz = CTSIZE_PTR; \
|
||||
} else /*if (sz == 2*sizeof(double))*/ { \
|
||||
mix = (CCallStructMix){ .val = MIX_FF }; \
|
||||
sz = 2*CTSIZE_PTR; \
|
||||
}
|
||||
|
||||
#define CCALL_HANDLE_RET \
|
||||
if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
|
||||
sp = (uint8_t *)&cc->fpr[0].f;
|
||||
|
||||
#define CCALL_HANDLE_STRUCTARG \
|
||||
/* Pass structs of size >16 by reference if not classifiable as MIX. */ \
|
||||
CCallMixFieldList mixlist = {0}; \
|
||||
CCallStructClass cl = ccall_classify_struct(cts, d, &mixlist, 0); \
|
||||
uint8_t ispod = cl.ispod; \
|
||||
mix = cl.mix; \
|
||||
/* Check if MIX registers available; fall back to integer conv if not */ \
|
||||
if (ispod && mix.val != MIX_FAILED && mix.val != MIX_UNINIT) { \
|
||||
int need_fpr = 0, need_gpr = 0; \
|
||||
switch (mix.val) { \
|
||||
case MIX_FX: need_fpr = 1; break; \
|
||||
case MIX_FF: need_fpr = 2; break; \
|
||||
case MIX_FI: case MIX_IF: need_fpr = 1; need_gpr = 1; break; \
|
||||
default: lj_assertCTS(0, "Invalid MIX %d", mix.val); \
|
||||
} \
|
||||
if ((need_fpr && nfpr + need_fpr > CCALL_NARG_FPR) || \
|
||||
(need_gpr && ngpr + need_gpr > CCALL_NARG_GPR)) \
|
||||
mix.val = MIX_FAILED; /* Fall back to integer convention */ \
|
||||
} \
|
||||
/* Integer convention >2xXLEN: pass by reference */ \
|
||||
if (ispod && (mix.val == MIX_FAILED || mix.val == MIX_UNINIT) && sz > 16) \
|
||||
ispod = 0; /* Route to by-reference path below. */ \
|
||||
if (!ispod && sz > 16) { \
|
||||
rp = cdataptr(lj_cdata_new(cts, did, sz)); \
|
||||
sz = CTSIZE_PTR; \
|
||||
} else if (ispod) { \
|
||||
/* POD struct: convert to stack, then place fields in registers. */ \
|
||||
MSize al = (1u << ctype_align(ccall_struct_align(cts, d))) - 1; \
|
||||
MSize ns = (nsp + al) & ~al; \
|
||||
if (ns + sz > CCALL_SIZE_STACK) goto err_nyi; \
|
||||
uint8_t *src = (uint8_t *)cc->stack + ns; \
|
||||
lj_cconv_ct_tv(cts, d, src, o, CCF_ARG(narg)); \
|
||||
if (mix.val == MIX_FAILED || mix.val == MIX_UNINIT) { \
|
||||
/* Integer convention: place 8-byte chunks in GPRs, rest on stack. */ \
|
||||
int nchunks = (sz + 7) >> 3; \
|
||||
int chunks_avail = CCALL_NARG_GPR - (int)ngpr; \
|
||||
int i; \
|
||||
for (i = 0; i < chunks_avail && i < nchunks; i++) \
|
||||
cc->gpr[ngpr++] = ((GPRArg *)src)[i]; \
|
||||
if (i < nchunks) { \
|
||||
memmove(src, src + i * 8, sz - i * 8); \
|
||||
nsp = ns + (nchunks - i) * 8; \
|
||||
} \
|
||||
} else { \
|
||||
/* MIX convention: place fields from flat descriptor list. */ \
|
||||
CTSize tail = ccall_place_mix_fields(cc, &mixlist, sz, \
|
||||
src, &nfpr, &ngpr, isva); \
|
||||
if (tail < sz) { \
|
||||
memmove(src, src + tail, sz - tail); \
|
||||
nsp = ns + (sz - tail); \
|
||||
} \
|
||||
} \
|
||||
continue; \
|
||||
}
|
||||
|
||||
#define CCALL_HANDLE_REGARG \
|
||||
if (!isva) { \
|
||||
if (isfp && nfpr + 1 <= CCALL_NARG_FPR) { \
|
||||
dp = &cc->fpr[nfpr]; \
|
||||
nfpr += 1; \
|
||||
goto done; \
|
||||
} \
|
||||
} \
|
||||
if (ngpr + n <= maxgpr) { \
|
||||
dp = &cc->gpr[ngpr]; \
|
||||
ngpr += n; \
|
||||
goto done; \
|
||||
}
|
||||
|
||||
/* Place MIX struct fields from a flat descriptor list into registers.
|
||||
** Returns byte offset of first unplaced field (sz if all placed). */
|
||||
static CTSize ccall_place_mix_fields(CCallState *cc, CCallMixFieldList *list,
|
||||
CTSize sz, uint8_t *src,
|
||||
MSize *pnfpr, MSize *pngpr,
|
||||
int isva)
|
||||
{
|
||||
for (int i = 0; i < list->nfields; i++) {
|
||||
CCallMixField *f = &list->fields[i];
|
||||
uint64_t val;
|
||||
if (f->isfp) {
|
||||
val = (f->esz == 4) ? (*(uint32_t *)(src + f->offset) | U64x(ffffffff,00000000))
|
||||
: *(uint64_t *)(src + f->offset);
|
||||
if (!isva && *pnfpr < CCALL_NARG_FPR) cc->fpr[(*pnfpr)++].u = val;
|
||||
else if (*pngpr < CCALL_NARG_GPR) cc->gpr[(*pngpr)++] = val;
|
||||
else return f->offset;
|
||||
} else {
|
||||
val = (f->esz < 8) ? (uint64_t)(int64_t)*(int32_t *)(src + f->offset)
|
||||
: *(uint64_t *)(src + f->offset);
|
||||
if (*pngpr < CCALL_NARG_GPR) cc->gpr[(*pngpr)++] = val;
|
||||
else return f->offset;
|
||||
}
|
||||
}
|
||||
return sz; /* all placed */
|
||||
}
|
||||
|
||||
/* Reconstruct MIX struct fields from registers using flat descriptor list. */
|
||||
static void ccall_unplace_mix_fields(CCallState *cc, CCallMixFieldList *list,
|
||||
uint8_t *dp)
|
||||
{
|
||||
int gpi = 0, fpi = 0;
|
||||
for (int i = 0; i < list->nfields; i++) {
|
||||
CCallMixField *f = &list->fields[i];
|
||||
if (f->isfp) {
|
||||
if (f->esz == 4) *(float *)(dp + f->offset) = cc->fpr[fpi++].f;
|
||||
else *(double *)(dp + f->offset) = cc->fpr[fpi++].d;
|
||||
} else {
|
||||
if (f->esz < 8) *(int32_t *)(dp + f->offset) = (int32_t)cc->gpr[gpi++];
|
||||
else *(int64_t *)(dp + f->offset) = cc->gpr[gpi++];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#else
|
||||
#error "Missing calling convention definitions for this architecture"
|
||||
#endif
|
||||
@ -929,6 +1120,206 @@ static void ccall_copy_struct(CCallState *cc, CType *ctr, void *dp, void *sp,
|
||||
|
||||
#endif
|
||||
|
||||
/* -- RISC-V ABI struct classification ---------------------------- */
|
||||
|
||||
#if LJ_TARGET_RISCV64
|
||||
|
||||
/* RISC-V 64 LP64D fp reg struct classification. */
|
||||
/* X: unknown/uninit, F: float, D: double, I: integer */
|
||||
|
||||
/* Flat descriptor for a single field element produced by classification.
|
||||
** Place/unplace iterate this list instead of re-walking the ctype tree. */
|
||||
#define CCALL_MIX_MAXFIELDS 2 /* MIX state machine has exactly 2 slots. */
|
||||
|
||||
/* Add a field descriptor to the flat list (if space remains). */
|
||||
static LJ_AINLINE void ccall_add_mix_field(CCallMixFieldList *list,
|
||||
CTSize offset, CTSize esz, int isfp)
|
||||
{
|
||||
if (list && list->nfields < CCALL_MIX_MAXFIELDS) {
|
||||
CCallMixField *f = &list->fields[list->nfields++];
|
||||
f->offset = offset;
|
||||
f->esz = (uint8_t)esz;
|
||||
f->isfp = (uint8_t)isfp;
|
||||
}
|
||||
}
|
||||
|
||||
typedef enum eCCallStructMixElem {
|
||||
MIX_ELEM_UNINIT = 0,
|
||||
MIX_ELEM_FLOAT = 1,
|
||||
MIX_ELEM_INT = 2,
|
||||
} eCCallStructMixElem;
|
||||
typedef enum eCCallStructMix {
|
||||
MIX_UNINIT = 0, /* XX */
|
||||
MIX_FX = 1, /* F X — single float/double */
|
||||
MIX_IX = 2, /* I X — single int */
|
||||
MIX_FF = 5, /* F F — two floats/doubles */
|
||||
MIX_IF = 6, /* I F — int then float/double */
|
||||
MIX_FI = 9, /* F I — float/double then int */
|
||||
MIX_FAILED = 10 /* anything else */
|
||||
} eCCallStructMix;
|
||||
|
||||
typedef union CCallStructMix {
|
||||
eCCallStructMix val : 4;
|
||||
struct {
|
||||
eCCallStructMixElem e1 : 2;
|
||||
eCCallStructMixElem e2 : 2;
|
||||
};
|
||||
} CCallStructMix;
|
||||
|
||||
typedef union CCallStructClass {
|
||||
uint32_t val;
|
||||
struct {
|
||||
uint8_t ispod;
|
||||
CCallStructMix mix;
|
||||
};
|
||||
} CCallStructClass;
|
||||
|
||||
static CCallStructClass ccall_classify_struct(CTState *cts, CType *ct,
|
||||
CCallMixFieldList *list, CTSize base_ofs)
|
||||
{
|
||||
CTSize sz = ct->size;
|
||||
CCallStructMix mix = { .val = MIX_UNINIT };
|
||||
if (ct->info & CTF_UNION) mix.val = MIX_FAILED;
|
||||
while (ct->sib && mix.val != MIX_FAILED) {
|
||||
unsigned int m = 1;
|
||||
CType *sct;
|
||||
ct = ctype_get(cts, ct->sib);
|
||||
{
|
||||
CTSize fofs = ct->size; /* Field offset within this struct. */
|
||||
if (ctype_isfield(ct->info)) {
|
||||
sct = ctype_rawchild(cts, ct);
|
||||
if (ctype_isarray(sct->info) && !sct->size) continue;
|
||||
while (ctype_isarray(sct->info)) {
|
||||
CType *cct = ctype_rawchild(cts, sct);
|
||||
m *= sct->size / cct->size;
|
||||
sct = cct;
|
||||
}
|
||||
{
|
||||
CTSize esz = sct->size;
|
||||
if (ctype_isfp(sct->info)) {
|
||||
unsigned int mi;
|
||||
for (mi = 0; mi < m; mi++) {
|
||||
/* Mix state trans: fp (float or double)
|
||||
* mix XX -> mix = FX
|
||||
* mix X[IF] -> mix = FAILED
|
||||
* mix [IF]X -> mix = [IF]F
|
||||
* mix [IF][IF] -> mix = FAILED
|
||||
*/
|
||||
eCCallStructMixElem ne = MIX_ELEM_FLOAT;
|
||||
if (mix.val == MIX_UNINIT) {
|
||||
mix = (CCallStructMix){ .e1 = ne, .e2 = MIX_ELEM_UNINIT };
|
||||
} else {
|
||||
eCCallStructMixElem o1 = mix.e1, o2 = mix.e2, n2 = ne;
|
||||
mix = (o2 != MIX_ELEM_UNINIT ? (CCallStructMix){ .val = MIX_FAILED }
|
||||
: (CCallStructMix){ .e1 = o1, .e2 = n2 });
|
||||
}
|
||||
ccall_add_mix_field(list, base_ofs + fofs + mi*esz, esz, 1);
|
||||
}
|
||||
} else if (ctype_iscomplex(sct->info)) {
|
||||
unsigned int mi;
|
||||
CType *esct = ctype_rawchild(cts, sct);
|
||||
CTSize cesz = esct->size;
|
||||
for (mi = 0; mi < m; mi++) {
|
||||
/* Mix state trans: complex
|
||||
* mix XX -> mix = FF
|
||||
* mix other -> mix = FAILED
|
||||
*/
|
||||
eCCallStructMixElem ne = MIX_ELEM_FLOAT;
|
||||
mix = (mix.val == MIX_UNINIT) ? (CCallStructMix){ .e1 = ne, .e2 = ne }
|
||||
: (CCallStructMix){ .val = MIX_FAILED };
|
||||
ccall_add_mix_field(list, base_ofs + fofs + mi*cesz*2, cesz, 1);
|
||||
ccall_add_mix_field(list, base_ofs + fofs + mi*cesz*2 + cesz, cesz, 1);
|
||||
}
|
||||
} else if (ctype_isinteger_or_bool(sct->info) || ctype_isenum(sct->info)) {
|
||||
unsigned int mi;
|
||||
for (mi = 0; mi < m; mi++) {
|
||||
/* Mix state trans: int
|
||||
* mix XX -> mix = IX
|
||||
* mix X[IF] -> mix = FAILED
|
||||
* mix [IF]X -> mix = [IF]I; this auto fails II
|
||||
* mix [IF][IF] -> mix = FAILED
|
||||
*/
|
||||
if (mix.val == MIX_UNINIT) {
|
||||
mix = (CCallStructMix){ .e1 = MIX_ELEM_INT, .e2 = MIX_ELEM_UNINIT };
|
||||
} else {
|
||||
eCCallStructMixElem o1 = mix.e1, o2 = mix.e2, n2 = MIX_ELEM_INT;
|
||||
mix = (o2 != MIX_ELEM_UNINIT ? (CCallStructMix){ .val = MIX_FAILED }
|
||||
: (CCallStructMix){ .e1 = o1, .e2 = n2 });
|
||||
}
|
||||
ccall_add_mix_field(list, base_ofs + fofs + mi*esz, esz, 0);
|
||||
}
|
||||
} else if (ctype_isstruct(sct->info)) {
|
||||
goto substruct;
|
||||
} else {
|
||||
goto not_ag;
|
||||
}
|
||||
}
|
||||
} else if (ctype_isbitfield(ct->info) && ctype_bitbsz(ct->info)) {
|
||||
/* Non-zero bitfield: treat as integer element for MIX classification. */
|
||||
CTSize fofs = ct->size;
|
||||
sct = ctype_rawchild(cts, ct);
|
||||
{
|
||||
/* Mix state trans: bitfield
|
||||
* mix XX -> mix = IX
|
||||
* mix X[IF] -> mix = FAILED
|
||||
* mix [IF]X -> mix = [IF]I
|
||||
* mix [IF][IF] -> mix = FAILED
|
||||
*/
|
||||
if (mix.val == MIX_UNINIT) {
|
||||
mix = (CCallStructMix){ .e1 = MIX_ELEM_INT, .e2 = MIX_ELEM_UNINIT };
|
||||
} else {
|
||||
eCCallStructMixElem o1 = mix.e1, o2 = mix.e2, n2 = MIX_ELEM_INT;
|
||||
mix = (o2 != MIX_ELEM_UNINIT ? (CCallStructMix){ .val = MIX_FAILED }
|
||||
: (CCallStructMix){ .e1 = o1, .e2 = n2 });
|
||||
}
|
||||
ccall_add_mix_field(list, base_ofs + fofs, sct->size, 0);
|
||||
}
|
||||
} else if (ctype_isxattrib(ct->info, CTA_SUBTYPE)) {
|
||||
sct = ctype_rawchild(cts, ct);
|
||||
substruct:
|
||||
if (sct->size > 0) {
|
||||
CCallStructClass s = ccall_classify_struct(cts, sct, list,
|
||||
base_ofs + fofs);
|
||||
CCallStructMix smix = s.mix;
|
||||
uint8_t spod = s.ispod;
|
||||
if (smix.val == MIX_FAILED) mix.val = MIX_FAILED;
|
||||
if (!spod) goto not_ag;
|
||||
while (m--) {
|
||||
/* Mix state transfer: substruct
|
||||
* mix XX, smix any -> mix = smix
|
||||
* mix X[IF], smix any -> mix = FAILED
|
||||
* mix [IF]X, smix [XIF]X -> mix = [IF][XIF]
|
||||
* smix other -> mix = FAILED
|
||||
* mix [IF][IF], smix XX -> mix = mix
|
||||
* smix other -> mix = FAILED; this keeps II fail
|
||||
*/
|
||||
if (mix.val == MIX_UNINIT) {
|
||||
mix = smix;
|
||||
} else {
|
||||
eCCallStructMixElem o1 = mix.e1, o2 = mix.e2;
|
||||
eCCallStructMixElem n1 = smix.e1, n2 = smix.e2;
|
||||
if (o2 != MIX_ELEM_UNINIT) {
|
||||
mix = (smix.val != MIX_UNINIT) ? (CCallStructMix){ .val = MIX_FAILED }
|
||||
: mix;
|
||||
} else {
|
||||
mix = (n2 != MIX_ELEM_UNINIT) ? (CCallStructMix){ .val = MIX_FAILED }
|
||||
: (CCallStructMix){ .e1 = o1, .e2 = n1 };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (MIX_UNINIT < mix.val && mix.val < MIX_FAILED && mix.e1 != MIX_ELEM_UNINIT) /* Mixed passing */
|
||||
return (CCallStructClass){ .ispod = 1, .mix = mix };
|
||||
not_ag: /* Not a float/double aggregate or int/fp mix pair aggregate */
|
||||
if (list) list->nfields = 0; /* Invalidate flat list on classification failure. */
|
||||
return (CCallStructClass){ .ispod = (sz <= 16), .mix = mix };
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#ifndef ccall_struct_align
|
||||
/* Alignment of pass-by-value structs. */
|
||||
#define ccall_struct_align(cts, ct) ((ct)->info & CTF_ALIGN)
|
||||
@ -985,6 +1376,7 @@ static int ccall_set_args(lua_State *L, CTState *cts, CType *ct,
|
||||
#endif
|
||||
#endif
|
||||
|
||||
|
||||
/* Clear unused regs to get some determinism in case of misdeclaration. */
|
||||
memset(cc->gpr, 0, sizeof(cc->gpr));
|
||||
#if CCALL_NUM_FPR
|
||||
@ -1047,6 +1439,9 @@ static int ccall_set_args(lua_State *L, CTState *cts, CType *ct,
|
||||
CType *d;
|
||||
CTSize sz;
|
||||
MSize n, isfp = 0, isva = 0;
|
||||
#if LJ_TARGET_RISCV64
|
||||
CCallStructMix mix = { .val = MIX_UNINIT };
|
||||
#endif
|
||||
void *dp, *rp = NULL;
|
||||
#if LJ_TARGET_X64 && !LJ_ABI_WIN
|
||||
int onstack = 0;
|
||||
@ -1131,7 +1526,11 @@ static int ccall_set_args(lua_State *L, CTState *cts, CType *ct,
|
||||
if (isfp && d->size == sizeof(float))
|
||||
((float *)dp)[1] = ((float *)dp)[0]; /* Floats occupy high slot. */
|
||||
#endif
|
||||
#if LJ_TARGET_MIPS64 || (LJ_TARGET_ARM64 && LJ_BE)
|
||||
#if LJ_TARGET_RISCV64
|
||||
if (isfp && d->size == sizeof(float))
|
||||
((uint32_t *)dp)[1] = 0xffffffffu; /* Float NaN boxing */
|
||||
#endif
|
||||
#if LJ_TARGET_MIPS64 || (LJ_TARGET_ARM64 && LJ_BE) || LJ_TARGET_RISCV64
|
||||
if ((ctype_isinteger_or_bool(d->info) || ctype_isenum(d->info)
|
||||
#if LJ_TARGET_MIPS64
|
||||
|| (isfp && nsp == 0)
|
||||
|
||||
@ -129,6 +129,23 @@ typedef union FPRArg {
|
||||
struct { LJ_ENDIAN_LOHI(float f; , float g;) };
|
||||
} FPRArg;
|
||||
|
||||
#elif LJ_TARGET_RISCV64
|
||||
|
||||
#define CCALL_NARG_GPR 8
|
||||
#define CCALL_NARG_FPR 8
|
||||
#define CCALL_NRET_GPR 2
|
||||
#define CCALL_NRET_FPR 2
|
||||
#define CCALL_SPS_EXTRA 3
|
||||
#define CCALL_SPS_FREE 1
|
||||
|
||||
typedef intptr_t GPRArg;
|
||||
typedef union FPRArg {
|
||||
double d;
|
||||
uint64_t u;
|
||||
struct { LJ_ENDIAN_LOHI(float f; , float g;) };
|
||||
struct { LJ_ENDIAN_LOHI(uint32_t lo; , uint32_t hi;) };
|
||||
} FPRArg;
|
||||
|
||||
#else
|
||||
#error "Missing calling convention definitions for this architecture"
|
||||
#endif
|
||||
|
||||
@ -82,6 +82,10 @@ static MSize CALLBACK_OFS2SLOT(MSize ofs)
|
||||
|
||||
#define CALLBACK_MCODE_HEAD 52
|
||||
|
||||
#elif LJ_TARGET_RISCV64
|
||||
|
||||
#define CALLBACK_MCODE_HEAD 68
|
||||
|
||||
#else
|
||||
|
||||
/* Missing support for this architecture. */
|
||||
@ -260,6 +264,39 @@ static void *callback_mcode_init(global_State *g, uint32_t *page)
|
||||
}
|
||||
return p;
|
||||
}
|
||||
#elif LJ_TARGET_RISCV64
|
||||
static void *callback_mcode_init(global_State *g, uint32_t *page)
|
||||
{
|
||||
uint32_t *p = page;
|
||||
uintptr_t target = (uintptr_t)(void *)lj_vm_ffi_callback;
|
||||
uintptr_t ug = (uintptr_t)(void *)g;
|
||||
uintptr_t target_hi = (target >> 32), target_lo = target & 0xffffffffULL;
|
||||
uintptr_t ug_hi = (ug >> 32), ug_lo = ug & 0xffffffffULL;
|
||||
MSize slot;
|
||||
*p++ = RISCVI_LUI | RISCVF_D(RID_X6) | RISCVF_IMMU(RISCVF_HI(target_hi));
|
||||
*p++ = RISCVI_LUI | RISCVF_D(RID_X7) | RISCVF_IMMU(RISCVF_HI(ug_hi));
|
||||
*p++ = RISCVI_ADDI | RISCVF_D(RID_X6) | RISCVF_S1(RID_X6) | RISCVF_IMMI(RISCVF_LO(target_hi));
|
||||
*p++ = RISCVI_ADDI | RISCVF_D(RID_X7) | RISCVF_S1(RID_X7) | RISCVF_IMMI(RISCVF_LO(ug_hi));
|
||||
*p++ = RISCVI_SLLI | RISCVF_D(RID_X6) | RISCVF_S1(RID_X6) | RISCVF_SHAMT(11);
|
||||
*p++ = RISCVI_SLLI | RISCVF_D(RID_X7) | RISCVF_S1(RID_X7) | RISCVF_SHAMT(11);
|
||||
*p++ = RISCVI_ADDI | RISCVF_D(RID_X6) | RISCVF_S1(RID_X6) | RISCVF_IMMI(target_lo >> 21);
|
||||
*p++ = RISCVI_ADDI | RISCVF_D(RID_X7) | RISCVF_S1(RID_X7) | RISCVF_IMMI(ug_lo >> 21);
|
||||
*p++ = RISCVI_SLLI | RISCVF_D(RID_X6) | RISCVF_S1(RID_X6) | RISCVF_SHAMT(11);
|
||||
*p++ = RISCVI_SLLI | RISCVF_D(RID_X7) | RISCVF_S1(RID_X7) | RISCVF_SHAMT(11);
|
||||
*p++ = RISCVI_ADDI | RISCVF_D(RID_X6) | RISCVF_S1(RID_X6) | RISCVF_IMMI((target_lo >> 10) & 0x7ff);
|
||||
*p++ = RISCVI_ADDI | RISCVF_D(RID_X7) | RISCVF_S1(RID_X7) | RISCVF_IMMI((ug_lo >> 10) & 0x7ff);
|
||||
*p++ = RISCVI_SLLI | RISCVF_D(RID_X6) | RISCVF_S1(RID_X6) | RISCVF_SHAMT(10);
|
||||
*p++ = RISCVI_SLLI | RISCVF_D(RID_X7) | RISCVF_S1(RID_X7) | RISCVF_SHAMT(10);
|
||||
*p++ = RISCVI_ADDI | RISCVF_D(RID_X6) | RISCVF_S1(RID_X6) | RISCVF_IMMI(target_lo & 0x3ff);
|
||||
*p++ = RISCVI_ADDI | RISCVF_D(RID_X7) | RISCVF_S1(RID_X7) | RISCVF_IMMI(ug_lo & 0x3ff);
|
||||
*p++ = RISCVI_JALR | RISCVF_D(RID_X0) | RISCVF_S1(RID_X6) | RISCVF_IMMJ(0);
|
||||
for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
|
||||
*p++ = RISCVI_LUI | RISCVF_D(RID_X5) | RISCVF_IMMU(slot);
|
||||
*p = RISCVI_JAL | RISCVF_IMMJ(((char *)page-(char *)p));
|
||||
p++;
|
||||
}
|
||||
return p;
|
||||
}
|
||||
#else
|
||||
/* Missing support for this architecture. */
|
||||
#define callback_mcode_init(g, p) (p)
|
||||
@ -549,6 +586,31 @@ void lj_ccallback_mcode_free(CTState *cts)
|
||||
}
|
||||
#endif
|
||||
|
||||
#define CALLBACK_HANDLE_RET \
|
||||
if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
|
||||
((float *)dp)[1] = *(float *)dp;
|
||||
|
||||
#elif LJ_TARGET_RISCV64
|
||||
|
||||
#define CALLBACK_HANDLE_REGARG \
|
||||
if (isfp) { \
|
||||
if (nfpr + n <= CCALL_NARG_FPR) { \
|
||||
sp = &cts->cb.fpr[nfpr]; \
|
||||
nfpr += n; \
|
||||
goto done; \
|
||||
} else if (ngpr + n <= maxgpr) { \
|
||||
sp = &cts->cb.gpr[ngpr]; \
|
||||
ngpr += n; \
|
||||
goto done; \
|
||||
} \
|
||||
} else { \
|
||||
if (ngpr + n <= maxgpr) { \
|
||||
sp = &cts->cb.gpr[ngpr]; \
|
||||
ngpr += n; \
|
||||
goto done; \
|
||||
} \
|
||||
}
|
||||
|
||||
#define CALLBACK_HANDLE_RET \
|
||||
if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
|
||||
((float *)dp)[1] = *(float *)dp;
|
||||
@ -699,7 +761,7 @@ static void callback_conv_result(CTState *cts, lua_State *L, TValue *o)
|
||||
*(int32_t *)dp = ctr->size == 1 ? (int32_t)*(int8_t *)dp :
|
||||
(int32_t)*(int16_t *)dp;
|
||||
}
|
||||
#if LJ_TARGET_MIPS64 || (LJ_TARGET_ARM64 && LJ_BE)
|
||||
#if LJ_TARGET_MIPS64 || (LJ_TARGET_ARM64 && LJ_BE) || LJ_TARGET_RISCV64
|
||||
/* Always sign-extend results to 64 bits. Even a soft-fp 'float'. */
|
||||
if (ctr->size <= 4 &&
|
||||
(LJ_ABI_SOFTFP || ctype_isinteger_or_bool(ctr->info)))
|
||||
|
||||
@ -811,14 +811,29 @@ static void build_subroutines(BuildCtx *ctx)
|
||||
| mv BASE, TMP2 // Restore caller BASE.
|
||||
| ld LFUNC:TMP1, FRAME_FUNC(TMP2)
|
||||
| ld PC, -24(RB) // Restore PC from [cont|PC].
|
||||
|.if FFI
|
||||
| sltiu TMP3, TMP0, 2
|
||||
|.endif
|
||||
| cleartp LFUNC:TMP1
|
||||
| add TMP2, RA, RD
|
||||
| ld TMP1, LFUNC:TMP1->pc
|
||||
| sd TISNIL, -8(TMP2) // Ensure one valid arg.
|
||||
|.if FFI
|
||||
| bnez TMP3, >1
|
||||
|.endif
|
||||
| // BASE = base, RA = resultptr, RB = meta base
|
||||
| ld KBASE, PC2PROTO(k)(TMP1)
|
||||
| jr TMP0 // Jump to continuation.
|
||||
|
|
||||
|.if FFI
|
||||
|1:
|
||||
| addi TMP1, RB, -32
|
||||
| bxnez TMP0, ->cont_ffi_callback // cont = 1: return from FFI callback.
|
||||
| // cont = 0: tailcall from C function.
|
||||
| sub RC, TMP1, BASE
|
||||
| j ->vm_call_tail
|
||||
|.endif
|
||||
|
|
||||
|->cont_cat: // RA = resultptr, RB = meta base
|
||||
| lw INS, -4(PC)
|
||||
| addi CARG2, RB, -32
|
||||
@ -1018,6 +1033,18 @@ static void build_subroutines(BuildCtx *ctx)
|
||||
| // Returns 0/1 or TValue * (metamethod).
|
||||
| j <3
|
||||
|
|
||||
|->vmeta_equal_cd:
|
||||
|.if FFI
|
||||
| addi PC, PC, -4
|
||||
| mv CARG1, L
|
||||
| mv CARG2, INS
|
||||
| sd BASE, L->base
|
||||
| sd PC, SAVE_PC(sp)
|
||||
| call_intern vmeta_equal_cd, lj_meta_equal_cd // (lua_State *L, BCIns op)
|
||||
| // Returns 0/1 or TValue * (metamethod).
|
||||
| j <3
|
||||
|.endif
|
||||
|
|
||||
|->vmeta_istype:
|
||||
| addi PC, PC, -4
|
||||
| sd BASE, L->base
|
||||
@ -2278,6 +2305,130 @@ static void build_subroutines(BuildCtx *ctx)
|
||||
|.endif
|
||||
|
|
||||
|//-----------------------------------------------------------------------
|
||||
|//-- FFI helper functions -----------------------------------------------
|
||||
|//-----------------------------------------------------------------------
|
||||
|
|
||||
|// Handler for callback functions. Callback slot number in x5, g in x7.
|
||||
|->vm_ffi_callback:
|
||||
|.if FFI
|
||||
|.type CTSTATE, CTState, PC
|
||||
| saveregs
|
||||
| ld CTSTATE, GL:x7->ctype_state
|
||||
| mv GL, x7
|
||||
| addxi DISPATCH, x7, GG_G2DISP
|
||||
| srli x5, x5, 12
|
||||
| sw x5, CTSTATE->cb.slot
|
||||
| sd CARG1, CTSTATE->cb.gpr[0]
|
||||
| fsd FARG1, CTSTATE->cb.fpr[0]
|
||||
| sd CARG2, CTSTATE->cb.gpr[1]
|
||||
| fsd FARG2, CTSTATE->cb.fpr[1]
|
||||
| sd CARG3, CTSTATE->cb.gpr[2]
|
||||
| fsd FARG3, CTSTATE->cb.fpr[2]
|
||||
| sd CARG4, CTSTATE->cb.gpr[3]
|
||||
| fsd FARG4, CTSTATE->cb.fpr[3]
|
||||
| sd CARG5, CTSTATE->cb.gpr[4]
|
||||
| fsd FARG5, CTSTATE->cb.fpr[4]
|
||||
| sd CARG6, CTSTATE->cb.gpr[5]
|
||||
| fsd FARG6, CTSTATE->cb.fpr[5]
|
||||
| sd CARG7, CTSTATE->cb.gpr[6]
|
||||
| fsd FARG7, CTSTATE->cb.fpr[6]
|
||||
| sd CARG8, CTSTATE->cb.gpr[7]
|
||||
| fsd FARG8, CTSTATE->cb.fpr[7]
|
||||
| addi TMP0, sp, CFRAME_SPACE
|
||||
| sd TMP0, CTSTATE->cb.stack
|
||||
| sd x0, SAVE_PC(sp) // Any value outside of bytecode is ok.
|
||||
| mv CARG1, CTSTATE
|
||||
| mv CARG2, sp
|
||||
| call_intern vm_ffi_callback, lj_ccallback_enter // (CTState *cts, void *cf)
|
||||
| // Returns lua_State *.
|
||||
| ld BASE, L:CRET1->base
|
||||
| ld RC, L:CRET1->top
|
||||
| mv L, CRET1
|
||||
| lui TMP3, 0x43380 // TOBIT = Hiword of 2^52 + 2^51 (double).
|
||||
| ld LFUNC:RB, FRAME_FUNC(BASE)
|
||||
| li TISNIL, LJ_TNIL
|
||||
| li TISNUM, LJ_TISNUM
|
||||
| slli TMP3, TMP3, 32
|
||||
| li_vmstate INTERP
|
||||
| subw RC, RC, BASE
|
||||
| cleartp LFUNC:RB
|
||||
| st_vmstate
|
||||
| fmv.d.x TOBIT, TMP3
|
||||
| ins_callt
|
||||
|.endif
|
||||
|
|
||||
|->cont_ffi_callback: // Return from FFI callback.
|
||||
|.if FFI
|
||||
| ld CTSTATE, GL->ctype_state
|
||||
| sd BASE, L->base
|
||||
| sd RB, L->top
|
||||
| sd L, CTSTATE->L
|
||||
| mv CARG1, CTSTATE
|
||||
| mv CARG2, RA
|
||||
| // (CTState *cts, TValue *o)
|
||||
| call_intern cont_ffi_callback, lj_ccallback_leave
|
||||
| fld FRET1, CTSTATE->cb.fpr[0]
|
||||
| ld CRET1, CTSTATE->cb.gpr[0]
|
||||
| fld FRET2, CTSTATE->cb.fpr[1]
|
||||
| ld CRET2, CTSTATE->cb.gpr[1]
|
||||
| j ->vm_leave_unw
|
||||
|.endif
|
||||
|
|
||||
|->vm_ffi_call: // Call C function via FFI.
|
||||
| // Caveat: needs special frame unwinding, see below.
|
||||
|.if FFI
|
||||
| .type CCSTATE, CCallState, CARG1
|
||||
| lw TMP1, CCSTATE->spadj
|
||||
| lbu CARG2, CCSTATE->nsp
|
||||
| mv TMP2, sp
|
||||
| sub sp, sp, TMP1
|
||||
| sd ra, -8(TMP2)
|
||||
| sd x18, -16(TMP2)
|
||||
| sd CCSTATE, -24(TMP2)
|
||||
| mv x18, TMP2
|
||||
| addi TMP1, CCSTATE, offsetof(CCallState, stack)
|
||||
| mv TMP2, sp
|
||||
| add TMP3, TMP1, CARG2
|
||||
| beqz CARG2, >2
|
||||
|1:
|
||||
| ld TMP0, 0(TMP1)
|
||||
| addi TMP1, TMP1, 8
|
||||
| sd TMP0, 0(TMP2)
|
||||
| addi TMP2, TMP2, 8
|
||||
| bltu TMP1, TMP3, <1
|
||||
|2:
|
||||
| fld FARG1, CCSTATE->fpr[0]
|
||||
| fld FARG2, CCSTATE->fpr[1]
|
||||
| fld FARG3, CCSTATE->fpr[2]
|
||||
| fld FARG4, CCSTATE->fpr[3]
|
||||
| fld FARG5, CCSTATE->fpr[4]
|
||||
| fld FARG6, CCSTATE->fpr[5]
|
||||
| fld FARG7, CCSTATE->fpr[6]
|
||||
| fld FARG8, CCSTATE->fpr[7]
|
||||
| ld CFUNCADDR, CCSTATE->func
|
||||
| ld CARG2, CCSTATE->gpr[1]
|
||||
| ld CARG3, CCSTATE->gpr[2]
|
||||
| ld CARG4, CCSTATE->gpr[3]
|
||||
| ld CARG5, CCSTATE->gpr[4]
|
||||
| ld CARG6, CCSTATE->gpr[5]
|
||||
| ld CARG7, CCSTATE->gpr[6]
|
||||
| ld CARG8, CCSTATE->gpr[7]
|
||||
| ld CARG1, CCSTATE->gpr[0] // Do this last, since CCSTATE is CARG1.
|
||||
| jalr CFUNCADDR
|
||||
| ld CCSTATE:TMP1, -24(x18)
|
||||
| ld TMP0, -16(x18)
|
||||
| ld ra, -8(x18)
|
||||
| sd CRET1, CCSTATE:TMP1->gpr[0]
|
||||
| sd CRET2, CCSTATE:TMP1->gpr[1]
|
||||
| fsd FRET1, CCSTATE:TMP1->fpr[0]
|
||||
| fsd FRET2, CCSTATE:TMP1->fpr[1]
|
||||
| mv sp, x18
|
||||
| mv x18, TMP0
|
||||
| ret
|
||||
|.endif
|
||||
|// Note: vm_ffi_call must be the last function in this object file!
|
||||
|
|
||||
|//-----------------------------------------------------------------------
|
||||
}
|
||||
|
||||
/* Generate the code for a single instruction. */
|
||||
@ -2402,6 +2553,15 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
|
||||
| beqz TMP0, ->BC_ISNEN_Z
|
||||
}
|
||||
|// Either or both types are not numbers.
|
||||
|.if FFI
|
||||
| // Check if RA or RD is a cdata.
|
||||
| xori TMP0, CARG3, LJ_TCDATA
|
||||
| xori TMP1, CARG4, LJ_TCDATA
|
||||
| seqz TMP0, TMP0
|
||||
| seqz TMP1, TMP1
|
||||
| or TMP0, TMP0, TMP1
|
||||
| bxnez TMP0, ->vmeta_equal_cd
|
||||
|.endif
|
||||
| lui TMP3, (-(BCBIAS_J*4 >> 12)) & 0xfffff // -BCBIAS_J*4
|
||||
| decode_BC4b TMP2
|
||||
| addw TMP2, TMP2, TMP3 // (jump-0x8000)<<2
|
||||
@ -2454,10 +2614,17 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
|
||||
| sub RD, KBASE, RD
|
||||
| lhu TMP2, -4+OFS_RD(PC)
|
||||
| ld CARG2, -8(RD) // KBASE-8-str_const*8
|
||||
|.if FFI
|
||||
| gettp CARG3, CARG1
|
||||
| li TMP1, LJ_TCDATA
|
||||
|.endif
|
||||
| li TMP0, LJ_TSTR
|
||||
| decode_BC4b TMP2
|
||||
| settp CARG2, TMP0
|
||||
| lui TMP3, (-(BCBIAS_J*4 >> 12)) & 0xfffff // -BCBIAS_J*4
|
||||
|.if FFI
|
||||
| bxeq CARG3, TMP1, ->vmeta_equal_cd
|
||||
|.endif
|
||||
| xor TMP0, CARG1, CARG2 // TMP2=0: A==D; TMP2!=0: A!=D
|
||||
| addw TMP2, TMP2, TMP3
|
||||
if (vk) {
|
||||
@ -2512,7 +2679,11 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
|
||||
|
|
||||
|4: // RA is not an integer.
|
||||
| addw TMP2, TMP2, TMP3
|
||||
|.if FFI
|
||||
| bgeu CARG3, TISNUM, >7
|
||||
|.else
|
||||
| bgeu CARG3, TISNUM, <2
|
||||
|.endif
|
||||
| fmv.d.x FTMP0, CARG1
|
||||
| fmv.d.x FTMP2, CARG2
|
||||
| bne CARG4, TISNUM, >5
|
||||
@ -2525,11 +2696,26 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
|
||||
| j <1
|
||||
|
|
||||
|6: // RA is an integer, RD is a number.
|
||||
|.if FFI
|
||||
| bgeu CARG4, TISNUM, >8
|
||||
|.else
|
||||
| bgeu CARG4, TISNUM, <2
|
||||
|.endif
|
||||
| fcvt.d.w FTMP0, CARG1
|
||||
| fmv.d.x FTMP2, CARG2
|
||||
| j <5
|
||||
|
|
||||
|.if FFI
|
||||
|7: // RA not int, not number
|
||||
| li TMP0, LJ_TCDATA
|
||||
| bne CARG3, TMP0, <2
|
||||
| j ->vmeta_equal_cd
|
||||
|
|
||||
|8: // RD not int, not number
|
||||
| li TMP0, LJ_TCDATA
|
||||
| bne CARG4, TMP0, <2
|
||||
| j ->vmeta_equal_cd
|
||||
|.endif
|
||||
break;
|
||||
|
||||
case BC_ISEQP: case BC_ISNEP:
|
||||
@ -2543,6 +2729,10 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
|
||||
| gettp TMP1, TMP1
|
||||
| addi PC, PC, 4
|
||||
| xor TMP0, TMP1, TMP0 // TMP0=0 A=D; TMP0!=0 A!=D
|
||||
|.if FFI
|
||||
| li TMP3, LJ_TCDATA
|
||||
| bxeq TMP1, TMP3, ->vmeta_equal_cd
|
||||
|.endif
|
||||
| decode_BC4b TMP2
|
||||
| lui TMP3, (-(BCBIAS_J*4 >> 12)) & 0xfffff // -BCBIAS_J*4
|
||||
| addw TMP2, TMP2, TMP3 // TMP2=(jump-0x8000)<<2
|
||||
@ -2942,6 +3132,16 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
|
||||
| ins_next
|
||||
break;
|
||||
case BC_KCDATA:
|
||||
|.if FFI
|
||||
| // RA = dst*8, RD = cdata_const*8 (~)
|
||||
| sub TMP1, KBASE, RD
|
||||
| ld TMP0, -8(TMP1) // KBASE-8-cdata_const*8
|
||||
| li TMP2, LJ_TCDATA
|
||||
| add RA, BASE, RA
|
||||
| settp TMP0, TMP2
|
||||
| sd TMP0, 0(RA)
|
||||
| ins_next
|
||||
|.endif
|
||||
break;
|
||||
case BC_KSHORT:
|
||||
| // RA = dst*8, RD = int16_literal*8
|
||||
|
||||
Loading…
Reference in New Issue
Block a user