q98/LuaJIT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

New GC

Browse Source
This commit is contained in:
Albert 2023-07-05 21:27:35 -04:00
parent 29b0b282f5
commit e403e4a1dc
43 changed files with 2894 additions and 553 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/Makefile
View File

@ -47,7 +47,7 @@ CCOPT= -O2 -fomit-frame-pointer
# the binaries to a different machine you could also use: -march=native
#
CCOPT_x86= -march=i686 -msse -msse2 -mfpmath=sse
CCOPT_x64=
CCOPT_x64= -mbmi -mbmi2 -mavx -mavx2
CCOPT_arm=
CCOPT_arm64=
CCOPT_ppc=
@ -511,7 +511,7 @@ LJCORE_O= lj_assert.o lj_gc.o lj_err.o lj_char.o lj_bc.o lj_obj.o lj_buf.o \
lj_mcode.o lj_snap.o lj_record.o lj_crecord.o lj_ffrecord.o \
lj_asm.o lj_trace.o lj_gdbjit.o \
lj_ctype.o lj_cdata.o lj_cconv.o lj_ccall.o lj_ccallback.o \
lj_carith.o lj_clib.o lj_cparse.o \
lj_carith.o lj_clib.o lj_cparse.o lj_arena.o \
lj_lib.o lj_alloc.o lib_aux.o \
$(LJLIB_O) lib_init.o

32
src/lib_aux.c
View File

@ -399,3 +399,35 @@ LUA_API lua_State *lua_newstate(lua_Alloc f, void *ud)
#endif
LUA_API lua_State *luaJIT_newstate(lua_Alloc f, void *ud,
luaJIT_allocpages allocp,
luaJIT_freepages freep,
luaJIT_reallochuge realloch,
void *page_ud)
{
lua_State *L;
#ifdef LUAJIT_USE_SYSMALLOC
if (!f) {
f = mem_alloc;
ud = NULL;
}
#else
if (f == NULL) {
f = LJ_ALLOCF_INTERNAL;
ud = NULL;
}
#endif
L = lj_newstate(f, ud, allocp, freep, realloch, page_ud);
if (L) {
G(L)->panic = panic;
#ifndef LUAJIT_DISABLE_VMEVENT
luaL_findtable(L, LUA_REGISTRYINDEX, LJ_VMEVENTS_REGKEY, LJ_VMEVENTS_HSIZE);
lua_pushcfunction(L, error_finalizer);
lua_rawseti(L, -2, VMEVENT_HASH(LJ_VMEVENT_ERRFIN));
G(L)->vmevmask = VMEVENT_MASK(LJ_VMEVENT_ERRFIN);
L->top--;
#endif
}
return L;
}

2
src/lib_buffer.c
View File

@ -317,6 +317,8 @@ LJLIB_CF(buffer_new)
ud->udtype = UDTYPE_BUFFER;
/* NOBARRIER: The GCudata is new (marked white). */
setgcref(ud->metatable, obj2gco(env));
if (lj_meta_fastg(G(L), tabref(ud->metatable), MM_gc))
lj_mem_registergc_udata(L, ud);
setudataV(L, L->top++, ud);
sbx = (SBufExt *)uddata(ud);
lj_bufx_init(L, sbx);

2
src/lib_debug.c
View File

@ -232,7 +232,7 @@ LJLIB_CF(debug_upvalueid)
if ((uint32_t)n >= fn->l.nupvalues)
lj_err_arg(L, 2, LJ_ERR_IDXRNG);
lua_pushlightuserdata(L, isluafunc(fn) ? (void *)gcref(fn->l.uvptr[n]) :
(void *)&fn->c.upvalue[n]);
(void *)&fn->c.data->upvalue[n]);
return 1;
}

4
src/lib_ffi.c
View File

@ -518,7 +518,7 @@ LJLIB_CF(ffi_new) LJLIB_REC(.)
/* Add to finalizer table, if still enabled. */
copyTV(L, lj_tab_set(L, t, o-1), tv);
lj_gc_anybarriert(L, t);
cd->marked |= LJ_GC_CDATA_FIN;
cd->gcflags |= LJ_GC_CDATA_FIN;
}
}
}
@ -573,7 +573,7 @@ LJLIB_CF(ffi_typeinfo)
setintV(lj_tab_setstr(L, t, lj_str_newlit(L, "sib")), (int32_t)ct->sib);
if (gcref(ct->name)) {
GCstr *s = gco2str(gcref(ct->name));
if (isdead(G(L), obj2gco(s))) flipwhite(obj2gco(s));
maybe_resurrect_str(G(L), s);
setstrV(L, lj_tab_setstr(L, t, lj_str_newlit(L, "name")), s);
}
lj_gc_check(L);

9
src/lib_io.c
View File

@ -25,6 +25,7 @@
#include "lj_strfmt.h"
#include "lj_ff.h"
#include "lj_lib.h"
#include "lj_meta.h"
/* Userdata payload for I/O file. */
typedef struct IOFileUD {
@ -75,6 +76,8 @@ static IOFileUD *io_file_new(lua_State *L)
ud->udtype = UDTYPE_IO_FILE;
/* NOBARRIER: The GCudata is new (marked white). */
setgcrefr(ud->metatable, curr_func(L)->c.env);
if (lj_meta_fastg(G(L), tabref(ud->metatable), MM_gc))
lj_mem_registergc_udata(L, ud);
iof->fp = NULL;
iof->type = IOFILE_TYPE_FILE;
return iof;
@ -249,14 +252,14 @@ static int io_file_write(lua_State *L, IOFileUD *iof, int start)
static int io_file_iter(lua_State *L)
{
GCfunc *fn = curr_func(L);
IOFileUD *iof = uddata(udataV(&fn->c.upvalue[0]));
IOFileUD *iof = uddata(udataV(&fn->c.data->upvalue[0]));
int n = fn->c.nupvalues - 1;
if (iof->fp == NULL)
lj_err_caller(L, LJ_ERR_IOCLFL);
L->top = L->base;
if (n) { /* Copy upvalues with options to stack. */
lj_state_checkstack(L, (MSize)n);
memcpy(L->top, &fn->c.upvalue[1], n*sizeof(TValue));
memcpy(L->top, &fn->c.data->upvalue[1], n * sizeof(TValue));
L->top += n;
}
n = io_file_read(L, iof, 0);
@ -531,6 +534,8 @@ static GCobj *io_std_new(lua_State *L, FILE *fp, const char *name)
ud->udtype = UDTYPE_IO_FILE;
/* NOBARRIER: The GCudata is new (marked white). */
setgcref(ud->metatable, gcV(L->top-3));
if (lj_meta_fastg(G(L), tabref(ud->metatable), MM_gc))
lj_mem_registergc_udata(L, ud);
iof->fp = fp;
iof->type = IOFILE_TYPE_STDF;
lua_setfield(L, -2, name);

2
src/lib_jit.c
View File

@ -220,7 +220,7 @@ LJLIB_CF(jit_util_funcinfo)
if (!iscfunc(fn))
setintfield(L, t, "ffid", fn->c.ffid);
setintptrV(lj_tab_setstr(L, t, lj_str_newlit(L, "addr")),
(intptr_t)(void *)fn->c.f);
(intptr_t)(void *)fn->c.data->f);
setintfield(L, t, "upvalues", fn->c.nupvalues);
}
return 1;

8
src/lib_table.c
View File

@ -105,6 +105,14 @@ LJLIB_CF(table_insert) LJLIB_REC(.)
return 0;
}
LJLIB_CF(table_newgc)
{
GCtab *t = lj_tab_newgc(L, 0, 0);
settabV(L, L->top++, t);
return 1;
}
LJLIB_LUA(table_remove) /*
function(t, pos)
CHECK_tab(t)

34
src/lj_api.c
View File

@ -56,7 +56,7 @@ static TValue *index2adr(lua_State *L, int idx)
return o;
} else {
idx = LUA_GLOBALSINDEX - idx;
return idx <= fn->c.nupvalues ? &fn->c.upvalue[idx-1] : niltv(L);
return idx <= fn->c.nupvalues ? &fn->c.data->upvalue[idx - 1] : niltv(L);
}
}
}
@ -602,7 +602,7 @@ LUA_API lua_CFunction lua_tocfunction(lua_State *L, int idx)
if (tvisfunc(o)) {
BCOp op = bc_op(*mref(funcV(o)->c.pc, BCIns));
if (op == BC_FUNCC || op == BC_FUNCCW)
return funcV(o)->c.f;
return funcV(o)->c.data->f;
}
return NULL;
}
@ -697,12 +697,12 @@ LUA_API void lua_pushcclosure(lua_State *L, lua_CFunction f, int n)
lj_gc_check(L);
lj_checkapi_slot(n);
fn = lj_func_newC(L, (MSize)n, getcurrenv(L));
fn->c.f = f;
fn->c.data->f = f;
L->top -= n;
while (n--)
copyTV(L, &fn->c.upvalue[n], L->top+n);
copyTV(L, &fn->c.data->upvalue[n], L->top + n);
setfuncV(L, L->top, fn);
lj_assertL(iswhite(obj2gco(fn)), "new GC object is not white");
lj_assertL(iswhite(G(L), obj2gco(fn)), "new GC object is not white");
incr_top(L);
}
@ -728,6 +728,15 @@ LUA_API void lua_createtable(lua_State *L, int narray, int nrec)
incr_top(L);
}
LUA_API void luaJIT_createtable(lua_State *L, int narray, int nrec)
{
lj_gc_check(L);
settabV(L, L->top,
lj_tab_newgc(L, (uint32_t)(narray > 0 ? narray + 1 : 0),
hsize2hbits(nrec)));
incr_top(L);
}
LUALIB_API int luaL_newmetatable(lua_State *L, const char *tname)
{
GCtab *regt = tabV(registry(L));
@ -927,7 +936,7 @@ LUA_API void *lua_upvalueid(lua_State *L, int idx, int n)
n--;
lj_checkapi((uint32_t)n < fn->l.nupvalues, "bad upvalue %d", n);
return isluafunc(fn) ? (void *)gcref(fn->l.uvptr[n]) :
(void *)&fn->c.upvalue[n];
(void *)&fn->c.data->upvalue[n];
}
LUA_API void lua_upvaluejoin(lua_State *L, int idx1, int n1, int idx2, int n2)
@ -1046,8 +1055,11 @@ LUA_API int lua_setmetatable(lua_State *L, int idx)
lj_gc_objbarriert(L, tabV(o), mt);
} else if (tvisudata(o)) {
setgcref(udataV(o)->metatable, obj2gco(mt));
if (mt)
if (mt) {
if (lj_meta_fastg(g, mt, MM_gc))
lj_mem_registergc_udata(L, udataV(o));
lj_gc_objbarrier(L, udataV(o), mt);
}
} else {
/* Flush cache, since traces specialize to basemt. But not during __gc. */
if (lj_trace_flushall(L))
@ -1156,7 +1168,7 @@ static TValue *cpcall(lua_State *L, lua_CFunction func, void *ud)
{
GCfunc *fn = lj_func_newC(L, 0, getcurrenv(L));
TValue *top = L->top;
fn->c.f = func;
fn->c.data->f = func;
setfuncV(L, top++, fn);
if (LJ_FR2) setnilV(top++);
#if LJ_64
@ -1267,7 +1279,7 @@ LUA_API int lua_gc(lua_State *L, int what, int data)
g->gc.threshold = data == -1 ? (g->gc.total/100)*g->gc.pause : g->gc.total;
break;
case LUA_GCCOLLECT:
lj_gc_fullgc(L);
lj_gc_fullgc(L, data);
break;
case LUA_GCCOUNT:
res = (int)(g->gc.total >> 10);
@ -1316,3 +1328,7 @@ LUA_API void lua_setallocf(lua_State *L, lua_Alloc f, void *ud)
g->allocf = f;
}
LUA_API size_t luaJIT_getpagesize()
{
return ARENA_SIZE;
}

397
src/lj_arena.c Normal file
View File

@ -0,0 +1,397 @@
/* To get the mremap prototype. Must be defined before any system includes. */
#if defined(__linux__) && !defined(_GNU_SOURCE)
#define _GNU_SOURCE
#endif
#include "lj_def.h"
#include "lj_arena.h"
#include "lj_gc.h"
#include "lj_obj.h"
#include <stdlib.h>
#include <string.h>
/* unmap() any misaligned huge pages */
#define LJ_ALLOC_FIXUP 1
#if LJ_TARGET_LINUX
#define LJ_ALLOC_MREMAP 1
#endif
/* COMMITLESS trades increased memory usage for many fewer syscalls. Very recommended */
#define COMMITLESS
#if LJ_TARGET_POSIX
#include <sys/mman.h>
#include <pthread.h>
#define RESERVE_PAGES(h, n) \
mmap(h, n, PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)
#define UNRESERVE_PAGES(p, n) munmap(p, n)
#define COMMIT_PAGES(p, n)
#ifdef COMMITLESS
#define UNCOMMIT_PAGES(p, n)
#else
#define UNCOMMIT_PAGES(p, n) madvise(p, n, MADV_DONTNEED)
#endif
#define RESERVE_AND_COMMIT_PAGES(n) RESERVE_PAGES(0, n)
#elif LJ_TARGET_WINDOWS
#define WIN32_LEAN_AND_MEAN
#include <Windows.h>
#define UNRESERVE_PAGES(p, n) VirtualFree(p, 0, MEM_RELEASE)
#define RESERVE_AND_COMMIT_PAGES(n) \
LJ_WIN_VALLOC(0, n, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)
#ifdef COMMITLESS
#define RESERVE_PAGES(h, n) LJ_WIN_VALLOC(h, n, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE)
#define COMMIT_PAGES(p, n)
#define UNCOMMIT_PAGES(p, n)
#else
#define RESERVE_PAGES(h, n) LJ_WIN_VALLOC(h, n, MEM_RESERVE, PAGE_READWRITE)
#define COMMIT_PAGES(p, n) LJ_WIN_VALLOC(p, n, MEM_COMMIT, PAGE_READWRITE)
#define UNCOMMIT_PAGES(p, n) VirtualFree(p, n, MEM_DECOMMIT)
#endif
#else
#error "No page allocation OS support!"
#endif
#define RESERVE_SIZE 8 * 1024 * 1024
/* output >= input */
#define PTRALIGN_UP(x, y) \
(void *)(((uintptr_t)(x) + ((y)-1)) & ~(uintptr_t)((y)-1))
/* output <= input */
#define PTRALIGN_DOWN(x, y) (void *)(((uintptr_t)(x)) & ~(uintptr_t)((y)-1))
typedef struct arena_alloc {
void **freelist;
uint32_t freelist_sz;
uint32_t freelist_at;
void **ptrs;
uint32_t ptrs_sz;
uint32_t ptrs_at;
uint8_t *at;
uint8_t *end;
void *hint;
global_State *g;
} arena_alloc;
int lj_arena_newchunk(arena_alloc *arenas)
{
if (arenas->ptrs_at == arenas->ptrs_sz) {
size_t oldsz = sizeof(void *) * arenas->ptrs_sz;
arenas->ptrs_sz *= 2;
void *newp = arenas->g->allocf(arenas->g->allocd, arenas->ptrs,
oldsz, sizeof(void *) * arenas->ptrs_sz);
if (!newp) {
return 0;
}
arenas->ptrs = newp;
}
void *p = RESERVE_PAGES(arenas->hint, RESERVE_SIZE);
if (!p) {
p = RESERVE_PAGES(NULL, RESERVE_SIZE);
if (!p) {
return 0;
}
}
arenas->ptrs[arenas->ptrs_at++] = p;
arenas->at = (uint8_t *)PTRALIGN_UP(p, ARENA_SIZE);
arenas->end = (uint8_t *)PTRALIGN_DOWN((uint8_t*)p + RESERVE_SIZE, ARENA_SIZE);
arenas->hint = PTRALIGN_UP((uint8_t *)p + RESERVE_SIZE, ARENA_SIZE);
return 1;
}
static void lj_arena_api_freepages(void *ud, void **pages, unsigned n)
{
arena_alloc *arenas = (arena_alloc*)ud;
if (LJ_UNLIKELY(!pages)) {
for (uint32_t i = 0; i < arenas->ptrs_at; i++)
UNRESERVE_PAGES(arenas->ptrs[i], RESERVE_SIZE);
arenas->g->allocf(arenas->g->allocd, arenas->freelist,
arenas->freelist_sz * sizeof(void *), 0);
arenas->g->allocf(arenas->g->allocd, arenas->ptrs,
arenas->ptrs_sz * sizeof(void*), 0);
/* This deletes the context so must be last */
arenas->g->allocf(arenas->g->allocd, ud, sizeof(arena_alloc), 0);
} else {
if (arenas->freelist_at + n > arenas->freelist_sz) {
size_t oldsz = sizeof(void *) * arenas->ptrs_sz;
arenas->freelist_sz *= 2;
void *newp = arenas->g->allocf(arenas->g->allocd, arenas->freelist, oldsz,
sizeof(void *) * arenas->freelist_sz);
if (!newp) {
/* Well this is awkward. We are freeing memory but can't allocate enough
* for the freelist. Maybe a smaller realloc will work? */
arenas->freelist_sz = arenas->freelist_at + n;
newp = arenas->g->allocf(arenas->g->allocd, arenas->freelist, oldsz,
sizeof(void *) * arenas->freelist_sz);
if (!newp) {
/* Just leak it, which does not help our OOM situation... */
/* TODO: use arenas for the freelist, maybe chained? Pretty wasteful
* for most scenarios that don't have gigabytes freed. Maybe release
* pages back to the OS? */
return;
}
}
arenas->freelist = (void**)newp;
}
for (uint32_t i = 0; i < n; i++) {
arenas->freelist[arenas->freelist_at++] = pages[i];
UNCOMMIT_PAGES(pages[i], ARENA_SIZE);
}
}
}
static unsigned lj_arena_api_allocpages(void *ud, void **pages, unsigned n)
{
arena_alloc *arenas = (arena_alloc*)ud;
if (arenas->freelist_at >= n) {
for (uint32_t i = 0; i < n; i++) {
pages[i] = arenas->freelist[--arenas->freelist_at];
COMMIT_PAGES(pages[i], ARENA_SIZE);
}
} else if (arenas->end - arenas->at >= n * ARENA_SIZE) {
COMMIT_PAGES(arenas->at, n * ARENA_SIZE);
for (uint32_t i = 0; i < n; i++) {
pages[i] = arenas->at;
arenas->at += ARENA_SIZE;
}
} else {
for (uint32_t i = 0; i < n; i++) {
if (arenas->at == arenas->end) {
lj_arena_newchunk(arenas);
}
COMMIT_PAGES(arenas->at, ARENA_SIZE);
pages[i] = arenas->at;
arenas->at += ARENA_SIZE;
}
}
return n;
}
struct posix_huge_arena
{
void *base;
size_t size;
};
static void *allochuge(size_t sz)
{
#if LJ_TARGET_WINDOWS
return RESERVE_AND_COMMIT_PAGES(sz);
#else
void *p = RESERVE_AND_COMMIT_PAGES(sz + ARENA_SIZE);
void *s = PTRALIGN_UP(p, ARENA_SIZE);
struct posix_huge_arena *a;
#if LJ_ALLOC_FIXUP
if (s != p)
UNRESERVE_PAGES(p, (uint8_t *)s - (uint8_t *)p);
a = (struct posix_huge_arena *)s;
a->base = s;
a->size = sz + ARENA_SIZE - ((uint8_t *)s - (uint8_t *)p);
#else
a->base = p;
a->size = sz + ARENA_SIZE;
#endif
return s;
#endif
}
static void *lj_arena_api_reallochuge(void *ud, void *p, size_t osz, size_t nsz)
{
void *newp;
if (!p)
return allochuge(nsz);
#if LJ_TARGET_WINDOWS
if (!nsz) {
UNRESERVE_PAGES(p, sz);
return NULL;
}
newp = RESERVE_AND_COMMIT_PAGES(nsz);
if (LJ_LIKELY(newp))
memcpy(newp, p, osz);
UNRESERVE_PAGES(p, osz);
#else
struct posix_huge_arena *a;
a = (struct posix_huge_arena *)p;
if (!nsz) {
UNRESERVE_PAGES(a->base, a->size);
return NULL;
}
if (nsz <= a->size)
return p;
#if LJ_ALLOC_MREMAP
/* We can use mremap, but we also require a 64k base address.
* We could just try it and adjust but you can't move a mapping
* to an overlapping range. But mremap overwrites existing mappings,
* so we can use mmap to query a suitable range. */
void *rawp = mmap(NULL, nsz + ARENA_SIZE, PROT_WRITE | PROT_READ,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
newp = PTRALIGN_UP(rawp, ARENA_SIZE);
if (rawp != newp)
UNRESERVE_PAGES(rawp, (uint8_t *)newp - (uint8_t *)rawp);
/* Can either unmap or adopt the extra memory. This is one fewer syscall */
nsz += ARENA_SIZE - ((uint8_t *)newp - (uint8_t *)rawp);
if (mremap(a->base, a->size, nsz, MREMAP_MAYMOVE | MREMAP_FIXED,
newp) != newp) {
UNRESERVE_PAGES(rawp, nsz + ARENA_SIZE);
return NULL;
}
a = (struct posix_huge_arena *)newp;
a->base = newp;
a->size = nsz;
#else
newp = allochuge(nsz);
memcpy(newp, p, osz);
UNRESERVE_PAGES(a->base, a->size);
#endif
#endif
return newp;
}
#define lj_arena_firstalloc(g, arena, id, atype, type) \
{ \
atype *a = (atype *)lj_arena_alloc(&g->gc.ctx); \
if (!a) \
return 0; \
arena = &a->hdr; \
do_arena_init(a, g, id, atype, type); \
}
static int lj_blob_firstalloc(global_State *g, GCAblob **h)
{
GCAblob *a = (GCAblob *)lj_arena_alloc(&g->gc.ctx);
if (!a)
return 0;
g->gc.bloblist[0] = a;
g->gc.bloblist_usage[0] = 0;
a->alloc = sizeof(GCAblob);
a->flags = LJ_GC_SWEEP0;
a->id = 0;
*h = a;
return 1;
}
int lj_arena_init(struct global_State *g, luaJIT_allocpages allocp,
luaJIT_freepages freep, luaJIT_reallochuge realloch,
void *page_ud)
{
g->gc.bloblist_alloc = 32;
g->gc.bloblist =
(GCAblob **)g->allocf(g->allocd, NULL, 0,
g->gc.bloblist_alloc * sizeof(void *));
if (!g->gc.bloblist)
return 0;
g->gc.bloblist_usage = (uint32_t *)g->allocf(
g->allocd, g->gc.bloblist_usage, 0,
g->gc.bloblist_alloc * sizeof(uint32_t));
if (!g->gc.bloblist_usage)
return 0;
g->gc.bloblist_wr = 1;
/* All must be provided to override */
if (allocp && freep && realloch) {
g->gc.ctx.allocpages = allocp;
g->gc.ctx.freepages = freep;
g->gc.ctx.reallochuge = realloch;
g->gc.ctx.pageud = page_ud;
} else {
arena_alloc *arenas = (arena_alloc*)g->allocf(g->allocd, NULL, 0, sizeof(arena_alloc));
if (!arenas)
return 0;
memset(arenas, 0, sizeof(arena_alloc));
g->gc.ctx.allocpages = &lj_arena_api_allocpages;
g->gc.ctx.freepages = &lj_arena_api_freepages;
g->gc.ctx.reallochuge = &lj_arena_api_reallochuge;
g->gc.ctx.pageud = arenas;
arenas->g = g;
arenas->freelist_sz = 32;
arenas->freelist = (void **)g->allocf(g->allocd, NULL, 0,
arenas->freelist_sz * sizeof(void *));
arenas->ptrs_sz = 8;
arenas->ptrs = (void **)g->allocf(g->allocd, NULL, 0,
arenas->ptrs_sz * sizeof(void *));
if (!arenas->freelist || !arenas->ptrs || !lj_arena_newchunk(arenas)) {
return 0;
}
}
/* Allocate all arenas */
lj_arena_firstalloc(g, g->gc.tab, ~LJ_TTAB, GCAtab, GCtab);
lj_arena_firstalloc(g, g->gc.fintab, ~LJ_TTAB, GCAtab, GCtab);
lj_arena_firstalloc(g, g->gc.uv, ~LJ_TUPVAL, GCAupval, GCupval);
lj_arena_firstalloc(g, g->gc.func, ~LJ_TFUNC, GCAfunc, GCfunc);
lj_arena_firstalloc(g, g->gc.udata, ~LJ_TUDATA, GCAudata, GCudata);
((GCAudata *)g->gc.udata)->free4_h = ((GCAudata *)g->gc.udata)->free_h;
return lj_blob_firstalloc(g, &g->gc.blob_generic);
}
static void release_one_arena(struct global_State *g, void *a)
{
lj_arena_free(&g->gc.ctx, a);
}
static void release_chain_arena(struct global_State *g, GCArenaHdr *a)
{
while (a) {
GCArenaHdr *n = a->next;
release_one_arena(g, a);
a = n;
}
}
static void release_all_arenas(struct global_State *g)
{
release_chain_arena(g, g->gc.tab);
release_chain_arena(g, g->gc.fintab);
release_chain_arena(g, g->gc.uv);
release_chain_arena(g, g->gc.func);
release_chain_arena(g, g->gc.udata);
for (uint32_t i = 0; i < g->gc.bloblist_wr; i++) {
GCAblob *a = g->gc.bloblist[i];
if (a->flags & GCA_BLOB_HUGE)
lj_arena_freehuge(&g->gc.ctx, a, a->alloc);
else
release_one_arena(g, a);
}
}
void lj_arena_cleanup(struct global_State *g)
{
release_all_arenas(g);
g->allocf(g->allocd, g->gc.bloblist, g->gc.bloblist_alloc * sizeof(void *), 0);
g->allocf(g->allocd, g->gc.bloblist_usage, g->gc.bloblist_alloc * sizeof(uint32_t), 0);
/* Early failures may not initialize this. release_all_arenas will not have
* any actual arenas to release, thus it is safe. */
if (!g->gc.ctx.freepages)
return;
g->gc.ctx.mem_commit -= g->gc.ctx.freelist_at;
if (g->gc.ctx.freelist_at > 0) {
g->gc.ctx.freepages(g->gc.ctx.pageud, g->gc.ctx.freelist,
g->gc.ctx.freelist_at);
}
/* Special notice to shut down */
g->gc.ctx.freepages(g->gc.ctx.pageud, NULL, 0);
g->gc.ctx.freelist_at = 0;
}

124
src/lj_arena.h Normal file
View File

@ -0,0 +1,124 @@
#ifndef _LJ_ARENA_H
#define _LJ_ARENA_H
#include <stddef.h>
#include <stdint.h>
#include <string.h>
#include "lj_arch.h"
#include "lj_def.h"
typedef unsigned (*luaJIT_allocpages)(void *ud, void **pages, unsigned n);
typedef void (*luaJIT_freepages)(void *ud, void **pages, unsigned n);
typedef void *(*luaJIT_reallochuge)(void *ud, void *p, size_t osz, size_t nsz);
#define ARENA_SHIFT 16
#define ARENA_SIZE (1u << ARENA_SHIFT)
#define ARENA_HUGE_THRESHOLD 16000
#define ARENA_MASK (~0ull << ARENA_SHIFT)
#define ARENA_OMASK (ARENA_SIZE - 1)
#define ARENA_FREELIST_SIZE 32
#define ARENA_FREELIST_CHUNK (ARENA_FREELIST_SIZE / 2)
#define CHUNK_SIZE (ARENA_SIZE * ARENA_FREELIST_CHUNK)
#define MINIMUM_QUOTA (32 * ARENA_SIZE)
struct global_State;
/* These are here so obj.h can see them */
/* Common header for all bitmap arenas */
typedef struct GCArenaHdr {
struct GCArenaHdr *prev, *next;
struct GCArenaHdr *freeprev, *freenext;
struct GCArenaHdr *gray;
uint32_t obj_type;
uint32_t flags;
} GCArenaHdr;
/* If set this arena is being moved from */
#define GCA_BLOB_REAP 0x08
#define GCA_BLOB_HUGE 0x10
/* Blob arenas don't hold objects but instead data pointed to by objects.
* All blob arenas are solely bump allocated and compacted as needed.
*/
typedef struct GCAblob {
uint8_t reserved_for_impl[20];
uint32_t id;
uint32_t alloc; /* Next byte to be allocated from */
uint32_t flags;
} GCAblob;
LJ_STATIC_ASSERT(sizeof(GCAblob) % 16 == 0);
typedef struct arena_context {
/* In arenas */
uint32_t mem_commit;
uint32_t mem_watermark;
/* Bytes */
uint64_t mem_huge;
/* The free cache represents the "working memory" of the state */
uint32_t freelist_at;
void *freelist[ARENA_FREELIST_SIZE];
luaJIT_allocpages allocpages;
luaJIT_freepages freepages;
luaJIT_reallochuge reallochuge;
void *pageud;
} arena_context;
int lj_arena_init(struct global_State *g, luaJIT_allocpages allocp,
luaJIT_freepages freep, luaJIT_reallochuge realloch,
void *page_ud);
void lj_arena_cleanup(struct global_State *g);
/* Add ARENA_FREELIST_CHUNK free arenas */
inline void *lj_arena_alloc(arena_context *ctx)
{
if (!ctx->freelist_at) {
ctx->freelist_at =
ctx->allocpages(ctx->pageud, ctx->freelist, ARENA_FREELIST_CHUNK);
ctx->mem_commit += ctx->freelist_at;
if (ctx->mem_commit > ctx->mem_watermark)
ctx->mem_watermark = ctx->mem_commit;
if (!ctx->freelist_at)
return NULL;
}
void *p = ctx->freelist[--ctx->freelist_at];
return p;
}
/* Free the last ARENA_FREELIST_CHUNK arenas */
inline void lj_arena_free(arena_context *ctx, void *p)
{
if (ctx->freelist_at == ARENA_FREELIST_SIZE) {
ctx->freepages(ctx->pageud,
ctx->freelist +
(ARENA_FREELIST_SIZE - ARENA_FREELIST_CHUNK),
ARENA_FREELIST_CHUNK);
ctx->freelist_at -= ARENA_FREELIST_CHUNK;
ctx->mem_commit -= ARENA_FREELIST_CHUNK;
}
ctx->freelist[ctx->freelist_at++] = p;
}
inline void *lj_arena_allochuge(arena_context *ctx, size_t sz)
{
ctx->mem_huge += sz;
return ctx->reallochuge(ctx->pageud, NULL, 0, sz);
}
inline void lj_arena_freehuge(arena_context *ctx, void *p, size_t sz)
{
ctx->mem_huge -= sz;
ctx->reallochuge(ctx->pageud, p, sz, 0);
}
inline void *lj_arena_reallochuge(arena_context *ctx, void *p, size_t osz, size_t nsz)
{
ctx->mem_huge = ctx->mem_huge - osz + nsz;
return ctx->reallochuge(ctx->pageud, p, osz, nsz);
}
#endif

49
src/lj_asm_x86.h
View File

@ -1398,9 +1398,10 @@ static void asm_uref(ASMState *as, IRIns *ir)
GCobj *o = gcref(fn->l.uvptr[(ir->op2 >> 8)]);
emit_loada(as, uv, o);
} else {
emit_rmro(as, XO_MOV, uv|REX_GC64, ra_alloc1(as, ir->op1, RSET_GPR),
(int32_t)offsetof(GCfuncL, uvptr) +
(int32_t)sizeof(MRef) * (int32_t)(ir->op2 >> 8));
emit_rmro(as, XO_MOV, uv|REX_GC64, uv,
(int32_t)sizeof(MRef) * (int32_t)(ir->op2 >> 8));
emit_rmro(as, XO_MOV, uv|REX_GC64,
ra_alloc1(as, ir->op1, RSET_GPR), offsetof(GCfuncL, uvptr));
}
}
}
@ -1892,11 +1893,8 @@ static void asm_cnew(ASMState *as, IRIns *ir)
}
/* Combine initialization of marked, gct and ctypeid. */
emit_movtomro(as, RID_ECX, RID_RET, offsetof(GCcdata, marked));
emit_gri(as, XG_ARITHi(XOg_OR), RID_ECX,
(int32_t)((~LJ_TCDATA<<8)+(id<<16)));
emit_gri(as, XG_ARITHi(XOg_AND), RID_ECX, LJ_GC_WHITES);
emit_opgl(as, XO_MOVZXb, RID_ECX, gc.currentwhite);
emit_movmroi(as, RID_RET, offsetof(GCcdata, gcflags),
(int32_t)((~LJ_TCDATA << 8) + (id << 16)));
args[0] = ASMREF_L; /* lua_State *L */
args[1] = ASMREF_TMP1; /* MSize size */
@ -1912,14 +1910,15 @@ static void asm_tbar(ASMState *as, IRIns *ir)
Reg tab = ra_alloc1(as, ir->op1, RSET_GPR);
Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, tab));
MCLabel l_end = emit_label(as);
emit_movtomro(as, tmp|REX_GC64, tab, offsetof(GCtab, gclist));
emit_movtomro(as, tmp | REX_GC64, tab, offsetof(GCtab, gclist));
emit_setgl(as, tab, gc.grayagain);
emit_getgl(as, tmp, gc.grayagain);
emit_i8(as, ~LJ_GC_BLACK);
emit_rmro(as, XO_ARITHib, XOg_AND, tab, offsetof(GCtab, marked));
emit_sjcc(as, CC_Z, l_end);
emit_i8(as, LJ_GC_BLACK);
emit_rmro(as, XO_GROUP3b, XOg_TEST, tab, offsetof(GCtab, marked));
emit_gmroi(as, XG_ARITHi(XOg_OR), tab, offsetof(GCtab, gcflags), LJ_GC_GRAY);
emit_sjcc(as, CC_NZ, l_end);
emit_opgl(as, XO_CMPb, tmp | REX_GC64, gc.currentblack);
emit_gri(as, XG_ARITHi(XOg_AND), tmp, LJ_GC_COLORS);
emit_rmro(as, XO_MOVZXb, tmp, tab, offsetof(GCtab, gcflags));
}
static void asm_obar(ASMState *as, IRIns *ir)
@ -1927,7 +1926,7 @@ static void asm_obar(ASMState *as, IRIns *ir)
const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_gc_barrieruv];
IRRef args[2];
MCLabel l_end;
Reg obj;
Reg obj, tmp;
/* No need for other object barriers (yet). */
lj_assertA(IR(ir->op1)->o == IR_UREFC, "bad OBAR type");
ra_evictset(as, RSET_SCRATCH);
@ -1937,19 +1936,21 @@ static void asm_obar(ASMState *as, IRIns *ir)
asm_gencall(as, ci, args);
emit_loada(as, ra_releasetmp(as, ASMREF_TMP1), J2G(as->J));
obj = IR(ir->op1)->r;
emit_sjcc(as, CC_Z, l_end);
emit_i8(as, LJ_GC_WHITES);
tmp = ra_scratch(as, rset_exclude(RSET_GPR, obj));
emit_sjcc(as, CC_NZ, l_end);
emit_opgl(as, XO_TESTb, tmp | REX_GC64, gc.currentblackgray);
if (irref_isk(ir->op2)) {
GCobj *vp = ir_kgc(IR(ir->op2));
emit_rma(as, XO_GROUP3b, XOg_TEST, &vp->gch.marked);
emit_rma(as, XO_MOVZXb, tmp, &vp->gch.gcflags);
} else {
Reg val = ra_alloc1(as, ir->op2, rset_exclude(RSET_SCRATCH&RSET_GPR, obj));
emit_rmro(as, XO_GROUP3b, XOg_TEST, val, (int32_t)offsetof(GChead, marked));
emit_rmro(as, XO_MOVZXb, tmp, obj, (int32_t) offsetof(GChead, gcflags));
}
emit_sjcc(as, CC_Z, l_end);
emit_i8(as, LJ_GC_BLACK);
emit_rmro(as, XO_GROUP3b, XOg_TEST, obj,
(int32_t)offsetof(GCupval, marked)-(int32_t)offsetof(GCupval, tv));
emit_sjcc(as, CC_NZ, l_end);
emit_opgl(as, XO_CMPb, tmp | REX_GC64, gc.currentblack);
emit_gri(as, XG_ARITHi(XOg_AND), tmp, LJ_GC_COLORS);
emit_rmro(as, XO_MOVZXb, tmp, obj,
(int32_t)offsetof(GCupval, gcflags) -
(int32_t)offsetof(GCupval, tv));
}
/* -- FP/int arithmetic and logic operations ------------------------------ */

11
src/lj_cdata.c
View File

@ -42,8 +42,8 @@ GCcdata *lj_cdata_newv(lua_State *L, CTypeID id, CTSize sz, CTSize align)
g = G(L);
setgcrefr(cd->nextgc, g->gc.root);
setgcref(g->gc.root, obj2gco(cd));
newwhite(g, obj2gco(cd));
cd->marked |= 0x80;
newwhite(obj2gco(cd));
cd->gcflags |= 0x80;
cd->gct = ~LJ_TCDATA;
cd->ctypeid = id;
return cd;
@ -61,9 +61,8 @@ GCcdata *lj_cdata_newx(CTState *cts, CTypeID id, CTSize sz, CTInfo info)
/* Free a C data object. */
void LJ_FASTCALL lj_cdata_free(global_State *g, GCcdata *cd)
{
if (LJ_UNLIKELY(cd->marked & LJ_GC_CDATA_FIN)) {
if (LJ_UNLIKELY(cd->gcflags & LJ_GC_CDATA_FIN)) {
GCobj *root;
makewhite(g, obj2gco(cd));
markfinalized(obj2gco(cd));
if ((root = gcref(g->gc.mmudata)) != NULL) {
setgcrefr(cd->nextgc, root->gch.nextgc);
@ -95,10 +94,10 @@ void lj_cdata_setfin(lua_State *L, GCcdata *cd, GCobj *obj, uint32_t it)
tv = lj_tab_set(L, t, &tmp);
if (it == LJ_TNIL) {
setnilV(tv);
cd->marked &= ~LJ_GC_CDATA_FIN;
cd->gcflags &= ~LJ_GC_CDATA_FIN;
} else {
setgcV(L, tv, obj, it);
cd->marked |= LJ_GC_CDATA_FIN;
cd->gcflags |= LJ_GC_CDATA_FIN;
}
}
}

3
src/lj_clib.c
View File

@ -17,6 +17,7 @@
#include "lj_cdata.h"
#include "lj_clib.h"
#include "lj_strfmt.h"
#include "lj_meta.h"
/* -- OS-specific functions ----------------------------------------------- */
@ -405,6 +406,8 @@ static CLibrary *clib_new(lua_State *L, GCtab *mt)
ud->udtype = UDTYPE_FFI_CLIB;
/* NOBARRIER: The GCudata is new (marked white). */
setgcref(ud->metatable, obj2gco(mt));
if (lj_meta_fastg(G(L), tabref(ud->metatable), MM_gc))
lj_mem_registergc_udata(L, ud);
setudataV(L, L->top++, ud);
return cl;
}

2
src/lj_crecord.c
View File

@ -623,7 +623,7 @@ static TRef crec_ct_tv(jit_State *J, CType *d, TRef dp, TRef sp, cTValue *sval)
ud->udtype == UDTYPE_IO_FILE ? IRFL_UDATA_FILE :
IRFL_SBUF_R);
} else {
sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCudata)));
sp = emitir(IRT(IR_FLOAD, IRT_PTR), sp, IRFL_UDATA_PAYLOAD);
}
} else if (tref_isstr(sp)) {
if (ctype_isenum(d->info)) { /* Match string against enum constant. */

6
src/lj_debug.c
View File

@ -240,7 +240,7 @@ const char *lj_debug_uvnamev(cTValue *o, uint32_t idx, TValue **tvp, GCobj **op)
}
} else {
if (idx < fn->c.nupvalues) {
*tvp = &fn->c.upvalue[idx];
*tvp = &fn->c.data->upvalue[idx];
*op = obj2gco(fn);
return "";
}
@ -632,7 +632,7 @@ void lj_debug_dumpstack(lua_State *L, SBuf *sb, const char *fmt, int depth)
lj_buf_putb(sb, ']');
} else { /* Dump C function address. */
lj_buf_putb(sb, '@');
lj_strfmt_putptr(sb, fn->c.f);
lj_strfmt_putptr(sb, fn->c.data->f);
}
break;
case 'Z': /* Zap trailing separator. */
@ -694,7 +694,7 @@ LUALIB_API void luaL_traceback (lua_State *L, lua_State *L1, const char *msg,
if (*ar.what == 'm') {
lua_pushliteral(L, " in main chunk");
} else if (*ar.what == 'C') {
lua_pushfstring(L, " at %p", fn->c.f);
lua_pushfstring(L, " at %p", fn->c.data->f);
} else {
lua_pushfstring(L, " in function <%s:%d>",
ar.short_src, ar.linedefined);

3
src/lj_def.h
View File

@ -59,7 +59,8 @@ typedef unsigned int uintptr_t;
#define LJ_MAX_HBITS 26 /* Max. hash bits. */
#define LJ_MAX_ABITS 28 /* Max. bits of array key. */
#define LJ_MAX_ASIZE ((1<<(LJ_MAX_ABITS-1))+1) /* Max. array part size. */
#define LJ_MAX_COLOSIZE 16 /* Max. elems for colocated array. */
#define LJ_MAX_COLOSIZE 0 /* Max. elems for colocated array. */
#define LJ_COLO_ENABLED 1
#define LJ_MAX_LINE LJ_MAX_MEM32 /* Max. source code line number. */
#define LJ_MAX_XLEVEL 200 /* Max. syntactic nesting level. */

2
src/lj_err.c
View File

@ -1077,7 +1077,7 @@ LJ_NOINLINE void lj_err_argtype(lua_State *L, int narg, const char *xname)
GCfunc *fn = curr_func(L);
int idx = LUA_GLOBALSINDEX - narg;
if (idx <= fn->c.nupvalues)
tname = lj_typename(&fn->c.upvalue[idx-1]);
tname = lj_typename(&fn->c.data->upvalue[idx-1]);
else
tname = lj_obj_typename[0];
}

6
src/lj_ffrecord.c
View File

@ -214,7 +214,7 @@ static void LJ_FASTCALL recff_type(jit_State *J, RecordFFData *rd)
t = ~LJ_TLIGHTUD;
else
t = ~itype(&rd->argv[0]);
J->base[0] = lj_ir_kstr(J, strV(&J->fn->c.upvalue[t]));
J->base[0] = lj_ir_kstr(J, strV(&J->fn->c.data->upvalue[t]));
UNUSED(rd);
}
@ -452,7 +452,7 @@ static void LJ_FASTCALL recff_xpairs(jit_State *J, RecordFFData *rd)
if (!((LJ_52 || (LJ_HASFFI && tref_iscdata(tr))) &&
recff_metacall(J, rd, MM_pairs + rd->data))) {
if (tref_istab(tr)) {
J->base[0] = lj_ir_kfunc(J, funcV(&J->fn->c.upvalue[0]));
J->base[0] = lj_ir_kfunc(J, funcV(&J->fn->c.data->upvalue[0]));
J->base[1] = tr;
J->base[2] = rd->data ? lj_ir_kint(J, 0) : TREF_NIL;
rd->nres = 3;
@ -663,7 +663,7 @@ static void LJ_FASTCALL recff_math_minmax(jit_State *J, RecordFFData *rd)
static void LJ_FASTCALL recff_math_random(jit_State *J, RecordFFData *rd)
{
GCudata *ud = udataV(&J->fn->c.upvalue[0]);
GCudata *ud = udataV(&J->fn->c.data->upvalue[0]);
TRef tr, one;
lj_ir_kgc(J, obj2gco(ud), IRT_UDATA); /* Prevent collection. */
tr = lj_ir_call(J, IRCALL_lj_prng_u64d, lj_ir_kptr(J, uddata(ud)));

64
src/lj_func.c
View File

@ -24,15 +24,6 @@ void LJ_FASTCALL lj_func_freeproto(global_State *g, GCproto *pt)
/* -- Upvalues ------------------------------------------------------------ */
static void unlinkuv(global_State *g, GCupval *uv)
{
UNUSED(g);
lj_assertG(uvprev(uvnext(uv)) == uv && uvnext(uvprev(uv)) == uv,
"broken upvalue chain");
setgcrefr(uvnext(uv)->prev, uv->prev);
setgcrefr(uvprev(uv)->next, uv->next);
}
/* Find existing open upvalue for a stack slot or create a new one. */
static GCupval *func_finduv(lua_State *L, TValue *slot)
{
@ -44,34 +35,27 @@ static GCupval *func_finduv(lua_State *L, TValue *slot)
while (gcref(*pp) != NULL && uvval((p = gco2uv(gcref(*pp)))) >= slot) {
lj_assertG(!p->closed && uvval(p) != &p->tv, "closed upvalue in chain");
if (uvval(p) == slot) { /* Found open upvalue pointing to same slot? */
if (isdead(g, obj2gco(p))) /* Resurrect it, if it's dead. */
flipwhite(obj2gco(p));
return p;
}
pp = &p->nextgc;
pp = &p->next;
}
/* No matching upvalue found. Create a new one. */
uv = lj_mem_newt(L, sizeof(GCupval), GCupval);
newwhite(g, uv);
uv = lj_mem_allocuv(L);
newwhite(uv);
uv->gct = ~LJ_TUPVAL;
uv->closed = 0; /* Still open. */
setmref(uv->v, slot); /* Pointing to the stack slot. */
/* NOBARRIER: The GCupval is new (marked white) and open. */
setgcrefr(uv->nextgc, *pp); /* Insert into sorted list of open upvalues. */
setgcrefr(uv->next, *pp); /* Insert into sorted list of open upvalues. */
setgcref(*pp, obj2gco(uv));
setgcref(uv->prev, obj2gco(&g->uvhead)); /* Insert into GC list, too. */
setgcrefr(uv->next, g->uvhead.next);
setgcref(uvnext(uv)->prev, obj2gco(uv));
setgcref(g->uvhead.next, obj2gco(uv));
lj_assertG(uvprev(uvnext(uv)) == uv && uvnext(uvprev(uv)) == uv,
"broken upvalue chain");
return uv;
}
/* Create an empty and closed upvalue. */
static GCupval *func_emptyuv(lua_State *L)
{
GCupval *uv = (GCupval *)lj_mem_newgco(L, sizeof(GCupval));
GCupval *uv = lj_mem_allocuv(L);
newwhite(uv);
uv->gct = ~LJ_TUPVAL;
uv->closed = 1;
setnilV(&uv->tv);
@ -86,32 +70,21 @@ void LJ_FASTCALL lj_func_closeuv(lua_State *L, TValue *level)
global_State *g = G(L);
while (gcref(L->openupval) != NULL &&
uvval((uv = gco2uv(gcref(L->openupval)))) >= level) {
GCobj *o = obj2gco(uv);
lj_assertG(!isblack(o), "bad black upvalue");
lj_assertG(gcref(L->openupval) == obj2gco(uv), "bad upvalue list");
lj_assertG(!uv->closed && uvval(uv) != &uv->tv, "closed upvalue in chain");
setgcrefr(L->openupval, uv->nextgc); /* No longer in open list. */
if (isdead(g, o)) {
lj_func_freeuv(g, uv);
} else {
unlinkuv(g, uv);
lj_gc_closeuv(g, uv);
}
}
}
setgcrefr(L->openupval, uv->next); /* No longer in open list. */
void LJ_FASTCALL lj_func_freeuv(global_State *g, GCupval *uv)
{
if (!uv->closed)
unlinkuv(g, uv);
lj_mem_freet(g, uv);
copyTV(mainthread(g), &uv->tv, uvval(uv));
setmref(uv->v, &uv->tv);
uv->closed = 1;
}
}
/* -- Functions (closures) ------------------------------------------------ */
GCfunc *lj_func_newC(lua_State *L, MSize nelems, GCtab *env)
{
GCfunc *fn = (GCfunc *)lj_mem_newgco(L, sizeCfunc(nelems));
fn->c.gct = ~LJ_TFUNC;
GCfunc *fn = lj_mem_allocfunc(L, sizeCfunc(nelems));
fn->c.ffid = FF_C;
fn->c.nupvalues = (uint8_t)nelems;
/* NOBARRIER: The GCfunc is new (marked white). */
@ -123,8 +96,7 @@ GCfunc *lj_func_newC(lua_State *L, MSize nelems, GCtab *env)
static GCfunc *func_newL(lua_State *L, GCproto *pt, GCtab *env)
{
uint32_t count;
GCfunc *fn = (GCfunc *)lj_mem_newgco(L, sizeLfunc((MSize)pt->sizeuv));
fn->l.gct = ~LJ_TFUNC;
GCfunc *fn = lj_mem_allocfunc(L, sizeLfunc((MSize)pt->sizeuv));
fn->l.ffid = FF_LUA;
fn->l.nupvalues = 0; /* Set to zero until upvalues are initialized. */
/* NOBARRIER: Really a setgcref. But the GCfunc is new (marked white). */
@ -181,11 +153,3 @@ GCfunc *lj_func_newL_gc(lua_State *L, GCproto *pt, GCfuncL *parent)
fn->l.nupvalues = (uint8_t)nuv;
return fn;
}
void LJ_FASTCALL lj_func_free(global_State *g, GCfunc *fn)
{
MSize size = isluafunc(fn) ? sizeLfunc((MSize)fn->l.nupvalues) :
sizeCfunc((MSize)fn->c.nupvalues);
lj_mem_free(g, fn, size);
}

2
src/lj_func.h
View File

@ -13,12 +13,10 @@ LJ_FUNC void LJ_FASTCALL lj_func_freeproto(global_State *g, GCproto *pt);
/* Upvalues. */
LJ_FUNCA void LJ_FASTCALL lj_func_closeuv(lua_State *L, TValue *level);
LJ_FUNC void LJ_FASTCALL lj_func_freeuv(global_State *g, GCupval *uv);
/* Functions (closures). */
LJ_FUNC GCfunc *lj_func_newC(lua_State *L, MSize nelems, GCtab *env);
LJ_FUNC GCfunc *lj_func_newL_empty(lua_State *L, GCproto *pt, GCtab *env);
LJ_FUNCA GCfunc *lj_func_newL_gc(lua_State *L, GCproto *pt, GCfuncL *parent);
LJ_FUNC void LJ_FASTCALL lj_func_free(global_State *g, GCfunc *c);
#endif

1884
src/lj_gc.c
View File

File diff suppressed because it is too large Load Diff

281
src/lj_gc.h
View File

@ -7,46 +7,71 @@
#define _LJ_GC_H
#include "lj_obj.h"
#include "lj_jit.h"
#include "lj_intrin.h"
/* Garbage collector states. Order matters. */
enum {
GCSpause, GCSpropagate, GCSatomic, GCSsweepstring, GCSsweep, GCSfinalize
GCSpause, GCSpropagate, GCSatomic, GCSsweepstring, GCSsweep,
GCSsweep_blob, GCSsweep_func, GCSsweep_tab, GCSsweep_fintab,
GCSsweep_uv, GCSsweep_udata, GCSfinalize_arena, GCSfinalize
};
/* Bitmasks for marked field of GCobj. */
#define LJ_GC_WHITE0 0x01
#define LJ_GC_WHITE1 0x02
#define LJ_GC_BLACK 0x04
#define LJ_GC_BLACK0 0x01
#define LJ_GC_BLACK1 0x02
#define LJ_GC_GRAY 0x04
#define LJ_GC_FINALIZED 0x08
#define LJ_GC_WEAKKEY 0x08
#define LJ_GC_WEAKVAL 0x10
#define LJ_GC_WEAKKEY 0x10
#define LJ_GC_WEAKVAL 0x08
#define LJ_GC_CDATA_FIN 0x10
#define LJ_GC_FIXED 0x20
#define LJ_GC_SFIXED 0x40
#define LJ_GC_WHITES (LJ_GC_WHITE0 | LJ_GC_WHITE1)
#define LJ_GC_COLORS (LJ_GC_WHITES | LJ_GC_BLACK)
#define LJ_GC_MARK_MASK 0xE0
#define size2flags(n) (((1u << (n - 1)) - 1) << 5)
#define flags2bitmask(o, bit) (((bitmap_t)(o)->gch.gcflags >> 5) << ((bit) + 1))
#define LJ_GCMODE_MINORSWEEP 0x01
#define LJ_GCMODE_ENABLE_MINORSWEEP 0x02
#define LJ_GC_BLACKS (LJ_GC_BLACK0 | LJ_GC_BLACK1)
#define LJ_GC_COLORS (LJ_GC_BLACKS | LJ_GC_GRAY)
#define LJ_GC_WEAK (LJ_GC_WEAKKEY | LJ_GC_WEAKVAL)
#define LJ_GC_SWEEP0 0x01
#define LJ_GC_SWEEP1 0x02
#define LJ_GC_SWEEPS (LJ_GC_SWEEP0 | LJ_GC_SWEEP1)
/* Macros to test and set GCobj colors. */
#define iswhite(x) ((x)->gch.marked & LJ_GC_WHITES)
#define isblack(x) ((x)->gch.marked & LJ_GC_BLACK)
#define isgray(x) (!((x)->gch.marked & (LJ_GC_BLACK|LJ_GC_WHITES)))
#define tviswhite(x) (tvisgcv(x) && iswhite(gcV(x)))
#define otherwhite(g) (g->gc.currentwhite ^ LJ_GC_WHITES)
#define isdead(g, v) ((v)->gch.marked & otherwhite(g) & LJ_GC_WHITES)
#define iswhite(g, x) (!((x)->gch.gcflags & (g)->gc.currentblackgray))
#define isblack(g, x) (((x)->gch.gcflags & LJ_GC_COLORS) == (g)->gc.currentblack)
#define isgray(x) ((x)->gch.gcflags & LJ_GC_GRAY)
#define tviswhite(g, x) (tvisgcv(x) && iswhite(g, gcV(x)))
/* Death checking should only be done in asserts */
LJ_FUNC int checkdead(global_State *g, GCobj *o);
#define curwhite(g) ((g)->gc.currentwhite & LJ_GC_WHITES)
#define newwhite(g, x) (obj2gco(x)->gch.marked = (uint8_t)curwhite(g))
#define makewhite(g, x) \
((x)->gch.marked = ((x)->gch.marked & (uint8_t)~LJ_GC_COLORS) | curwhite(g))
#define flipwhite(x) ((x)->gch.marked ^= LJ_GC_WHITES)
#define black2gray(x) ((x)->gch.marked &= (uint8_t)~LJ_GC_BLACK)
#define fixstring(s) ((s)->marked |= LJ_GC_FIXED)
#define markfinalized(x) ((x)->gch.marked |= LJ_GC_FINALIZED)
#define newwhite(x) (obj2gco(x)->gch.gcflags = 0)
#define makewhite(x) \
((x)->gch.gcflags = ((x)->gch.gcflags & (uint8_t)~LJ_GC_COLORS))
#define black2gray(x) ((x)->gch.gcflags |= (uint8_t)LJ_GC_GRAY)
#define fixstring(s) ((s)->gcflags |= LJ_GC_FIXED)
#define markfinalized(x) ((x)->gch.gcflags |= LJ_GC_FINALIZED)
#define maybe_resurrect_str(g, s) \
if (LJ_UNLIKELY(iswhite(g, obj2gco(s)) && (g)->gc.state == GCSsweepstring && \
((s)->hash & (g)->str.mask) >= (g)->gc.sweepstr)) \
{ \
(s)->gcflags |= (g)->gc.currentblack; \
}
#define isminor(g) (g->gc.gcmode & LJ_GCMODE_MINORSWEEP)
/* Collector. */
LJ_FUNC size_t lj_gc_separateudata(global_State *g, int all);
LJ_FUNC void lj_gc_separateudata(global_State *g);
LJ_FUNC void lj_gc_finalize_udata(lua_State *L);
#if LJ_HASFFI
LJ_FUNC void lj_gc_finalize_cdata(lua_State *L);
@ -59,7 +84,7 @@ LJ_FUNCA void LJ_FASTCALL lj_gc_step_fixtop(lua_State *L);
#if LJ_HASJIT
LJ_FUNC int LJ_FASTCALL lj_gc_step_jit(global_State *g, MSize steps);
#endif
LJ_FUNC void lj_gc_fullgc(lua_State *L);
LJ_FUNC void lj_gc_fullgc(lua_State *L, int maximal);
/* GC check: drive collector forward if the GC threshold has been reached. */
#define lj_gc_check(L) \
@ -72,7 +97,6 @@ LJ_FUNC void lj_gc_fullgc(lua_State *L);
/* Write barriers. */
LJ_FUNC void lj_gc_barrierf(global_State *g, GCobj *o, GCobj *v);
LJ_FUNCA void LJ_FASTCALL lj_gc_barrieruv(global_State *g, TValue *tv);
LJ_FUNC void lj_gc_closeuv(global_State *g, GCupval *uv);
#if LJ_HASJIT
LJ_FUNC void lj_gc_barriertrace(global_State *g, uint32_t traceno);
#endif
@ -81,7 +105,7 @@ LJ_FUNC void lj_gc_barriertrace(global_State *g, uint32_t traceno);
static LJ_AINLINE void lj_gc_barrierback(global_State *g, GCtab *t)
{
GCobj *o = obj2gco(t);
lj_assertG(isblack(o) && !isdead(g, o),
lj_assertG(isblack(g, o) && !checkdead(g, o),
"bad object states for backward barrier");
lj_assertG(g->gc.state != GCSfinalize && g->gc.state != GCSpause,
"bad GC state");
@ -92,20 +116,20 @@ static LJ_AINLINE void lj_gc_barrierback(global_State *g, GCtab *t)
/* Barrier for stores to table objects. TValue and GCobj variant. */
#define lj_gc_anybarriert(L, t) \
{ if (LJ_UNLIKELY(isblack(obj2gco(t)))) lj_gc_barrierback(G(L), (t)); }
{ if (LJ_UNLIKELY(isblack(G(L), obj2gco(t)))) lj_gc_barrierback(G(L), (t)); }
#define lj_gc_barriert(L, t, tv) \
{ if (tviswhite(tv) && isblack(obj2gco(t))) \
{ if (tviswhite(G(L), tv) && isblack(G(L), obj2gco(t))) \
lj_gc_barrierback(G(L), (t)); }
#define lj_gc_objbarriert(L, t, o) \
{ if (iswhite(obj2gco(o)) && isblack(obj2gco(t))) \
{ if (iswhite(G(L), obj2gco(o)) && isblack(G(L), obj2gco(t))) \
lj_gc_barrierback(G(L), (t)); }
/* Barrier for stores to any other object. TValue and GCobj variant. */
#define lj_gc_barrier(L, p, tv) \
{ if (tviswhite(tv) && isblack(obj2gco(p))) \
{ if (tviswhite(G(L), tv) && isblack(G(L), obj2gco(p))) \
lj_gc_barrierf(G(L), obj2gco(p), gcV(tv)); }
#define lj_gc_objbarrier(L, p, o) \
{ if (iswhite(obj2gco(o)) && isblack(obj2gco(p))) \
{ if (iswhite(G(L), obj2gco(o)) && isblack(G(L), obj2gco(p))) \
lj_gc_barrierf(G(L), obj2gco(p), obj2gco(o)); }
/* Allocator. */
@ -119,18 +143,203 @@ LJ_FUNC void *lj_mem_grow(lua_State *L, void *p,
static LJ_AINLINE void lj_mem_free(global_State *g, void *p, size_t osize)
{
g->gc.total -= (GCSize)osize;
g->gc.malloc -= (GCSize)osize;
g->allocf(g->allocd, p, osize, 0);
}
#define lj_mem_newvec(L, n, t) ((t *)lj_mem_new(L, (GCSize)((n)*sizeof(t))))
#define lj_mem_reallocvec(L, p, on, n, t) \
((p) = (t *)lj_mem_realloc(L, p, (on)*sizeof(t), (GCSize)((n)*sizeof(t))))
#define lj_mem_growvec(L, p, n, m, t) \
#define lj_mem_newvec(L, n, t) ((t *)lj_mem_new(L, (GCSize)((n) * sizeof(t))))
#define lj_mem_reallocvec(L, p, on, n, t) \
((p) = (t *)lj_mem_realloc(L, p, (on) * sizeof(t), (GCSize)((n) * sizeof(t))))
#define lj_mem_growvec(L, p, n, m, t) \
((p) = (t *)lj_mem_grow(L, (p), &(n), (m), (MSize)sizeof(t)))
#define lj_mem_freevec(g, p, n, t) lj_mem_free(g, (p), (n)*sizeof(t))
#define lj_mem_freevec(g, p, n, t) lj_mem_free(g, (p), (n) * sizeof(t))
#define lj_mem_newobj(L, t) ((t *)lj_mem_newgco(L, sizeof(t)))
#define lj_mem_newv(L, n, t) ((t *)lj_mem_newblob(L, (MSize)((n) * sizeof(t))))
#define lj_mem_reallocv(L, p, on, n, t) \
(t *)lj_mem_reallocblob(L, p, (on) * sizeof(t), \
(MSize)((n) * sizeof(t)))
#define lj_mem_newobj(L, t) ((t *)lj_mem_newgco(L, sizeof(t)))
#define lj_mem_newt(L, s, t) ((t *)lj_mem_new(L, (s)))
#define lj_mem_freet(g, p) lj_mem_free(g, (p), sizeof(*(p)))
/* New GC */
LJ_FUNC GCtab *lj_mem_alloctab(lua_State *L, uint32_t asize);
LJ_FUNC GCtab *lj_mem_alloctabempty_gc(lua_State *L);
LJ_FUNC GCstr *lj_mem_allocstr(lua_State *L, MSize len);
LJ_FUNC GCupval *lj_mem_allocuv(lua_State *L);
LJ_FUNC GCudata *lj_mem_allocudata(lua_State *L, MSize bytes);
LJ_FUNC GCfunc *lj_mem_allocfunc(lua_State *L, MSize bytes);
LJ_FUNC void *lj_mem_newblob(lua_State *L, MSize sz);
LJ_FUNC void *lj_mem_reallocblob(lua_State *L, void *p, MSize osz, MSize nsz);
LJ_FUNC void lj_mem_registergc_udata(lua_State *L, GCudata *ud);
#define lj_gc_markblob(L, blob, size) \
G(L)->gc.bloblist_usage[gcablob(blob)->id] += size
typedef uint64_t bitmap_t;
#define WORD_BITS 64
#define SIMD_BITS 256
#define BLOB_REAP_THRESHOLD (ARENA_SIZE / 3)
/* Basic requirement is that each bitmap have an increment of 256 and that each
* area be 32-byte aligned. ELEMENTS_MAX is the maximum index allowed.
* WORDS_FOR_TYPE is the number of 64-bit words in each header bitmap
* WORDS_FOR_TYPE_UNROUNDED is WORDS_FOR_TYPE but not rounded up to 256 bits
* ELEMENTS_OCCUPIED is the number of elements occupied by the header
* HIGH_ELEMENTS_OCCUPIED is the number of elements in the last word
*/
#define ELEMENTS_MAX(type) (ARENA_SIZE / sizeof(type))
#define WORDS_FOR_TYPE(type) \
(((ELEMENTS_MAX(type) + SIMD_BITS - 1) / SIMD_BITS) * (SIMD_BITS / WORD_BITS))
#define WORDS_FOR_SZ(sz) ((((ARENA_SIZE / sz) + SIMD_BITS - 1) / SIMD_BITS) * 4)
#define SIMD_WORDS_FOR_TYPE(type) \
((ELEMENTS_MAX(type) + SIMD_BITS - 1) / SIMD_BITS)
#define SIMD_MULTIPLIER (sizeof(I256) / sizeof(bitmap_t))
#define WORDS_FOR_TYPE_UNROUNDED(type) \
((ELEMENTS_MAX(type) + WORD_BITS - 1) / WORD_BITS)
#define HIGH_ELEMENTS_OCCUPIED(type) \
((ELEMENTS_MAX(type)) & (WORD_BITS - 1))
#define ELEMENTS_OCCUPIED(hdr, type) \
((sizeof(hdr) + sizeof(type) - 1) / sizeof(type))
#define ELEMENTS_AVAILABLE(atype, otype) (ELEMENTS_MAX(otype) - ELEMENTS_OCCUPIED(atype, otype))
#define FREE_EXTRA_MASK(type) (~0ull >> (WORD_BITS - WORDS_FOR_TYPE(type)))
#define FREE_MASK(type) (~0ull >> (WORD_BITS - WORDS_FOR_TYPE_UNROUNDED(type)))
#define FREE_LOW(atype, type) ~0ull << ELEMENTS_OCCUPIED(atype, type)
/* The else branch of the ternary is incorrect and must be guarded against,
* but it eliminates UB an a warning. It should be resolved at compile time */
#define FREE_HIGH(type) \
(~0ull >> \
HIGH_ELEMENTS_OCCUPIED(type) ? (WORD_BITS - HIGH_ELEMENTS_OCCUPIED(type)) : 1)
#define FREE_HIGH_INDEX(type) (WORDS_FOR_TYPE_UNROUNDED(type) - 1)
#define MAX_BMARRAY_SIZE (ARENA_SIZE / 16 / WORD_BITS)
LJ_STATIC_ASSERT(MAX_BMARRAY_SIZE <= WORD_BITS);
#define arena(p) ((GCAcommon *)((uintptr_t)(p)&ARENA_MASK))
#define gcat(p, t) ((t *)((uintptr_t)(p)&ARENA_MASK))
#define gcablob(p) gcat(p, GCAblob)
#define aobj(base, type, index) ((type *)(base) + (index))
#define objmask(p) ((uintptr_t)(p)&ARENA_OMASK)
/* Any good compiler should be able to turn this divide into a multiply */
#define aidx(p) (objmask(p) / sizeof(*p))
#define aidxl(i) ((i)&63)
#define aidxh(i) ((i) >> 6)
#define abit(i) (1ull << (i))
#define free_enq(a, h) \
do { \
(a)->freenext = h; \
(a)->freeprev = NULL; \
if (h) \
(h)->freeprev = a; \
h = a; \
} while (0)
#define do_arena_init(a, g, id, atype, otype) \
memset(a, 0, sizeof(atype)); \
a->hdr.obj_type = id; \
a->hdr.flags = g->gc.currentsweep; \
a->free_h = FREE_MASK(otype); \
for (uint32_t i = 0; i < WORDS_FOR_TYPE_UNROUNDED(otype); i++) { \
a->free[i] = ~0ull; \
} \
a->free[0] = FREE_LOW(atype, otype); \
if (HIGH_ELEMENTS_OCCUPIED(otype) != 0) \
a->free[FREE_HIGH_INDEX(otype)] = FREE_HIGH(otype)
typedef struct GCAcommon {
GCArenaHdr hdr;
bitmap_t unspecified;
bitmap_t gray_h;
bitmap_t mark[MAX_BMARRAY_SIZE];
bitmap_t gray[MAX_BMARRAY_SIZE];
} GCAcommon;
/* The requirement is that the offset between gray and mark is identical,
* but the extra space is unused so we can stuff the other arrays in there.
* This works because tables are relatively large objects.
*/
typedef struct GCAtab {
GCArenaHdr hdr;
bitmap_t free_h;
bitmap_t gray_h;
bitmap_t mark[WORDS_FOR_TYPE(GCtab)];
bitmap_t free[WORDS_FOR_TYPE(GCtab)];
/*bitmap_t padding[MAX_BMARRAY_SIZE - 4 * WORDS_FOR_TYPE(GCtab)];*/
bitmap_t weak[WORDS_FOR_TYPE(GCtab)];
bitmap_t fin[WORDS_FOR_TYPE(GCtab)];
bitmap_t gray[WORDS_FOR_TYPE(GCtab)];
struct GCAtab *fin_next;
} GCAtab;
LJ_STATIC_ASSERT(sizeof(GCtab) == 64);
typedef struct GCAudata {
GCArenaHdr hdr;
/* This works because we cannot have a bit set in free4 if that same bit is
* cleared in free, so the general case will stop at the set bit in free
* first. */
LJ_ENDIAN_LOHI(uint32_t free_h;, uint32_t free4_h);
bitmap_t gray_h;
bitmap_t mark[WORDS_FOR_TYPE(GCudata)];
bitmap_t free[WORDS_FOR_TYPE(GCudata)];
bitmap_t padding[MAX_BMARRAY_SIZE - 2 * WORDS_FOR_TYPE(GCudata)];
bitmap_t gray[WORDS_FOR_TYPE(GCudata)];
bitmap_t fin[WORDS_FOR_TYPE(GCudata)];
bitmap_t fin_req[WORDS_FOR_TYPE(GCudata)];
} GCAudata;
LJ_STATIC_ASSERT(sizeof(GCudata) >= sizeof(PRNGState));
typedef struct GCAupval {
GCArenaHdr hdr;
bitmap_t free_h;
bitmap_t gray_h;
bitmap_t mark[WORDS_FOR_TYPE(GCupval)];
bitmap_t padding[MAX_BMARRAY_SIZE - WORDS_FOR_TYPE(GCupval)];
bitmap_t gray[WORDS_FOR_TYPE(GCupval)];
bitmap_t free[WORDS_FOR_TYPE(GCupval)];
} GCAupval;
typedef struct GCAfunc {
GCArenaHdr hdr;
bitmap_t free_h;
bitmap_t gray_h;
bitmap_t mark[WORDS_FOR_TYPE(GCfunc)];
bitmap_t free[WORDS_FOR_TYPE(GCfunc)];
/*bitmap_t padding[MAX_BMARRAY_SIZE - 2 * WORDS_FOR_TYPE(GCfunc)];*/
bitmap_t gray[WORDS_FOR_TYPE(GCfunc)];
} GCAfunc;
/* All offsets must match the common arena */
LJ_STATIC_ASSERT(offsetof(GCAtab, gray) == offsetof(GCAcommon, gray));
LJ_STATIC_ASSERT(offsetof(GCAtab, mark) == offsetof(GCAcommon, mark));
LJ_STATIC_ASSERT(offsetof(GCAudata, gray) == offsetof(GCAcommon, gray));
LJ_STATIC_ASSERT(offsetof(GCAudata, mark) == offsetof(GCAcommon, mark));
LJ_STATIC_ASSERT(offsetof(GCAupval, gray) == offsetof(GCAcommon, gray));
LJ_STATIC_ASSERT(offsetof(GCAupval, mark) == offsetof(GCAcommon, mark));
LJ_STATIC_ASSERT(offsetof(GCAfunc, gray) == offsetof(GCAcommon, gray));
LJ_STATIC_ASSERT(offsetof(GCAfunc, mark) == offsetof(GCAcommon, mark));
#if LJ_HASJIT
typedef struct GCAtrace {
GCArenaHdr hdr;
bitmap_t free_h;
bitmap_t gray_h;
bitmap_t mark[WORDS_FOR_TYPE(GCtrace)];
bitmap_t free[WORDS_FOR_TYPE(GCtrace)];
bitmap_t padding[MAX_BMARRAY_SIZE - 2 * WORDS_FOR_TYPE(GCtrace)];
bitmap_t gray[WORDS_FOR_TYPE(GCtrace)];
} GCAtrace;
LJ_STATIC_ASSERT(offsetof(GCAtrace, gray) == offsetof(GCAcommon, gray));
LJ_STATIC_ASSERT(offsetof(GCAtrace, mark) == offsetof(GCAcommon, mark));
#endif
#endif

38
src/lj_intrin.h Normal file
View File

@ -0,0 +1,38 @@
#ifndef _LJ_INTRIN_H_
#define _LJ_INTRIN_H_
#include "lj_arch.h"
#if LUAJIT_TARGET == LUAJIT_ARCH_X64
#if LJ_TARGET_WINDOWS
#include <immintrin.h>
#else
#include <x86intrin.h>
#endif
/* # of contiguous low 0 bits */
#define tzcount32(x) (unsigned)_tzcnt_u32(x)
#define tzcount64(x) (unsigned)_tzcnt_u64(x)
/* x & (x - 1) */
#define reset_lowest32(x) _blsr_u32(x)
#define reset_lowest64(x) _blsr_u64(x)
/* 256 bit SIMD */
#define LJ_SIMD_256 1
#define I256 __m256i
#define I256_ZERO(o) o = _mm256_setzero_si256()
/* vpxor a, a, a; vpcmpeqq a, a, a sets all bits to 1 */
#define I256_ONES(o) o = _mm256_cmpeq_epi64(_mm256_setzero_si256(), _mm256_setzero_si256())
#define I256_EQ_64_MASK(x, y) (uint64_t)_mm256_movemask_pd(_mm256_castsi256_pd(_mm256_cmpeq_epi64(x, y)))
#define I256_AND(o, x, y) o = _mm256_and_si256(x, y)
#define I256_XOR(o, x, y) o = _mm256_xor_si256(x, y)
#define I256_OR(o, x, y) o = _mm256_or_si256(x, y)
#define I256_LOADA(o, ptr) o = _mm256_load_si256((__m256i *)ptr)
#define I256_STOREA(ptr, v) _mm256_store_si256((__m256i *)ptr, v)
#else
#error "No intrinsics defined for arch"
#endif
#endif

2
src/lj_ir.c
View File

@ -280,7 +280,7 @@ TRef lj_ir_kgc(jit_State *J, GCobj *o, IRType t)
{
IRIns *ir, *cir = J->cur.ir;
IRRef ref;
lj_assertJ(!isdead(J2G(J), o), "interning of dead GC object");
lj_assertJ(!checkdead(J2G(J), o), "interning of dead GC object");
for (ref = J->chain[IR_KGC]; ref; ref = cir[ref].prev)
if (ir_kgc(&cir[ref]) == o)
goto found;

3
src/lj_ir.h
View File

@ -204,7 +204,8 @@ IRFPMDEF(FPMENUM)
_(TAB_NOMM, offsetof(GCtab, nomm)) \
_(UDATA_META, offsetof(GCudata, metatable)) \
_(UDATA_UDTYPE, offsetof(GCudata, udtype)) \
_(UDATA_FILE, sizeof(GCudata)) \
_(UDATA_FILE, sizeof(GCudata)) \
_(UDATA_PAYLOAD, offsetof(GCudata, payload)) \
_(SBUF_W, sizeof(GCudata) + offsetof(SBufExt, w)) \
_(SBUF_E, sizeof(GCudata) + offsetof(SBufExt, e)) \
_(SBUF_B, sizeof(GCudata) + offsetof(SBufExt, b)) \

4
src/lj_jit.h
View File

@ -259,11 +259,12 @@ typedef struct GCtrace {
GCHeader;
uint16_t nsnap; /* Number of snapshots. */
IRRef nins; /* Next IR instruction. Biased with REF_BIAS. */
GCRef nextgc;
#if LJ_GC64
uint32_t unused_gc64;
#endif
GCRef gclist;
IRIns *ir; /* IR instructions/constants. Biased with REF_BIAS. */
GCRef gclist;
IRRef nk; /* Lowest IR constant. Biased with REF_BIAS. */
uint32_t nsnapmap; /* Number of snapshot map elements. */
SnapShot *snap; /* Snapshot array. */
@ -298,6 +299,7 @@ typedef struct GCtrace {
check_exp((n)>0 && (MSize)(n)<J->sizetrace, (GCtrace *)gcref(J->trace[(n)]))
LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCtrace, gclist));
LJ_STATIC_ASSERT(offsetof(GChead, nextgc) == offsetof(GCtrace, nextgc));
static LJ_AINLINE MSize snap_nextofs(GCtrace *T, SnapShot *snap)
{

6
src/lj_lib.c
View File

@ -95,7 +95,7 @@ void lj_lib_register(lua_State *L, const char *libname,
GCfunc *fn = lj_func_newC(L, nuv, env);
if (nuv) {
L->top = L->base + tpos;
memcpy(fn->c.upvalue, L->top, sizeof(TValue)*nuv);
memcpy(fn->c.data->upvalue, L->top, sizeof(TValue)*nuv);
}
fn->c.ffid = (uint8_t)(ffid++);
name = (const char *)p;
@ -105,9 +105,9 @@ void lj_lib_register(lua_State *L, const char *libname,
else
setmref(fn->c.pc, bcff++);
if (tag == LIBINIT_ASM_)
fn->c.f = ofn->c.f; /* Copy handler from previous function. */
fn->c.data->f = ofn->c.data->f; /* Copy handler from previous function. */
else
fn->c.f = *cf++; /* Get cf or handler from C function table. */
fn->c.data->f = *cf++; /* Get cf or handler from C function table. */
if (len) {
/* NOBARRIER: See above for common barrier. */
setfuncV(L, lj_tab_setstr(L, tab, lj_str_new(L, name, len)), fn);

2
src/lj_lib.h
View File

@ -55,7 +55,7 @@ LJ_FUNC int32_t lj_lib_checkintrange(lua_State *L, int narg,
/* Avoid including lj_frame.h. */
#if LJ_GC64
#define lj_lib_upvalue(L, n) \
(&gcval(L->base-2)->fn.c.upvalue[(n)-1])
(&gcval(L->base-2)->fn.c.data->upvalue[(n)-1])
#elif LJ_FR2
#define lj_lib_upvalue(L, n) \
(&gcref((L->base-2)->gcr)->fn.c.upvalue[(n)-1])

126
src/lj_obj.h
View File

@ -12,6 +12,7 @@
#include "lua.h"
#include "lj_def.h"
#include "lj_arch.h"
#include "lj_arena.h"
/* -- Memory references --------------------------------------------------- */
@ -60,7 +61,7 @@ typedef struct GCRef {
} GCRef;
/* Common GC header for all collectable objects. */
#define GCHeader GCRef nextgc; uint8_t marked; uint8_t gct
#define GCHeader uint8_t gcflags; uint8_t gct
/* This occupies 6 bytes, so use the next 2 bytes for non-32 bit fields. */
#if LJ_GC64
@ -307,7 +308,8 @@ typedef struct GCstr {
GCHeader;
uint8_t reserved; /* Used by lexer for fast lookup of reserved words. */
uint8_t hashalg; /* Hash algorithm. */
StrID sid; /* Interned string ID. */
GCRef nextgc;
StrID sid; /* Interned string ID. */
StrHash hash; /* Hash of string. */
MSize len; /* Size of string. */
} GCstr;
@ -319,15 +321,16 @@ typedef struct GCstr {
/* -- Userdata object ----------------------------------------------------- */
/* Userdata object. Payload follows. */
/* Userdata object. */
typedef struct GCudata {
GCHeader;
uint8_t udtype; /* Userdata type. */
uint8_t unused2;
GCRef env; /* Should be at same offset in GCfunc. */
MSize len; /* Size of payload. */
MRef payload;
GCRef env; /* Should be at same offset in GCfunc. */
GCRef metatable; /* Must be at same offset in GCtab. */
uint32_t align1; /* To force 8 byte alignment of the payload. */
GCRef gclist;
} GCudata;
/* Userdata types. */
@ -339,7 +342,7 @@ enum {
UDTYPE__MAX
};
#define uddata(u) ((void *)((u)+1))
#define uddata(u) mref((u)->payload, void)
#define sizeudata(u) (sizeof(struct GCudata)+(u)->len)
/* -- C data object ------------------------------------------------------- */
@ -348,6 +351,7 @@ enum {
typedef struct GCcdata {
GCHeader;
uint16_t ctypeid; /* C type ID. */
GCRef nextgc;
} GCcdata;
/* Prepended to variable-sized or realigned C data objects. */
@ -358,7 +362,7 @@ typedef struct GCcdataVar {
} GCcdataVar;
#define cdataptr(cd) ((void *)((cd)+1))
#define cdataisv(cd) ((cd)->marked & 0x80)
#define cdataisv(cd) ((cd)->gcflags & 0x80)
#define cdatav(cd) ((GCcdataVar *)((char *)(cd) - sizeof(GCcdataVar)))
#define cdatavlen(cd) check_exp(cdataisv(cd), cdatav(cd)->len)
#define sizecdatav(cd) (cdatavlen(cd) + cdatav(cd)->extra)
@ -374,11 +378,12 @@ typedef struct GCproto {
uint8_t numparams; /* Number of parameters. */
uint8_t framesize; /* Fixed frame size. */
MSize sizebc; /* Number of bytecode instructions. */
GCRef nextgc;
#if LJ_GC64
uint32_t unused_gc64;
#endif
GCRef gclist;
MRef k; /* Split constant array (points to the middle). */
GCRef gclist;
MRef uv; /* Upvalue list. local slot|0x8000 or parent uv idx. */
MSize sizekgc; /* Number of collectable constants. */
MSize sizekn; /* Number of lua_Number constants. */
@ -433,19 +438,14 @@ typedef struct GCupval {
GCHeader;
uint8_t closed; /* Set if closed (i.e. uv->v == &uv->u.value). */
uint8_t immutable; /* Immutable value. */
uint32_t dhash; /* Disambiguation hash: dh1 != dh2 => cannot alias. */
union {
TValue tv; /* If closed: the value itself. */
struct { /* If open: double linked list, anchored at thread. */
GCRef prev;
GCRef next;
};
GCRef next; /* If open: single linked list, anchored at thread. */
};
MRef v; /* Points to stack slot (open) or above (closed). */
uint32_t dhash; /* Disambiguation hash: dh1 != dh2 => cannot alias. */
} GCupval;
#define uvprev(uv_) (&gcref((uv_)->prev)->uv)
#define uvnext(uv_) (&gcref((uv_)->next)->uv)
#define uvval(uv_) (mref((uv_)->v, TValue))
/* -- Function object (closures) ------------------------------------------ */
@ -453,22 +453,32 @@ typedef struct GCupval {
/* Common header for functions. env should be at same offset in GCudata. */
#define GCfuncHeader \
GCHeader; uint8_t ffid; uint8_t nupvalues; \
GCRef env; GCRef gclist; MRef pc
MRef pc; GCRef env
typedef struct GCfuncCData {
lua_CFunction f; /* C function to be called. */
TValue upvalue[1]; /* Array of upvalues (TValue). */
} GCfuncCData;
typedef struct GCfuncGen {
GCfuncHeader;
MRef data;
} GCfuncGen;
typedef struct GCfuncC {
GCfuncHeader;
lua_CFunction f; /* C function to be called. */
TValue upvalue[1]; /* Array of upvalues (TValue). */
GCfuncCData *data;
} GCfuncC;
typedef struct GCfuncL {
GCfuncHeader;
GCRef uvptr[1]; /* Array of _pointers_ to upvalue objects (GCupval). */
GCRef *uvptr; /* Pointer to array of _pointers_ to upvalue objects (GCupval). */
} GCfuncL;
typedef union GCfunc {
GCfuncC c;
GCfuncL l;
GCfuncGen gen;
} GCfunc;
#define FF_LUA 0
@ -478,8 +488,8 @@ typedef union GCfunc {
#define isffunc(fn) ((fn)->c.ffid > FF_C)
#define funcproto(fn) \
check_exp(isluafunc(fn), (GCproto *)(mref((fn)->l.pc, char)-sizeof(GCproto)))
#define sizeCfunc(n) (sizeof(GCfuncC)-sizeof(TValue)+sizeof(TValue)*(n))
#define sizeLfunc(n) (sizeof(GCfuncL)-sizeof(GCRef)+sizeof(GCRef)*(n))
#define sizeCfunc(n) (sizeof(TValue)+sizeof(TValue)*(n))
#define sizeLfunc(n) (sizeof(GCRef)*(n))
/* -- Table object -------------------------------------------------------- */
@ -498,15 +508,16 @@ LJ_STATIC_ASSERT(offsetof(Node, val) == 0);
typedef struct GCtab {
GCHeader;
uint8_t nomm; /* Negative cache for fast metamethods. */
int8_t colo; /* Array colocation. */
uint8_t colo; /* Array colocation. */
MRef array; /* Array part. */
GCRef gclist;
GCRef metatable; /* Must be at same offset in GCudata. */
MRef node; /* Hash part. */
GCRef metatable; /* Must be at same offset in GCudata. */
GCRef gclist;
uint32_t asize; /* Size of array part (keys [0, asize-1]). */
uint32_t hmask; /* Hash part mask (size of hash part - 1). */
#if LJ_GC64
MRef freetop; /* Top of free elements. */
uint64_t unused;
#endif
} GCtab;
@ -590,7 +601,11 @@ typedef enum {
typedef struct GCState {
GCSize total; /* Memory currently allocated. */
GCSize threshold; /* Memory threshold. */
uint8_t currentwhite; /* Current white color. */
uint8_t currentblack;
uint8_t currentblackgray;
uint8_t safecolor;
uint8_t currentsweep;
uint8_t gcmode;
uint8_t state; /* GC state. */
uint8_t nocdatafin; /* No cdata finalizer called. */
#if LJ_64
@ -602,16 +617,52 @@ typedef struct GCState {
GCRef root; /* List of all collectable objects. */
MRef sweep; /* Sweep position in root list. */
GCRef gray; /* List of gray objects. */
GCRef grayagain; /* List of objects for atomic traversal. */
GCRef grayagain; /* List of tables for atomic traversal. */
GCRef grayagain_th; /* List of threads for atomic traversal. */
GCRef weak; /* List of weak tables (to be cleared). */
GCRef ephemeron;
GCRef mmudata; /* List of userdata (to be finalized). */
GCRef fin_list;
GCSize debt; /* Debt (how much GC is behind schedule). */
GCSize estimate; /* Estimate of memory actually in use. */
GCSize accum; /* Accumulated memory marked. */
GCSize malloc; /* Malloced memory. */
MSize stepmul; /* Incremental GC step granularity. */
MSize pause; /* Pause between successive GC cycles. */
#if LJ_64
MRef lightudseg; /* Upper bits of lightuserdata segments. */
#endif
/* New GC */
arena_context ctx;
/* Arenas to allocate from. Never NULL. Chains with prev/next */
GCArenaHdr *tab;
GCArenaHdr *fintab;
GCArenaHdr *uv;
GCArenaHdr *func;
GCArenaHdr *udata;
/* This is the allocated-from blob arena. Never NULL */
GCAblob *blob_generic;
GCAblob **bloblist;
uint32_t *bloblist_usage;
uint32_t bloblist_alloc;
uint32_t bloblist_wr;
uint32_t bloblist_sweep;
/* These are the arenas that have at least one gray object, may be NULL.
* Chains with 'gray' */
GCArenaHdr *gray_head;
GCArenaHdr *gray_tail;
/* Freelist, may be NULL. Contains at least one free object. Chains with
* 'freenext' & 'freeprev' */
GCArenaHdr *free_tab;
GCArenaHdr *free_fintab;
GCArenaHdr *free_uv;
GCArenaHdr *free_func;
GCArenaHdr *free_udata;
} GCState;
/* String interning state. */
@ -644,7 +695,6 @@ typedef struct global_State {
TValue tmptv, tmptv2; /* Temporary TValues. */
Node nilnode; /* Fallback 1-element hash part (nil key and value). */
TValue registrytv; /* Anchor for registry. */
GCupval uvhead; /* Head of double-linked list of all open upvalues. */
int32_t hookcount; /* Instruction hook countdown. */
int32_t hookcstart; /* Start count for instruction hook counter. */
lua_Hook hookf; /* Hook function. */
@ -687,9 +737,10 @@ struct lua_State {
GCHeader;
uint8_t dummy_ffid; /* Fake FF_C for curr_funcisL() on dummy frames. */
uint8_t status; /* Thread status. */
MRef glref; /* Link to global state. */
GCRef gclist; /* GC chain. */
GCRef nextgc;
MRef glref; /* Link to global state. */
TValue *base; /* Base of currently executing function. */
GCRef gclist; /* GC chain. */
TValue *top; /* First free slot in the stack. */
MRef maxstack; /* Last free slot in the stack. */
MRef stack; /* Stack base. */
@ -727,9 +778,10 @@ typedef struct GChead {
GCHeader;
uint8_t unused1;
uint8_t unused2;
GCRef nextgc;
GCRef env;
GCRef gclist;
GCRef metatable;
GCRef gclist;
} GChead;
/* The env field SHOULD be at the same offset for all GC objects. */
@ -742,9 +794,14 @@ LJ_STATIC_ASSERT(offsetof(GChead, metatable) == offsetof(GCudata, metatable));
/* The gclist field MUST be at the same offset for all GC objects. */
LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(lua_State, gclist));
LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCproto, gclist));
LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCfuncL, gclist));
LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCtab, gclist));
LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCproto, gclist));
/* The nextgc field MUST be at the same offset for all LL GC objects. */
LJ_STATIC_ASSERT(offsetof(GChead, nextgc) == offsetof(lua_State, nextgc));
LJ_STATIC_ASSERT(offsetof(GChead, nextgc) == offsetof(GCproto, nextgc));
LJ_STATIC_ASSERT(offsetof(GChead, nextgc) == offsetof(GCcdata, nextgc));
LJ_STATIC_ASSERT(offsetof(GChead, nextgc) == offsetof(GCstr, nextgc));
typedef union GCobj {
GChead gch;
@ -897,6 +954,9 @@ static LJ_AINLINE void setrawlightudV(TValue *o, void *p)
((o)->u64 = (uint64_t)(void *)(f) - (uint64_t)lj_vm_asm_begin)
#endif
/* Copy from lj_gc.h */
LJ_FUNC int checkdead(global_State *g, GCobj *o);
static LJ_AINLINE void checklivetv(lua_State *L, TValue *o, const char *msg)
{
UNUSED(L); UNUSED(o); UNUSED(msg);
@ -906,7 +966,7 @@ static LJ_AINLINE void checklivetv(lua_State *L, TValue *o, const char *msg)
"mismatch of TValue type %d vs GC type %d",
~itype(o), gcval(o)->gch.gct);
/* Copy of isdead check from lj_gc.h to avoid circular include. */
lj_assertL(!(gcval(o)->gch.marked & (G(L)->gc.currentwhite ^ 3) & 3), msg);
lj_assertL(!checkdead(G(L), gcval(o)), msg);
}
#endif
}

2
src/lj_record.c
View File

@ -1727,7 +1727,7 @@ static int rec_upvalue_constify(jit_State *J, GCupval *uvp)
#if LJ_HASFFI
if (tviscdata(o)) {
GCcdata *cd = cdataV(o);
if (!cdataisv(cd) && !(cd->marked & LJ_GC_CDATA_FIN)) {
if (!cdataisv(cd) && !(cd->gcflags & LJ_GC_CDATA_FIN)) {
CType *ct = ctype_raw(ctype_ctsG(J2G(J)), cd->ctypeid);
if (!ctype_hassize(ct->info) || ct->size <= 16)
return 1;

2
src/lj_snap.c
View File

@ -224,7 +224,7 @@ static BCReg snap_usedef(jit_State *J, uint8_t *udf,
while (o) {
if (uvval(gco2uv(o)) < J->L->base) break;
udf[uvval(gco2uv(o)) - J->L->base] = 0;
o = gcref(o->gch.nextgc);
o = gcref(o->uv.next);
}
#define USE_SLOT(s) udf[(s)] &= ~1

52
src/lj_state.c
View File

@ -28,6 +28,7 @@
#include "lj_prng.h"
#include "lj_lex.h"
#include "lj_alloc.h"
#include "lj_arena.h"
#include "luajit.h"
/* -- Stack handling ------------------------------------------------------ */
@ -76,7 +77,7 @@ static void resizestack(lua_State *L, MSize n)
setmref(G(L)->jit_base, mref(G(L)->jit_base, char) + delta);
L->base = (TValue *)((char *)L->base + delta);
L->top = (TValue *)((char *)L->top + delta);
for (up = gcref(L->openupval); up != NULL; up = gcnext(up))
for (up = gcref(L->openupval); up != NULL; up = gcref(up->uv.next))
setmref(gco2uv(up)->v, (TValue *)((char *)uvval(gco2uv(up)) + delta));
}
@ -200,6 +201,11 @@ static void close_state(lua_State *L)
lj_mem_freevec(g, mref(g->gc.lightudseg, uint32_t), segnum, uint32_t);
}
#endif
lj_arena_cleanup(g);
lj_assertG(g->gc.ctx.mem_commit == 0, "memory leak of %u arenas",
g->gc.ctx.mem_commit);
lj_assertG(g->gc.ctx.mem_huge == 0, "memory leak of %llu huge arena bytes",
g->gc.ctx.mem_huge);
lj_assertG(g->gc.total == sizeof(GG_State),
"memory leak of %lld bytes",
(long long)(g->gc.total - sizeof(GG_State)));
@ -215,7 +221,16 @@ static void close_state(lua_State *L)
lua_State *lj_state_newstate(lua_Alloc allocf, void *allocd)
#else
LUA_API lua_State *lua_newstate(lua_Alloc allocf, void *allocd)
{
return lj_newstate(allocf, allocd, NULL, NULL, NULL, NULL);
}
#endif
lua_State *lj_newstate(lua_Alloc allocf, void *allocd,
luaJIT_allocpages allocp,
luaJIT_freepages freep,
luaJIT_reallochuge realloch,
void *page_ud)
{
PRNGState prng;
GG_State *GG;
@ -239,15 +254,22 @@ LUA_API lua_State *lua_newstate(lua_Alloc allocf, void *allocd)
memset(GG, 0, sizeof(GG_State));
L = &GG->L;
g = &GG->g;
L->gct = ~LJ_TTHREAD;
L->marked = LJ_GC_WHITE0 | LJ_GC_FIXED | LJ_GC_SFIXED; /* Prevent free. */
L->dummy_ffid = FF_C;
setmref(L->glref, g);
g->gc.currentwhite = LJ_GC_WHITE0 | LJ_GC_FIXED;
g->strempty.marked = LJ_GC_WHITE0;
g->strempty.gct = ~LJ_TSTR;
g->allocf = allocf;
g->allocd = allocd;
g->gc.currentsweep = LJ_GC_SWEEP0;
if (!lj_arena_init(g, allocp, freep, realloch, page_ud)) {
close_state(L);
return NULL;
}
L->gct = ~LJ_TTHREAD;
L->gcflags = LJ_GC_FIXED | LJ_GC_SFIXED; /* Prevent free. */
L->dummy_ffid = FF_C;
setmref(L->glref, g);
g->gc.currentblack = LJ_GC_BLACK0;
g->gc.currentblackgray = LJ_GC_BLACK0 | LJ_GC_GRAY;
g->gc.safecolor = 0;
g->strempty.gcflags = 0;
g->strempty.gct = ~LJ_TSTR;
g->prng = prng;
#ifndef LUAJIT_USE_SYSMALLOC
if (allocf == lj_alloc_f) {
@ -255,8 +277,6 @@ LUA_API lua_State *lua_newstate(lua_Alloc allocf, void *allocd)
}
#endif
setgcref(g->mainthref, obj2gco(L));
setgcref(g->uvhead.prev, obj2gco(&g->uvhead));
setgcref(g->uvhead.next, obj2gco(&g->uvhead));
g->str.mask = ~(MSize)0;
setnilV(registry(L));
setnilV(&g->nilnode.val);
@ -269,9 +289,10 @@ LUA_API lua_State *lua_newstate(lua_Alloc allocf, void *allocd)
setgcref(g->gc.root, obj2gco(L));
setmref(g->gc.sweep, &g->gc.root);
g->gc.total = sizeof(GG_State);
g->gc.malloc = g->gc.total;
g->gc.pause = LUAI_GCPAUSE;
g->gc.stepmul = LUAI_GCMUL;
lj_dispatch_init((GG_State *)L);
lj_dispatch_init(GG);
L->status = LUA_ERRERR+1; /* Avoid touching the stack upon memory error. */
if (lj_vm_cpcall(L, NULL, NULL, cpluaopen) != 0) {
/* Memory allocation error: free partial state. */
@ -302,7 +323,7 @@ LUA_API void lua_close(lua_State *L)
#endif
setgcrefnull(g->cur_L);
lj_func_closeuv(L, tvref(L->stack));
lj_gc_separateudata(g, 1); /* Separate udata which have GC metamethods. */
lj_gc_separateudata(g); /* Separate udata which have GC metamethods. */
#if LJ_HASJIT
G2J(g)->flags &= ~JIT_F_ON;
G2J(g)->state = LJ_TRACE_IDLE;
@ -315,8 +336,9 @@ LUA_API void lua_close(lua_State *L)
L->cframe = NULL;
if (lj_vm_cpcall(L, NULL, NULL, cpfinalize) == LUA_OK) {
if (++i >= 10) break;
lj_gc_separateudata(g, 1); /* Separate udata again. */
if (gcref(g->gc.mmudata) == NULL) /* Until nothing is left to do. */
lj_gc_separateudata(g); /* Separate udata again. */
if (gcref(g->gc.mmudata) == NULL && gcref(g->gc.fin_list) == NULL)
/* Until nothing is left to do. */
break;
}
}
@ -337,7 +359,7 @@ lua_State *lj_state_new(lua_State *L)
setmrefr(L1->glref, L->glref);
setgcrefr(L1->env, L->env);
stack_init(L1, L); /* init stack */
lj_assertL(iswhite(obj2gco(L1)), "new thread object is not white");
lj_assertL(iswhite(G(L), obj2gco(L1)), "new thread object is not white");
return L1;
}

7
src/lj_state.h
View File

@ -35,4 +35,11 @@ LJ_FUNC lua_State *lj_state_newstate(lua_Alloc f, void *ud);
#define LJ_ALLOCF_INTERNAL ((lua_Alloc)(void *)(uintptr_t)(1237<<4))
LJ_FUNC lua_State *lj_newstate(lua_Alloc f, void *ud,
luaJIT_allocpages allocp,
luaJIT_freepages freep,
luaJIT_reallochuge realloch,
void *page_ud);
#endif

20
src/lj_str.c
View File

@ -220,7 +220,9 @@ static LJ_NOINLINE GCstr *lj_str_rehash_chain(lua_State *L, StrHash hashc,
const char *str, MSize len)
{
global_State *g = G(L);
int ow = g->gc.state == GCSsweepstring ? otherwhite(g) : 0; /* Sweeping? */
int sweep =
g->gc.state == GCSsweepstring ? g->gc.safecolor : 0; /* Sweeping? */
uint8_t mask = isminor(g) ? 0xFF : ~LJ_GC_COLORS;
GCRef *strtab = g->str.tab;
MSize strmask = g->str.mask;
GCobj *o = gcref(strtab[hashc & strmask]);
@ -231,13 +233,13 @@ static LJ_NOINLINE GCstr *lj_str_rehash_chain(lua_State *L, StrHash hashc,
GCobj *next = gcnext(o);
GCstr *s = gco2str(o);
StrHash hash;
if (ow) { /* Must sweep while rechaining. */
if (((o->gch.marked ^ LJ_GC_WHITES) & ow)) { /* String alive? */
lj_assertG(!isdead(g, o) || (o->gch.marked & LJ_GC_FIXED),
if (sweep) { /* Must sweep while rechaining. */
if (o->gch.gcflags & sweep) { /* String alive? */
lj_assertG(!checkdead(g, o) || (o->gch.gcflags & LJ_GC_FIXED),
"sweep of undead string");
makewhite(g, o);
o->gch.gcflags &= mask;
} else { /* Free dead string. */
lj_assertG(isdead(g, o) || ow == LJ_GC_SFIXED,
lj_assertG(checkdead(g, o) || (sweep & LJ_GC_SFIXED),
"sweep of unlive string");
lj_str_free(g, s);
o = next;
@ -275,10 +277,10 @@ static LJ_NOINLINE GCstr *lj_str_rehash_chain(lua_State *L, StrHash hashc,
static GCstr *lj_str_alloc(lua_State *L, const char *str, MSize len,
StrHash hash, int hashalg)
{
GCstr *s = lj_mem_newt(L, lj_str_size(len), GCstr);
GCstr *s = lj_mem_allocstr(L, len);
global_State *g = G(L);
uintptr_t u;
newwhite(g, s);
newwhite(s);
s->gct = ~LJ_TSTR;
s->len = len;
s->hash = hash;
@ -332,7 +334,7 @@ GCstr *lj_str_new(lua_State *L, const char *str, size_t lenx)
GCstr *sx = gco2str(o);
if (sx->hash == hash && sx->len == len) {
if (memcmp(str, strdata(sx), len) == 0) {
if (isdead(g, o)) flipwhite(o); /* Resurrect if dead. */
maybe_resurrect_str(g, sx);
return sx; /* Return existing string. */
}
coll++;

111
src/lj_tab.c
View File

@ -42,7 +42,7 @@ static LJ_AINLINE void newhpart(lua_State *L, GCtab *t, uint32_t hbits)
if (hbits > LJ_MAX_HBITS)
lj_err_msg(L, LJ_ERR_TABOV);
hsize = 1u << hbits;
node = lj_mem_newvec(L, hsize, Node);
node = lj_mem_newv(L, hsize, Node);
setmref(t->node, node);
setfreetop(t, node, &node[hsize]);
t->hmask = hsize-1;
@ -80,51 +80,40 @@ static LJ_AINLINE void clearapart(GCtab *t)
/* Create a new table. Note: the slots are not initialized (yet). */
static GCtab *newtab(lua_State *L, uint32_t asize, uint32_t hbits)
{
GCtab *t;
/* First try to colocate the array part. */
if (LJ_MAX_COLOSIZE != 0 && asize > 0 && asize <= LJ_MAX_COLOSIZE) {
Node *nilnode;
lj_assertL((sizeof(GCtab) & 7) == 0, "bad GCtab size");
t = (GCtab *)lj_mem_newgco(L, sizetabcolo(asize));
t->gct = ~LJ_TTAB;
t->nomm = (uint8_t)~0;
t->colo = (int8_t)asize;
setmref(t->array, (TValue *)((char *)t + sizeof(GCtab)));
setgcrefnull(t->metatable);
GCtab *t = lj_mem_alloctab(L, asize);
Node *nilnode = &G(L)->nilnode;
setmref(t->node, nilnode);
#if LJ_GC64
setmref(t->freetop, nilnode);
#endif
if (hbits)
newhpart(L, t, hbits);
return t;
}
GCtab *lj_tab_newgc(lua_State *L, uint32_t asize, uint32_t hbits)
{
Node *nilnode;
GCtab *t = lj_mem_alloctabempty_gc(L);
t->hmask = 0;
nilnode = &G(L)->nilnode;
setmref(t->node, nilnode);
#if LJ_GC64
setmref(t->freetop, nilnode);
#endif
if (asize > 0) {
if (asize > LJ_MAX_ASIZE)
lj_err_msg(L, LJ_ERR_TABOV);
setmref(t->array, lj_mem_newv(L, asize, TValue));
t->asize = asize;
t->hmask = 0;
nilnode = &G(L)->nilnode;
setmref(t->node, nilnode);
#if LJ_GC64
setmref(t->freetop, nilnode);
#endif
} else { /* Otherwise separately allocate the array part. */
Node *nilnode;
t = lj_mem_newobj(L, GCtab);
t->gct = ~LJ_TTAB;
t->nomm = (uint8_t)~0;
t->colo = 0;
setmref(t->array, NULL);
setgcrefnull(t->metatable);
t->asize = 0; /* In case the array allocation fails. */
t->hmask = 0;
nilnode = &G(L)->nilnode;
setmref(t->node, nilnode);
#if LJ_GC64
setmref(t->freetop, nilnode);
#endif
if (asize > 0) {
if (asize > LJ_MAX_ASIZE)
lj_err_msg(L, LJ_ERR_TABOV);
setmref(t->array, lj_mem_newvec(L, asize, TValue));
t->asize = asize;
}
}
if (hbits)
newhpart(L, t, hbits);
return t;
}
/* Create a new table.
**
** IMPORTANT NOTE: The API differs from lua_createtable()!
@ -211,19 +200,6 @@ void LJ_FASTCALL lj_tab_clear(GCtab *t)
}
}
/* Free a table. */
void LJ_FASTCALL lj_tab_free(global_State *g, GCtab *t)
{
if (t->hmask > 0)
lj_mem_freevec(g, noderef(t->node), t->hmask+1, Node);
if (t->asize > 0 && LJ_MAX_COLOSIZE != 0 && t->colo <= 0)
lj_mem_freevec(g, tvref(t->array), t->asize, TValue);
if (LJ_MAX_COLOSIZE != 0 && t->colo)
lj_mem_free(g, t, sizetabcolo((uint32_t)t->colo & 0x7f));
else
lj_mem_freet(g, t);
}
/* -- Table resizing ------------------------------------------------------ */
/* Resize a table to fit the new array/hash part sizes. */
@ -237,26 +213,34 @@ void lj_tab_resize(lua_State *L, GCtab *t, uint32_t asize, uint32_t hbits)
uint32_t i;
if (asize > LJ_MAX_ASIZE)
lj_err_msg(L, LJ_ERR_TABOV);
if (LJ_MAX_COLOSIZE != 0 && t->colo > 0) {
if (LJ_COLO_ENABLED && t->colo > 0) {
/* A colocated array must be separated and copied. */
TValue *oarray = tvref(t->array);
array = lj_mem_newvec(L, asize, TValue);
t->colo = (int8_t)(t->colo | 0x80); /* Mark as separated (colo < 0). */
array = lj_mem_newv(L, asize, TValue);
t->colo = 0; /* Mark as separated (colo == 0). */
t->gcflags &= (uint8_t)~LJ_GC_MARK_MASK; /* Release the slots. */
for (i = 0; i < oldasize; i++)
copyTV(L, &array[i], &oarray[i]);
} else {
array = (TValue *)lj_mem_realloc(L, tvref(t->array),
oldasize*sizeof(TValue), asize*sizeof(TValue));
array = lj_mem_reallocv(L, tvref(t->array), oldasize, asize, TValue);
}
setmref(t->array, array);
t->asize = asize;
for (i = oldasize; i < asize; i++) /* Clear newly allocated slots. */
setnilV(&array[i]);
if (isblack(G(L), obj2gco(t))) {
/* If this is black then the array has been marked,
* so the new array needs marking too */
lj_gc_markblob(L, mref(t->array, void), asize * sizeof(TValue));
}
}
/* Create new (empty) hash part. */
if (hbits) {
newhpart(L, t, hbits);
clearhpart(t);
if (isblack(G(L), obj2gco(t))) {
lj_gc_markblob(L, mref(t->node, void), (t->hmask+1) * sizeof(Node));
}
} else {
global_State *g = G(L);
setmref(t->node, &g->nilnode);
@ -273,20 +257,21 @@ void lj_tab_resize(lua_State *L, GCtab *t, uint32_t asize, uint32_t hbits)
if (!tvisnil(&array[i]))
copyTV(L, lj_tab_setinth(L, t, (int32_t)i), &array[i]);
/* Physically shrink only separated arrays. */
if (LJ_MAX_COLOSIZE != 0 && t->colo <= 0)
setmref(t->array, lj_mem_realloc(L, array,
oldasize*sizeof(TValue), asize*sizeof(TValue)));
if (!LJ_COLO_ENABLED || t->colo == 0) {
setmref(t->array,
lj_mem_reallocv(L, tvref(t->array), oldasize, asize, TValue));
if (isblack(G(L), obj2gco(t))) {
lj_gc_markblob(L, mref(t->array, void), asize * sizeof(TValue));
}
}
}
if (oldhmask > 0) { /* Reinsert pairs from old hash part. */
global_State *g;
uint32_t i;
for (i = 0; i <= oldhmask; i++) {
Node *n = &oldnode[i];
if (!tvisnil(&n->val))
copyTV(L, lj_tab_set(L, t, &n->key), &n->val);
}
g = G(L);
lj_mem_freevec(g, oldnode, oldhmask+1, Node);
}
}

2
src/lj_tab.h
View File

@ -53,13 +53,13 @@ static LJ_AINLINE Node *hashmask(const GCtab *t, uint32_t hash)
#define hsize2hbits(s) ((s) ? ((s)==1 ? 1 : 1+lj_fls((uint32_t)((s)-1))) : 0)
LJ_FUNCA GCtab *lj_tab_new(lua_State *L, uint32_t asize, uint32_t hbits);
LJ_FUNCA GCtab *lj_tab_newgc(lua_State *L, uint32_t asize, uint32_t hbits);
LJ_FUNC GCtab *lj_tab_new_ah(lua_State *L, int32_t a, int32_t h);
#if LJ_HASJIT
LJ_FUNC GCtab * LJ_FASTCALL lj_tab_new1(lua_State *L, uint32_t ahsize);
#endif
LJ_FUNCA GCtab * LJ_FASTCALL lj_tab_dup(lua_State *L, const GCtab *kt);
LJ_FUNC void LJ_FASTCALL lj_tab_clear(GCtab *t);
LJ_FUNC void LJ_FASTCALL lj_tab_free(global_State *g, GCtab *t);
#if LJ_HASFFI
LJ_FUNC void lj_tab_rehash(lua_State *L, GCtab *t);
#endif

1
src/lj_target_x86.h
View File

@ -261,6 +261,7 @@ typedef enum {
XO_IMUL = XO_0f(af),
XO_IMULi = XO_(69),
XO_IMULi8 = XO_(6b),
XO_CMPb = XO_(38),
XO_CMP = XO_(3b),
XO_TESTb = XO_(84),
XO_TEST = XO_(85),

4
src/lj_trace.c
View File

@ -130,7 +130,7 @@ GCtrace * LJ_FASTCALL lj_trace_alloc(lua_State *L, GCtrace *T)
GCtrace *T2 = lj_mem_newt(L, (MSize)sz, GCtrace);
char *p = (char *)T2 + sztr;
T2->gct = ~LJ_TTRACE;
T2->marked = 0;
T2->gcflags = 0;
T2->traceno = 0;
T2->ir = (IRIns *)p - T->nk;
T2->nins = T->nins;
@ -150,7 +150,7 @@ static void trace_save(jit_State *J, GCtrace *T)
memcpy(T, &J->cur, sizeof(GCtrace));
setgcrefr(T->nextgc, J2G(J)->gc.root);
setgcrefp(J2G(J)->gc.root, T);
newwhite(J2G(J), T);
newwhite(T);
T->gct = ~LJ_TTRACE;
T->ir = (IRIns *)p - J->cur.nk; /* The IR has already been copied above. */
#if LJ_ABI_PAUTH

13
src/lj_udata.c
View File

@ -13,26 +13,15 @@
GCudata *lj_udata_new(lua_State *L, MSize sz, GCtab *env)
{
GCudata *ud = lj_mem_newt(L, sizeof(GCudata) + sz, GCudata);
global_State *g = G(L);
newwhite(g, ud); /* Not finalized. */
ud->gct = ~LJ_TUDATA;
GCudata *ud = lj_mem_allocudata(L, sz);
ud->udtype = UDTYPE_USERDATA;
ud->len = sz;
/* NOBARRIER: The GCudata is new (marked white). */
setgcrefnull(ud->metatable);
setgcref(ud->env, obj2gco(env));
/* Chain to userdata list (after main thread). */
setgcrefr(ud->nextgc, mainthread(g)->nextgc);
setgcref(mainthread(g)->nextgc, obj2gco(ud));
return ud;
}
void LJ_FASTCALL lj_udata_free(global_State *g, GCudata *ud)
{
lj_mem_free(g, ud, sizeudata(ud));
}
#if LJ_64
void *lj_lightud_intern(lua_State *L, void *p)
{

1
src/lj_udata.h
View File

@ -9,7 +9,6 @@
#include "lj_obj.h"
LJ_FUNC GCudata *lj_udata_new(lua_State *L, MSize sz, GCtab *env);
LJ_FUNC void LJ_FASTCALL lj_udata_free(global_State *g, GCudata *ud);
#if LJ_64
LJ_FUNC void * LJ_FASTCALL lj_lightud_intern(lua_State *L, void *p);
#endif

32
src/luajit_rolling.h
View File

@ -73,6 +73,38 @@ LUA_API void luaJIT_profile_stop(lua_State *L);
LUA_API const char *luaJIT_profile_dumpstack(lua_State *L, const char *fmt,
int depth, size_t *len);
typedef unsigned (*luaJIT_allocpages)(void *ud, void **pages, unsigned n);
/* Free is called one last time with NULL, 0 to indicate any state should be released */
typedef void (*luaJIT_freepages)(void *ud, void **pages, unsigned n);
typedef void* (*luaJIT_reallochuge)(void *ud, void *p, size_t osz, size_t nsz);
/* This many bytes are reserved at the start of each huge arena for the allocator's use */
#define LUAJIT_HUGE_RESERVED_SPACE 20
LUA_API size_t luaJIT_getpagesize();
/* Custom allocator support.
* Page allocators must yield naturally aligned regions in the size provided by
* luaJIT_getpagesize. Page allocators allocate and free N pages at once. Huge
* allocations are one at a time.
* The lua_Alloc allocator is used for allocations that require variable size
* and a fixed address.
* Allocators are permitted to fail. The huge page allocator can return NULL and
* the page allocator can return fewer than the requested amount. Allocators are not
* permitted to throw.
* Huge allocations have the same alignment requirements as normal arenas.
* Either the normal or page allocation can be NULL and the default will be used.
* Passing all NULL is functionally the same as luaL_newstate()
*/
LUA_API lua_State *luaJIT_newstate(lua_Alloc f, void *ud,
luaJIT_allocpages allocp,
luaJIT_freepages freep,
luaJIT_reallochuge realloch,
void *page_ud);
/* As lua_createtable, but can be used with __gc */
LUA_API void luaJIT_createtable(lua_State *L, int narray, int nrec);
/* Enforce (dynamic) linker error for version mismatches. Call from main. */
LUA_API void LUAJIT_VERSION_SYM(void);

88
src/vm_x64.dasc
View File

@ -53,8 +53,10 @@
|.define RDL, RCL
|.define TMPR, r10
|.define TMPRd, r10d
|.define TMPRb, r10b
|.define ITYPE, r11
|.define ITYPEd, r11d
|.define ITYPEb, r11b
|
|.if X64WIN
|.define CARG1, rcx // x64/WIN64 C call arguments.
@ -89,6 +91,7 @@
|.type TAB, GCtab
|.type LFUNC, GCfuncL
|.type CFUNC, GCfuncC
|.type CFUNCD, GCfuncCData
|.type PROTO, GCproto
|.type UPVAL, GCupval
|.type NODE, Node
@ -368,12 +371,26 @@
|
|// Move table write barrier back. Overwrites reg.
|.macro barrierback, tab, reg
| and byte tab->marked, (uint8_t)~LJ_GC_BLACK // black2gray(tab)
| or byte tab->gcflags, (uint8_t)LJ_GC_GRAY // black2gray(tab)
| mov reg, [DISPATCH+DISPATCH_GL(gc.grayagain)]
| mov [DISPATCH+DISPATCH_GL(gc.grayagain)], tab
| mov tab->gclist, reg
|.endmacro
|
|// Generic isblack(o) macro
|.macro isblack, rD, rB, o, j
| movzx rD, byte o->gcflags
| and rD, LJ_GC_COLORS
| cmp rB, byte [DISPATCH+DISPATCH_GL(gc.currentblack)]
| jz j
|.endmacro
|.macro isnblack, rD, rB, o, j
| movzx rD, byte o->gcflags
| and rD, LJ_GC_COLORS
| cmp rB, byte [DISPATCH+DISPATCH_GL(gc.currentblack)]
| jnz j
|.endmacro
|
|//-----------------------------------------------------------------------
/* Generate subroutines used by opcodes and other parts of the VM. */
@ -1205,7 +1222,8 @@ static void build_subroutines(BuildCtx *ctx)
|2:
| mov CFUNC:RB, [BASE-16]
| cleartp CFUNC:RB
| mov STR:RC, [CFUNC:RB+RC*8+((char *)(&((GCfuncC *)0)->upvalue))]
| mov PC, CFUNC:RB->data
| mov STR:RC, [PC+RC*8+((char *)(&((GCfuncCData *)0)->upvalue))]
| mov PC, [BASE-8]
| settp STR:RC, LJ_TSTR
| mov [BASE-16], STR:RC
@ -1265,8 +1283,7 @@ static void build_subroutines(BuildCtx *ctx)
| mov TAB:RB->metatable, TAB:RA
| mov PC, [BASE-8]
| mov [BASE-16], TAB:TMPR // Return original table.
| test byte TAB:RB->marked, LJ_GC_BLACK // isblack(table)
| jz >1
| isnblack RCd, RCL, TAB:RB, >1
| // Possible write barrier. Table is black, but skip iswhite(mt) check.
| barrierback TAB:RB, RC
|1:
@ -1375,7 +1392,8 @@ static void build_subroutines(BuildCtx *ctx)
#endif
| mov CFUNC:RD, [BASE-16]
| cleartp CFUNC:RD
| mov CFUNC:RD, CFUNC:RD->upvalue[0]
| mov CFUNCD:RD, CFUNC:RD->data
| mov CFUNC:RD, CFUNCD:RD->upvalue[0]
| settp CFUNC:RD, LJ_TFUNC
| mov PC, [BASE-8]
| mov [BASE-16], CFUNC:RD
@ -1446,7 +1464,8 @@ static void build_subroutines(BuildCtx *ctx)
#endif
| mov CFUNC:RD, [BASE-16]
| cleartp CFUNC:RD
| mov CFUNC:RD, CFUNC:RD->upvalue[0]
| mov CFUNCD:RD, CFUNC:RD->data
| mov CFUNC:RD, CFUNCD:RD->upvalue[0]
| settp CFUNC:RD, LJ_TFUNC
| mov PC, [BASE-8]
| mov [BASE-16], CFUNC:RD
@ -1508,7 +1527,8 @@ static void build_subroutines(BuildCtx *ctx)
|.ffunc coroutine_wrap_aux
| mov CFUNC:RB, [BASE-16]
| cleartp CFUNC:RB
| mov L:RB, CFUNC:RB->upvalue[0].gcr
| mov CFUNCD:RB, CFUNC:RB->data
| mov L:RB, CFUNCD:RB->upvalue[0].gcr
| cleartp L:RB
|.endif
| mov PC, [BASE-8]
@ -2143,7 +2163,8 @@ static void build_subroutines(BuildCtx *ctx)
| cmp RA, L:RB->maxstack
| ja >5 // Need to grow stack.
| mov CARG1, L:RB
| call aword CFUNC:RD->f // (lua_State *L)
| mov CFUNCD:RD, CFUNC:RD->data
| call aword CFUNCD:RD->f // (lua_State *L)
| mov BASE, L:RB->base
| // Either throws an error, or recovers and returns -1, 0 or nresults+1.
| test RDd, RDd; jg ->fff_res // Returned nresults+1?
@ -3445,7 +3466,8 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
| ins_AD // RA = dst, RD = upvalue #
| mov LFUNC:RB, [BASE-16]
| cleartp LFUNC:RB
| mov UPVAL:RB, [LFUNC:RB+RD*8+offsetof(GCfuncL, uvptr)]
| mov RB, LFUNC:RB->uvptr
| mov UPVAL:RB, [RB+RD*8]
| mov RB, UPVAL:RB->v
| mov RD, [RB]
| mov [BASE+RA*8], RD
@ -3453,18 +3475,20 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
break;
case BC_USETV:
#define TV2MARKOFS \
((int32_t)offsetof(GCupval, marked)-(int32_t)offsetof(GCupval, tv))
((int32_t)offsetof(GCupval, gcflags)-(int32_t)offsetof(GCupval, tv))
| ins_AD // RA = upvalue #, RD = src
| mov LFUNC:RB, [BASE-16]
| cleartp LFUNC:RB
| mov UPVAL:RB, [LFUNC:RB+RA*8+offsetof(GCfuncL, uvptr)]
| mov UPVAL:RB, LFUNC:RB->uvptr
| mov UPVAL:RB, [RB+RA*8]
| cmp byte UPVAL:RB->closed, 0
| mov RB, UPVAL:RB->v
| mov RA, [BASE+RD*8]
| mov [RB], RA
| jz >1
| // Check barrier for closed upvalue.
| test byte [RB+TV2MARKOFS], LJ_GC_BLACK // isblack(uv)
| movzx RDd, byte [DISPATCH+DISPATCH_GL(gc.currentblack)]
| test byte [RB+TV2MARKOFS], RDL // isblack(uv)
| jnz >2
|1:
| ins_next
@ -3476,8 +3500,9 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
| cmp RDd, LJ_TNUMX - LJ_TISGCV // tvisgcv(v)
| jbe <1
| cleartp GCOBJ:RA
| test byte GCOBJ:RA->gch.marked, LJ_GC_WHITES // iswhite(v)
| jz <1
| movzx TMPRd, byte [DISPATCH+DISPATCH_GL(gc.currentblackgray)]
| test byte GCOBJ:RA->gch.gcflags, TMPRb // iswhite(v)
| jnz <1
| // Crossed a write barrier. Move the barrier forward.
|.if not X64WIN
| mov CARG2, RB
@ -3495,19 +3520,20 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
| ins_AND // RA = upvalue #, RD = str const (~)
| mov LFUNC:RB, [BASE-16]
| cleartp LFUNC:RB
| mov UPVAL:RB, [LFUNC:RB+RA*8+offsetof(GCfuncL, uvptr)]
| mov UPVAL:RB, LFUNC:RB->uvptr
| mov UPVAL:RB, [RB+RA*8]
| mov STR:RA, [KBASE+RD*8]
| mov RD, UPVAL:RB->v
| settp STR:ITYPE, STR:RA, LJ_TSTR
| mov [RD], STR:ITYPE
| test byte UPVAL:RB->marked, LJ_GC_BLACK // isblack(uv)
| jnz >2
| isblack TMPRd, TMPRb, UPVAL:RB, >2 // isblack(uv)
|1:
| ins_next
|
|2: // Check if string is white and ensure upvalue is closed.
| test byte GCOBJ:RA->gch.marked, LJ_GC_WHITES // iswhite(str)
| jz <1
| movzx TMPRd, byte [DISPATCH+DISPATCH_GL(gc.currentblackgray)]
| test byte GCOBJ:RA->gch.gcflags, TMPRb // iswhite(str)
| jnz <1
| cmp byte UPVAL:RB->closed, 0
| jz <1
| // Crossed a write barrier. Move the barrier forward.
@ -3523,7 +3549,8 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
| mov LFUNC:RB, [BASE-16]
| cleartp LFUNC:RB
| movsd xmm0, qword [KBASE+RD*8]
| mov UPVAL:RB, [LFUNC:RB+RA*8+offsetof(GCfuncL, uvptr)]
| mov UPVAL:RB, LFUNC:RB->uvptr
| mov UPVAL:RB, [RB+RA*8]
| mov RA, UPVAL:RB->v
| movsd qword [RA], xmm0
| ins_next
@ -3532,7 +3559,8 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
| ins_AD // RA = upvalue #, RD = primitive type (~)
| mov LFUNC:RB, [BASE-16]
| cleartp LFUNC:RB
| mov UPVAL:RB, [LFUNC:RB+RA*8+offsetof(GCfuncL, uvptr)]
| mov UPVAL:RB, LFUNC:RB->uvptr
| mov UPVAL:RB, [RB+RA*8]
| shl RD, 47
| not RD
| mov RA, UPVAL:RB->v
@ -3802,8 +3830,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
| cmp aword [RC], LJ_TNIL
| je >3 // Previous value is nil?
|1:
| test byte TAB:RB->marked, LJ_GC_BLACK // isblack(table)
| jnz >7
| isblack TMPRd, TMPRb, TAB:RB, >7 // isblack(table)
|2: // Set array slot.
| mov RB, [BASE+RA*8]
| mov [RC], RB
@ -3846,8 +3873,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
| cmp aword [TMPR], LJ_TNIL
| je >4 // Previous value is nil?
|2:
| test byte TAB:RB->marked, LJ_GC_BLACK // isblack(table)
| jnz >7
| isblack ITYPEd, ITYPEb, TAB:RB, >7 // isblack(table)
|3: // Set node value.
| mov ITYPE, [BASE+RA*8]
| mov [TMPR], ITYPE
@ -3903,8 +3929,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
| cmp aword [RC], LJ_TNIL
| je >3 // Previous value is nil?
|1:
| test byte TAB:RB->marked, LJ_GC_BLACK // isblack(table)
| jnz >7
| isblack TMPRd, TMPRb, TAB:RB, >7 // isblack(table)
|2: // Set array slot.
| mov ITYPE, [BASE+RA*8]
| mov [RC], ITYPE
@ -3931,8 +3956,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
|.else
| cvttsd2si RCd, qword [BASE+RC*8]
|.endif
| test byte TAB:RB->marked, LJ_GC_BLACK // isblack(table)
| jnz >7
| isblack TMPRd, TMPRb, TAB:RB, >7 // isblack(table)
|2:
| cmp RCd, TAB:RB->asize
| jae ->vmeta_tsetr
@ -3956,8 +3980,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
| lea RA, [BASE+RA*8]
| mov TAB:RB, [RA-8] // Guaranteed to be a table.
| cleartp TAB:RB
| test byte TAB:RB->marked, LJ_GC_BLACK // isblack(table)
| jnz >7
| isblack RDd, RDL, TAB:RB, >7 // isblack(table)
|2:
| mov RDd, MULTRES
| sub RDd, 1
@ -4670,7 +4693,8 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
| ins_AD // BASE = new base, RA = ins RA|RD (unused), RD = nargs+1
| mov CFUNC:RB, [BASE-16]
| cleartp CFUNC:RB
| mov KBASE, CFUNC:RB->f
| mov CFUNCD:KBASE, CFUNC:RB->data
| mov KBASE, CFUNCD:KBASE->f
| mov L:RB, SAVE_L
| lea RD, [BASE+NARGS:RD*8-8]
| mov L:RB->base, BASE