From e403e4a1dcdcd9f093084bee22da849a4b0a209b Mon Sep 17 00:00:00 2001 From: Albert <-> Date: Wed, 5 Jul 2023 21:27:35 -0400 Subject: [PATCH] New GC --- src/Makefile | 4 +- src/lib_aux.c | 32 + src/lib_buffer.c | 2 + src/lib_debug.c | 2 +- src/lib_ffi.c | 4 +- src/lib_io.c | 9 +- src/lib_jit.c | 2 +- src/lib_table.c | 8 + src/lj_api.c | 34 +- src/lj_arena.c | 397 +++++++++ src/lj_arena.h | 124 +++ src/lj_asm_x86.h | 49 +- src/lj_cdata.c | 11 +- src/lj_clib.c | 3 + src/lj_crecord.c | 2 +- src/lj_debug.c | 6 +- src/lj_def.h | 3 +- src/lj_err.c | 2 +- src/lj_ffrecord.c | 6 +- src/lj_func.c | 64 +- src/lj_func.h | 2 - src/lj_gc.c | 1884 ++++++++++++++++++++++++++++++++++++------ src/lj_gc.h | 281 ++++++- src/lj_intrin.h | 38 + src/lj_ir.c | 2 +- src/lj_ir.h | 3 +- src/lj_jit.h | 4 +- src/lj_lib.c | 6 +- src/lj_lib.h | 2 +- src/lj_obj.h | 126 ++- src/lj_record.c | 2 +- src/lj_snap.c | 2 +- src/lj_state.c | 52 +- src/lj_state.h | 7 + src/lj_str.c | 20 +- src/lj_tab.c | 111 ++- src/lj_tab.h | 2 +- src/lj_target_x86.h | 1 + src/lj_trace.c | 4 +- src/lj_udata.c | 13 +- src/lj_udata.h | 1 - src/luajit_rolling.h | 32 + src/vm_x64.dasc | 88 +- 43 files changed, 2894 insertions(+), 553 deletions(-) create mode 100644 src/lj_arena.c create mode 100644 src/lj_arena.h create mode 100644 src/lj_intrin.h diff --git a/src/Makefile b/src/Makefile index 224d21e7..1cb4b281 100644 --- a/src/Makefile +++ b/src/Makefile @@ -47,7 +47,7 @@ CCOPT= -O2 -fomit-frame-pointer # the binaries to a different machine you could also use: -march=native # CCOPT_x86= -march=i686 -msse -msse2 -mfpmath=sse -CCOPT_x64= +CCOPT_x64= -mbmi -mbmi2 -mavx -mavx2 CCOPT_arm= CCOPT_arm64= CCOPT_ppc= @@ -511,7 +511,7 @@ LJCORE_O= lj_assert.o lj_gc.o lj_err.o lj_char.o lj_bc.o lj_obj.o lj_buf.o \ lj_mcode.o lj_snap.o lj_record.o lj_crecord.o lj_ffrecord.o \ lj_asm.o lj_trace.o lj_gdbjit.o \ lj_ctype.o lj_cdata.o lj_cconv.o lj_ccall.o lj_ccallback.o \ - lj_carith.o lj_clib.o lj_cparse.o \ + lj_carith.o lj_clib.o lj_cparse.o lj_arena.o \ lj_lib.o lj_alloc.o lib_aux.o \ $(LJLIB_O) lib_init.o diff --git a/src/lib_aux.c b/src/lib_aux.c index 7e81ac30..1607648b 100644 --- a/src/lib_aux.c +++ b/src/lib_aux.c @@ -399,3 +399,35 @@ LUA_API lua_State *lua_newstate(lua_Alloc f, void *ud) #endif +LUA_API lua_State *luaJIT_newstate(lua_Alloc f, void *ud, + luaJIT_allocpages allocp, + luaJIT_freepages freep, + luaJIT_reallochuge realloch, + void *page_ud) +{ + lua_State *L; +#ifdef LUAJIT_USE_SYSMALLOC + if (!f) { + f = mem_alloc; + ud = NULL; + } +#else + if (f == NULL) { + f = LJ_ALLOCF_INTERNAL; + ud = NULL; + } +#endif + L = lj_newstate(f, ud, allocp, freep, realloch, page_ud); + + if (L) { + G(L)->panic = panic; +#ifndef LUAJIT_DISABLE_VMEVENT + luaL_findtable(L, LUA_REGISTRYINDEX, LJ_VMEVENTS_REGKEY, LJ_VMEVENTS_HSIZE); + lua_pushcfunction(L, error_finalizer); + lua_rawseti(L, -2, VMEVENT_HASH(LJ_VMEVENT_ERRFIN)); + G(L)->vmevmask = VMEVENT_MASK(LJ_VMEVENT_ERRFIN); + L->top--; +#endif + } + return L; +} diff --git a/src/lib_buffer.c b/src/lib_buffer.c index e4ec9d9d..261c5674 100644 --- a/src/lib_buffer.c +++ b/src/lib_buffer.c @@ -317,6 +317,8 @@ LJLIB_CF(buffer_new) ud->udtype = UDTYPE_BUFFER; /* NOBARRIER: The GCudata is new (marked white). */ setgcref(ud->metatable, obj2gco(env)); + if (lj_meta_fastg(G(L), tabref(ud->metatable), MM_gc)) + lj_mem_registergc_udata(L, ud); setudataV(L, L->top++, ud); sbx = (SBufExt *)uddata(ud); lj_bufx_init(L, sbx); diff --git a/src/lib_debug.c b/src/lib_debug.c index 2ad51a74..818cdd7b 100644 --- a/src/lib_debug.c +++ b/src/lib_debug.c @@ -232,7 +232,7 @@ LJLIB_CF(debug_upvalueid) if ((uint32_t)n >= fn->l.nupvalues) lj_err_arg(L, 2, LJ_ERR_IDXRNG); lua_pushlightuserdata(L, isluafunc(fn) ? (void *)gcref(fn->l.uvptr[n]) : - (void *)&fn->c.upvalue[n]); + (void *)&fn->c.data->upvalue[n]); return 1; } diff --git a/src/lib_ffi.c b/src/lib_ffi.c index ba783173..065ac923 100644 --- a/src/lib_ffi.c +++ b/src/lib_ffi.c @@ -518,7 +518,7 @@ LJLIB_CF(ffi_new) LJLIB_REC(.) /* Add to finalizer table, if still enabled. */ copyTV(L, lj_tab_set(L, t, o-1), tv); lj_gc_anybarriert(L, t); - cd->marked |= LJ_GC_CDATA_FIN; + cd->gcflags |= LJ_GC_CDATA_FIN; } } } @@ -573,7 +573,7 @@ LJLIB_CF(ffi_typeinfo) setintV(lj_tab_setstr(L, t, lj_str_newlit(L, "sib")), (int32_t)ct->sib); if (gcref(ct->name)) { GCstr *s = gco2str(gcref(ct->name)); - if (isdead(G(L), obj2gco(s))) flipwhite(obj2gco(s)); + maybe_resurrect_str(G(L), s); setstrV(L, lj_tab_setstr(L, t, lj_str_newlit(L, "name")), s); } lj_gc_check(L); diff --git a/src/lib_io.c b/src/lib_io.c index a13d4315..21b2d31c 100644 --- a/src/lib_io.c +++ b/src/lib_io.c @@ -25,6 +25,7 @@ #include "lj_strfmt.h" #include "lj_ff.h" #include "lj_lib.h" +#include "lj_meta.h" /* Userdata payload for I/O file. */ typedef struct IOFileUD { @@ -75,6 +76,8 @@ static IOFileUD *io_file_new(lua_State *L) ud->udtype = UDTYPE_IO_FILE; /* NOBARRIER: The GCudata is new (marked white). */ setgcrefr(ud->metatable, curr_func(L)->c.env); + if (lj_meta_fastg(G(L), tabref(ud->metatable), MM_gc)) + lj_mem_registergc_udata(L, ud); iof->fp = NULL; iof->type = IOFILE_TYPE_FILE; return iof; @@ -249,14 +252,14 @@ static int io_file_write(lua_State *L, IOFileUD *iof, int start) static int io_file_iter(lua_State *L) { GCfunc *fn = curr_func(L); - IOFileUD *iof = uddata(udataV(&fn->c.upvalue[0])); + IOFileUD *iof = uddata(udataV(&fn->c.data->upvalue[0])); int n = fn->c.nupvalues - 1; if (iof->fp == NULL) lj_err_caller(L, LJ_ERR_IOCLFL); L->top = L->base; if (n) { /* Copy upvalues with options to stack. */ lj_state_checkstack(L, (MSize)n); - memcpy(L->top, &fn->c.upvalue[1], n*sizeof(TValue)); + memcpy(L->top, &fn->c.data->upvalue[1], n * sizeof(TValue)); L->top += n; } n = io_file_read(L, iof, 0); @@ -531,6 +534,8 @@ static GCobj *io_std_new(lua_State *L, FILE *fp, const char *name) ud->udtype = UDTYPE_IO_FILE; /* NOBARRIER: The GCudata is new (marked white). */ setgcref(ud->metatable, gcV(L->top-3)); + if (lj_meta_fastg(G(L), tabref(ud->metatable), MM_gc)) + lj_mem_registergc_udata(L, ud); iof->fp = fp; iof->type = IOFILE_TYPE_STDF; lua_setfield(L, -2, name); diff --git a/src/lib_jit.c b/src/lib_jit.c index c0294927..ae9dcd41 100644 --- a/src/lib_jit.c +++ b/src/lib_jit.c @@ -220,7 +220,7 @@ LJLIB_CF(jit_util_funcinfo) if (!iscfunc(fn)) setintfield(L, t, "ffid", fn->c.ffid); setintptrV(lj_tab_setstr(L, t, lj_str_newlit(L, "addr")), - (intptr_t)(void *)fn->c.f); + (intptr_t)(void *)fn->c.data->f); setintfield(L, t, "upvalues", fn->c.nupvalues); } return 1; diff --git a/src/lib_table.c b/src/lib_table.c index 97ba2847..b1a81330 100644 --- a/src/lib_table.c +++ b/src/lib_table.c @@ -105,6 +105,14 @@ LJLIB_CF(table_insert) LJLIB_REC(.) return 0; } +LJLIB_CF(table_newgc) +{ + GCtab *t = lj_tab_newgc(L, 0, 0); + settabV(L, L->top++, t); + + return 1; +} + LJLIB_LUA(table_remove) /* function(t, pos) CHECK_tab(t) diff --git a/src/lj_api.c b/src/lj_api.c index d4048d79..9c462f54 100644 --- a/src/lj_api.c +++ b/src/lj_api.c @@ -56,7 +56,7 @@ static TValue *index2adr(lua_State *L, int idx) return o; } else { idx = LUA_GLOBALSINDEX - idx; - return idx <= fn->c.nupvalues ? &fn->c.upvalue[idx-1] : niltv(L); + return idx <= fn->c.nupvalues ? &fn->c.data->upvalue[idx - 1] : niltv(L); } } } @@ -602,7 +602,7 @@ LUA_API lua_CFunction lua_tocfunction(lua_State *L, int idx) if (tvisfunc(o)) { BCOp op = bc_op(*mref(funcV(o)->c.pc, BCIns)); if (op == BC_FUNCC || op == BC_FUNCCW) - return funcV(o)->c.f; + return funcV(o)->c.data->f; } return NULL; } @@ -697,12 +697,12 @@ LUA_API void lua_pushcclosure(lua_State *L, lua_CFunction f, int n) lj_gc_check(L); lj_checkapi_slot(n); fn = lj_func_newC(L, (MSize)n, getcurrenv(L)); - fn->c.f = f; + fn->c.data->f = f; L->top -= n; while (n--) - copyTV(L, &fn->c.upvalue[n], L->top+n); + copyTV(L, &fn->c.data->upvalue[n], L->top + n); setfuncV(L, L->top, fn); - lj_assertL(iswhite(obj2gco(fn)), "new GC object is not white"); + lj_assertL(iswhite(G(L), obj2gco(fn)), "new GC object is not white"); incr_top(L); } @@ -728,6 +728,15 @@ LUA_API void lua_createtable(lua_State *L, int narray, int nrec) incr_top(L); } +LUA_API void luaJIT_createtable(lua_State *L, int narray, int nrec) +{ + lj_gc_check(L); + settabV(L, L->top, + lj_tab_newgc(L, (uint32_t)(narray > 0 ? narray + 1 : 0), + hsize2hbits(nrec))); + incr_top(L); +} + LUALIB_API int luaL_newmetatable(lua_State *L, const char *tname) { GCtab *regt = tabV(registry(L)); @@ -927,7 +936,7 @@ LUA_API void *lua_upvalueid(lua_State *L, int idx, int n) n--; lj_checkapi((uint32_t)n < fn->l.nupvalues, "bad upvalue %d", n); return isluafunc(fn) ? (void *)gcref(fn->l.uvptr[n]) : - (void *)&fn->c.upvalue[n]; + (void *)&fn->c.data->upvalue[n]; } LUA_API void lua_upvaluejoin(lua_State *L, int idx1, int n1, int idx2, int n2) @@ -1046,8 +1055,11 @@ LUA_API int lua_setmetatable(lua_State *L, int idx) lj_gc_objbarriert(L, tabV(o), mt); } else if (tvisudata(o)) { setgcref(udataV(o)->metatable, obj2gco(mt)); - if (mt) + if (mt) { + if (lj_meta_fastg(g, mt, MM_gc)) + lj_mem_registergc_udata(L, udataV(o)); lj_gc_objbarrier(L, udataV(o), mt); + } } else { /* Flush cache, since traces specialize to basemt. But not during __gc. */ if (lj_trace_flushall(L)) @@ -1156,7 +1168,7 @@ static TValue *cpcall(lua_State *L, lua_CFunction func, void *ud) { GCfunc *fn = lj_func_newC(L, 0, getcurrenv(L)); TValue *top = L->top; - fn->c.f = func; + fn->c.data->f = func; setfuncV(L, top++, fn); if (LJ_FR2) setnilV(top++); #if LJ_64 @@ -1267,7 +1279,7 @@ LUA_API int lua_gc(lua_State *L, int what, int data) g->gc.threshold = data == -1 ? (g->gc.total/100)*g->gc.pause : g->gc.total; break; case LUA_GCCOLLECT: - lj_gc_fullgc(L); + lj_gc_fullgc(L, data); break; case LUA_GCCOUNT: res = (int)(g->gc.total >> 10); @@ -1316,3 +1328,7 @@ LUA_API void lua_setallocf(lua_State *L, lua_Alloc f, void *ud) g->allocf = f; } +LUA_API size_t luaJIT_getpagesize() +{ + return ARENA_SIZE; +} diff --git a/src/lj_arena.c b/src/lj_arena.c new file mode 100644 index 00000000..8147ce52 --- /dev/null +++ b/src/lj_arena.c @@ -0,0 +1,397 @@ +/* To get the mremap prototype. Must be defined before any system includes. */ +#if defined(__linux__) && !defined(_GNU_SOURCE) +#define _GNU_SOURCE +#endif + +#include "lj_def.h" +#include "lj_arena.h" +#include "lj_gc.h" +#include "lj_obj.h" + +#include +#include + +/* unmap() any misaligned huge pages */ +#define LJ_ALLOC_FIXUP 1 + +#if LJ_TARGET_LINUX +#define LJ_ALLOC_MREMAP 1 +#endif + +/* COMMITLESS trades increased memory usage for many fewer syscalls. Very recommended */ +#define COMMITLESS + +#if LJ_TARGET_POSIX +#include +#include + +#define RESERVE_PAGES(h, n) \ + mmap(h, n, PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0) +#define UNRESERVE_PAGES(p, n) munmap(p, n) +#define COMMIT_PAGES(p, n) + +#ifdef COMMITLESS +#define UNCOMMIT_PAGES(p, n) +#else +#define UNCOMMIT_PAGES(p, n) madvise(p, n, MADV_DONTNEED) +#endif + +#define RESERVE_AND_COMMIT_PAGES(n) RESERVE_PAGES(0, n) + +#elif LJ_TARGET_WINDOWS + +#define WIN32_LEAN_AND_MEAN +#include + +#define UNRESERVE_PAGES(p, n) VirtualFree(p, 0, MEM_RELEASE) +#define RESERVE_AND_COMMIT_PAGES(n) \ + LJ_WIN_VALLOC(0, n, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE) +#ifdef COMMITLESS +#define RESERVE_PAGES(h, n) LJ_WIN_VALLOC(h, n, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE) +#define COMMIT_PAGES(p, n) +#define UNCOMMIT_PAGES(p, n) +#else +#define RESERVE_PAGES(h, n) LJ_WIN_VALLOC(h, n, MEM_RESERVE, PAGE_READWRITE) +#define COMMIT_PAGES(p, n) LJ_WIN_VALLOC(p, n, MEM_COMMIT, PAGE_READWRITE) +#define UNCOMMIT_PAGES(p, n) VirtualFree(p, n, MEM_DECOMMIT) +#endif + +#else +#error "No page allocation OS support!" +#endif + +#define RESERVE_SIZE 8 * 1024 * 1024 +/* output >= input */ +#define PTRALIGN_UP(x, y) \ + (void *)(((uintptr_t)(x) + ((y)-1)) & ~(uintptr_t)((y)-1)) +/* output <= input */ +#define PTRALIGN_DOWN(x, y) (void *)(((uintptr_t)(x)) & ~(uintptr_t)((y)-1)) + +typedef struct arena_alloc { + void **freelist; + uint32_t freelist_sz; + uint32_t freelist_at; + + void **ptrs; + uint32_t ptrs_sz; + uint32_t ptrs_at; + + uint8_t *at; + uint8_t *end; + + void *hint; + + global_State *g; +} arena_alloc; + +int lj_arena_newchunk(arena_alloc *arenas) +{ + if (arenas->ptrs_at == arenas->ptrs_sz) { + size_t oldsz = sizeof(void *) * arenas->ptrs_sz; + arenas->ptrs_sz *= 2; + void *newp = arenas->g->allocf(arenas->g->allocd, arenas->ptrs, + oldsz, sizeof(void *) * arenas->ptrs_sz); + if (!newp) { + return 0; + } + arenas->ptrs = newp; + } + void *p = RESERVE_PAGES(arenas->hint, RESERVE_SIZE); + if (!p) { + p = RESERVE_PAGES(NULL, RESERVE_SIZE); + if (!p) { + return 0; + } + } + arenas->ptrs[arenas->ptrs_at++] = p; + + arenas->at = (uint8_t *)PTRALIGN_UP(p, ARENA_SIZE); + arenas->end = (uint8_t *)PTRALIGN_DOWN((uint8_t*)p + RESERVE_SIZE, ARENA_SIZE); + arenas->hint = PTRALIGN_UP((uint8_t *)p + RESERVE_SIZE, ARENA_SIZE); + return 1; +} + +static void lj_arena_api_freepages(void *ud, void **pages, unsigned n) +{ + arena_alloc *arenas = (arena_alloc*)ud; + if (LJ_UNLIKELY(!pages)) { + for (uint32_t i = 0; i < arenas->ptrs_at; i++) + UNRESERVE_PAGES(arenas->ptrs[i], RESERVE_SIZE); + arenas->g->allocf(arenas->g->allocd, arenas->freelist, + arenas->freelist_sz * sizeof(void *), 0); + arenas->g->allocf(arenas->g->allocd, arenas->ptrs, + arenas->ptrs_sz * sizeof(void*), 0); + /* This deletes the context so must be last */ + arenas->g->allocf(arenas->g->allocd, ud, sizeof(arena_alloc), 0); + } else { + if (arenas->freelist_at + n > arenas->freelist_sz) { + size_t oldsz = sizeof(void *) * arenas->ptrs_sz; + arenas->freelist_sz *= 2; + void *newp = arenas->g->allocf(arenas->g->allocd, arenas->freelist, oldsz, + sizeof(void *) * arenas->freelist_sz); + if (!newp) { + /* Well this is awkward. We are freeing memory but can't allocate enough + * for the freelist. Maybe a smaller realloc will work? */ + arenas->freelist_sz = arenas->freelist_at + n; + newp = arenas->g->allocf(arenas->g->allocd, arenas->freelist, oldsz, + sizeof(void *) * arenas->freelist_sz); + if (!newp) { + /* Just leak it, which does not help our OOM situation... */ + /* TODO: use arenas for the freelist, maybe chained? Pretty wasteful + * for most scenarios that don't have gigabytes freed. Maybe release + * pages back to the OS? */ + return; + } + } + arenas->freelist = (void**)newp; + } + + for (uint32_t i = 0; i < n; i++) { + arenas->freelist[arenas->freelist_at++] = pages[i]; + UNCOMMIT_PAGES(pages[i], ARENA_SIZE); + } + } +} + +static unsigned lj_arena_api_allocpages(void *ud, void **pages, unsigned n) +{ + arena_alloc *arenas = (arena_alloc*)ud; + if (arenas->freelist_at >= n) { + for (uint32_t i = 0; i < n; i++) { + pages[i] = arenas->freelist[--arenas->freelist_at]; + COMMIT_PAGES(pages[i], ARENA_SIZE); + } + } else if (arenas->end - arenas->at >= n * ARENA_SIZE) { + COMMIT_PAGES(arenas->at, n * ARENA_SIZE); + for (uint32_t i = 0; i < n; i++) { + pages[i] = arenas->at; + arenas->at += ARENA_SIZE; + } + } else { + for (uint32_t i = 0; i < n; i++) { + if (arenas->at == arenas->end) { + lj_arena_newchunk(arenas); + } + COMMIT_PAGES(arenas->at, ARENA_SIZE); + pages[i] = arenas->at; + arenas->at += ARENA_SIZE; + } + } + return n; +} + +struct posix_huge_arena +{ + void *base; + size_t size; +}; + +static void *allochuge(size_t sz) +{ +#if LJ_TARGET_WINDOWS + return RESERVE_AND_COMMIT_PAGES(sz); +#else + void *p = RESERVE_AND_COMMIT_PAGES(sz + ARENA_SIZE); + void *s = PTRALIGN_UP(p, ARENA_SIZE); + struct posix_huge_arena *a; +#if LJ_ALLOC_FIXUP + if (s != p) + UNRESERVE_PAGES(p, (uint8_t *)s - (uint8_t *)p); + a = (struct posix_huge_arena *)s; + a->base = s; + a->size = sz + ARENA_SIZE - ((uint8_t *)s - (uint8_t *)p); +#else + a->base = p; + a->size = sz + ARENA_SIZE; +#endif + return s; +#endif +} + +static void *lj_arena_api_reallochuge(void *ud, void *p, size_t osz, size_t nsz) +{ + void *newp; + if (!p) + return allochuge(nsz); +#if LJ_TARGET_WINDOWS + if (!nsz) { + UNRESERVE_PAGES(p, sz); + return NULL; + } + + newp = RESERVE_AND_COMMIT_PAGES(nsz); + if (LJ_LIKELY(newp)) + memcpy(newp, p, osz); + UNRESERVE_PAGES(p, osz); +#else + struct posix_huge_arena *a; + a = (struct posix_huge_arena *)p; + + if (!nsz) { + UNRESERVE_PAGES(a->base, a->size); + return NULL; + } + + if (nsz <= a->size) + return p; + +#if LJ_ALLOC_MREMAP + /* We can use mremap, but we also require a 64k base address. + * We could just try it and adjust but you can't move a mapping + * to an overlapping range. But mremap overwrites existing mappings, + * so we can use mmap to query a suitable range. */ + void *rawp = mmap(NULL, nsz + ARENA_SIZE, PROT_WRITE | PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + newp = PTRALIGN_UP(rawp, ARENA_SIZE); + if (rawp != newp) + UNRESERVE_PAGES(rawp, (uint8_t *)newp - (uint8_t *)rawp); + + /* Can either unmap or adopt the extra memory. This is one fewer syscall */ + nsz += ARENA_SIZE - ((uint8_t *)newp - (uint8_t *)rawp); + if (mremap(a->base, a->size, nsz, MREMAP_MAYMOVE | MREMAP_FIXED, + newp) != newp) { + UNRESERVE_PAGES(rawp, nsz + ARENA_SIZE); + return NULL; + } + a = (struct posix_huge_arena *)newp; + a->base = newp; + a->size = nsz; +#else + newp = allochuge(nsz); + memcpy(newp, p, osz); + UNRESERVE_PAGES(a->base, a->size); +#endif +#endif + return newp; +} + +#define lj_arena_firstalloc(g, arena, id, atype, type) \ + { \ + atype *a = (atype *)lj_arena_alloc(&g->gc.ctx); \ + if (!a) \ + return 0; \ + arena = &a->hdr; \ + do_arena_init(a, g, id, atype, type); \ + } + +static int lj_blob_firstalloc(global_State *g, GCAblob **h) +{ + GCAblob *a = (GCAblob *)lj_arena_alloc(&g->gc.ctx); + if (!a) + return 0; + g->gc.bloblist[0] = a; + g->gc.bloblist_usage[0] = 0; + a->alloc = sizeof(GCAblob); + a->flags = LJ_GC_SWEEP0; + a->id = 0; + *h = a; + return 1; +} + +int lj_arena_init(struct global_State *g, luaJIT_allocpages allocp, + luaJIT_freepages freep, luaJIT_reallochuge realloch, + void *page_ud) +{ + g->gc.bloblist_alloc = 32; + g->gc.bloblist = + (GCAblob **)g->allocf(g->allocd, NULL, 0, + g->gc.bloblist_alloc * sizeof(void *)); + if (!g->gc.bloblist) + return 0; + g->gc.bloblist_usage = (uint32_t *)g->allocf( + g->allocd, g->gc.bloblist_usage, 0, + g->gc.bloblist_alloc * sizeof(uint32_t)); + if (!g->gc.bloblist_usage) + return 0; + g->gc.bloblist_wr = 1; + + /* All must be provided to override */ + if (allocp && freep && realloch) { + g->gc.ctx.allocpages = allocp; + g->gc.ctx.freepages = freep; + g->gc.ctx.reallochuge = realloch; + g->gc.ctx.pageud = page_ud; + } else { + arena_alloc *arenas = (arena_alloc*)g->allocf(g->allocd, NULL, 0, sizeof(arena_alloc)); + if (!arenas) + return 0; + memset(arenas, 0, sizeof(arena_alloc)); + g->gc.ctx.allocpages = &lj_arena_api_allocpages; + g->gc.ctx.freepages = &lj_arena_api_freepages; + g->gc.ctx.reallochuge = &lj_arena_api_reallochuge; + g->gc.ctx.pageud = arenas; + + arenas->g = g; + arenas->freelist_sz = 32; + arenas->freelist = (void **)g->allocf(g->allocd, NULL, 0, + arenas->freelist_sz * sizeof(void *)); + arenas->ptrs_sz = 8; + arenas->ptrs = (void **)g->allocf(g->allocd, NULL, 0, + arenas->ptrs_sz * sizeof(void *)); + if (!arenas->freelist || !arenas->ptrs || !lj_arena_newchunk(arenas)) { + return 0; + } + } + + /* Allocate all arenas */ + lj_arena_firstalloc(g, g->gc.tab, ~LJ_TTAB, GCAtab, GCtab); + lj_arena_firstalloc(g, g->gc.fintab, ~LJ_TTAB, GCAtab, GCtab); + lj_arena_firstalloc(g, g->gc.uv, ~LJ_TUPVAL, GCAupval, GCupval); + lj_arena_firstalloc(g, g->gc.func, ~LJ_TFUNC, GCAfunc, GCfunc); + lj_arena_firstalloc(g, g->gc.udata, ~LJ_TUDATA, GCAudata, GCudata); + + ((GCAudata *)g->gc.udata)->free4_h = ((GCAudata *)g->gc.udata)->free_h; + + return lj_blob_firstalloc(g, &g->gc.blob_generic); +} + +static void release_one_arena(struct global_State *g, void *a) +{ + lj_arena_free(&g->gc.ctx, a); +} + +static void release_chain_arena(struct global_State *g, GCArenaHdr *a) +{ + while (a) { + GCArenaHdr *n = a->next; + release_one_arena(g, a); + a = n; + } +} + +static void release_all_arenas(struct global_State *g) +{ + release_chain_arena(g, g->gc.tab); + release_chain_arena(g, g->gc.fintab); + release_chain_arena(g, g->gc.uv); + release_chain_arena(g, g->gc.func); + release_chain_arena(g, g->gc.udata); + for (uint32_t i = 0; i < g->gc.bloblist_wr; i++) { + GCAblob *a = g->gc.bloblist[i]; + if (a->flags & GCA_BLOB_HUGE) + lj_arena_freehuge(&g->gc.ctx, a, a->alloc); + else + release_one_arena(g, a); + } +} + +void lj_arena_cleanup(struct global_State *g) +{ + release_all_arenas(g); + g->allocf(g->allocd, g->gc.bloblist, g->gc.bloblist_alloc * sizeof(void *), 0); + g->allocf(g->allocd, g->gc.bloblist_usage, g->gc.bloblist_alloc * sizeof(uint32_t), 0); + + /* Early failures may not initialize this. release_all_arenas will not have + * any actual arenas to release, thus it is safe. */ + if (!g->gc.ctx.freepages) + return; + + g->gc.ctx.mem_commit -= g->gc.ctx.freelist_at; + if (g->gc.ctx.freelist_at > 0) { + g->gc.ctx.freepages(g->gc.ctx.pageud, g->gc.ctx.freelist, + g->gc.ctx.freelist_at); + } + /* Special notice to shut down */ + g->gc.ctx.freepages(g->gc.ctx.pageud, NULL, 0); + g->gc.ctx.freelist_at = 0; +} diff --git a/src/lj_arena.h b/src/lj_arena.h new file mode 100644 index 00000000..4147ccb0 --- /dev/null +++ b/src/lj_arena.h @@ -0,0 +1,124 @@ +#ifndef _LJ_ARENA_H +#define _LJ_ARENA_H + +#include +#include +#include + +#include "lj_arch.h" +#include "lj_def.h" + +typedef unsigned (*luaJIT_allocpages)(void *ud, void **pages, unsigned n); +typedef void (*luaJIT_freepages)(void *ud, void **pages, unsigned n); +typedef void *(*luaJIT_reallochuge)(void *ud, void *p, size_t osz, size_t nsz); + +#define ARENA_SHIFT 16 +#define ARENA_SIZE (1u << ARENA_SHIFT) +#define ARENA_HUGE_THRESHOLD 16000 +#define ARENA_MASK (~0ull << ARENA_SHIFT) +#define ARENA_OMASK (ARENA_SIZE - 1) + +#define ARENA_FREELIST_SIZE 32 +#define ARENA_FREELIST_CHUNK (ARENA_FREELIST_SIZE / 2) +#define CHUNK_SIZE (ARENA_SIZE * ARENA_FREELIST_CHUNK) +#define MINIMUM_QUOTA (32 * ARENA_SIZE) + +struct global_State; + +/* These are here so obj.h can see them */ +/* Common header for all bitmap arenas */ +typedef struct GCArenaHdr { + struct GCArenaHdr *prev, *next; + struct GCArenaHdr *freeprev, *freenext; + struct GCArenaHdr *gray; + uint32_t obj_type; + uint32_t flags; +} GCArenaHdr; + +/* If set this arena is being moved from */ +#define GCA_BLOB_REAP 0x08 +#define GCA_BLOB_HUGE 0x10 + +/* Blob arenas don't hold objects but instead data pointed to by objects. + * All blob arenas are solely bump allocated and compacted as needed. + */ +typedef struct GCAblob { + uint8_t reserved_for_impl[20]; + uint32_t id; + uint32_t alloc; /* Next byte to be allocated from */ + uint32_t flags; +} GCAblob; + +LJ_STATIC_ASSERT(sizeof(GCAblob) % 16 == 0); + +typedef struct arena_context { + /* In arenas */ + uint32_t mem_commit; + uint32_t mem_watermark; + /* Bytes */ + uint64_t mem_huge; + + /* The free cache represents the "working memory" of the state */ + uint32_t freelist_at; + void *freelist[ARENA_FREELIST_SIZE]; + + luaJIT_allocpages allocpages; + luaJIT_freepages freepages; + luaJIT_reallochuge reallochuge; + void *pageud; +} arena_context; + +int lj_arena_init(struct global_State *g, luaJIT_allocpages allocp, + luaJIT_freepages freep, luaJIT_reallochuge realloch, + void *page_ud); +void lj_arena_cleanup(struct global_State *g); + +/* Add ARENA_FREELIST_CHUNK free arenas */ +inline void *lj_arena_alloc(arena_context *ctx) +{ + if (!ctx->freelist_at) { + ctx->freelist_at = + ctx->allocpages(ctx->pageud, ctx->freelist, ARENA_FREELIST_CHUNK); + ctx->mem_commit += ctx->freelist_at; + if (ctx->mem_commit > ctx->mem_watermark) + ctx->mem_watermark = ctx->mem_commit; + if (!ctx->freelist_at) + return NULL; + } + void *p = ctx->freelist[--ctx->freelist_at]; + return p; +} + +/* Free the last ARENA_FREELIST_CHUNK arenas */ +inline void lj_arena_free(arena_context *ctx, void *p) +{ + if (ctx->freelist_at == ARENA_FREELIST_SIZE) { + ctx->freepages(ctx->pageud, + ctx->freelist + + (ARENA_FREELIST_SIZE - ARENA_FREELIST_CHUNK), + ARENA_FREELIST_CHUNK); + ctx->freelist_at -= ARENA_FREELIST_CHUNK; + ctx->mem_commit -= ARENA_FREELIST_CHUNK; + } + ctx->freelist[ctx->freelist_at++] = p; +} + +inline void *lj_arena_allochuge(arena_context *ctx, size_t sz) +{ + ctx->mem_huge += sz; + return ctx->reallochuge(ctx->pageud, NULL, 0, sz); +} + +inline void lj_arena_freehuge(arena_context *ctx, void *p, size_t sz) +{ + ctx->mem_huge -= sz; + ctx->reallochuge(ctx->pageud, p, sz, 0); +} + +inline void *lj_arena_reallochuge(arena_context *ctx, void *p, size_t osz, size_t nsz) +{ + ctx->mem_huge = ctx->mem_huge - osz + nsz; + return ctx->reallochuge(ctx->pageud, p, osz, nsz); +} + +#endif diff --git a/src/lj_asm_x86.h b/src/lj_asm_x86.h index aee33716..144db54c 100644 --- a/src/lj_asm_x86.h +++ b/src/lj_asm_x86.h @@ -1398,9 +1398,10 @@ static void asm_uref(ASMState *as, IRIns *ir) GCobj *o = gcref(fn->l.uvptr[(ir->op2 >> 8)]); emit_loada(as, uv, o); } else { - emit_rmro(as, XO_MOV, uv|REX_GC64, ra_alloc1(as, ir->op1, RSET_GPR), - (int32_t)offsetof(GCfuncL, uvptr) + - (int32_t)sizeof(MRef) * (int32_t)(ir->op2 >> 8)); + emit_rmro(as, XO_MOV, uv|REX_GC64, uv, + (int32_t)sizeof(MRef) * (int32_t)(ir->op2 >> 8)); + emit_rmro(as, XO_MOV, uv|REX_GC64, + ra_alloc1(as, ir->op1, RSET_GPR), offsetof(GCfuncL, uvptr)); } } } @@ -1892,11 +1893,8 @@ static void asm_cnew(ASMState *as, IRIns *ir) } /* Combine initialization of marked, gct and ctypeid. */ - emit_movtomro(as, RID_ECX, RID_RET, offsetof(GCcdata, marked)); - emit_gri(as, XG_ARITHi(XOg_OR), RID_ECX, - (int32_t)((~LJ_TCDATA<<8)+(id<<16))); - emit_gri(as, XG_ARITHi(XOg_AND), RID_ECX, LJ_GC_WHITES); - emit_opgl(as, XO_MOVZXb, RID_ECX, gc.currentwhite); + emit_movmroi(as, RID_RET, offsetof(GCcdata, gcflags), + (int32_t)((~LJ_TCDATA << 8) + (id << 16))); args[0] = ASMREF_L; /* lua_State *L */ args[1] = ASMREF_TMP1; /* MSize size */ @@ -1912,14 +1910,15 @@ static void asm_tbar(ASMState *as, IRIns *ir) Reg tab = ra_alloc1(as, ir->op1, RSET_GPR); Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, tab)); MCLabel l_end = emit_label(as); - emit_movtomro(as, tmp|REX_GC64, tab, offsetof(GCtab, gclist)); + + emit_movtomro(as, tmp | REX_GC64, tab, offsetof(GCtab, gclist)); emit_setgl(as, tab, gc.grayagain); emit_getgl(as, tmp, gc.grayagain); - emit_i8(as, ~LJ_GC_BLACK); - emit_rmro(as, XO_ARITHib, XOg_AND, tab, offsetof(GCtab, marked)); - emit_sjcc(as, CC_Z, l_end); - emit_i8(as, LJ_GC_BLACK); - emit_rmro(as, XO_GROUP3b, XOg_TEST, tab, offsetof(GCtab, marked)); + emit_gmroi(as, XG_ARITHi(XOg_OR), tab, offsetof(GCtab, gcflags), LJ_GC_GRAY); + emit_sjcc(as, CC_NZ, l_end); + emit_opgl(as, XO_CMPb, tmp | REX_GC64, gc.currentblack); + emit_gri(as, XG_ARITHi(XOg_AND), tmp, LJ_GC_COLORS); + emit_rmro(as, XO_MOVZXb, tmp, tab, offsetof(GCtab, gcflags)); } static void asm_obar(ASMState *as, IRIns *ir) @@ -1927,7 +1926,7 @@ static void asm_obar(ASMState *as, IRIns *ir) const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_gc_barrieruv]; IRRef args[2]; MCLabel l_end; - Reg obj; + Reg obj, tmp; /* No need for other object barriers (yet). */ lj_assertA(IR(ir->op1)->o == IR_UREFC, "bad OBAR type"); ra_evictset(as, RSET_SCRATCH); @@ -1937,19 +1936,21 @@ static void asm_obar(ASMState *as, IRIns *ir) asm_gencall(as, ci, args); emit_loada(as, ra_releasetmp(as, ASMREF_TMP1), J2G(as->J)); obj = IR(ir->op1)->r; - emit_sjcc(as, CC_Z, l_end); - emit_i8(as, LJ_GC_WHITES); + tmp = ra_scratch(as, rset_exclude(RSET_GPR, obj)); + emit_sjcc(as, CC_NZ, l_end); + emit_opgl(as, XO_TESTb, tmp | REX_GC64, gc.currentblackgray); if (irref_isk(ir->op2)) { GCobj *vp = ir_kgc(IR(ir->op2)); - emit_rma(as, XO_GROUP3b, XOg_TEST, &vp->gch.marked); + emit_rma(as, XO_MOVZXb, tmp, &vp->gch.gcflags); } else { - Reg val = ra_alloc1(as, ir->op2, rset_exclude(RSET_SCRATCH&RSET_GPR, obj)); - emit_rmro(as, XO_GROUP3b, XOg_TEST, val, (int32_t)offsetof(GChead, marked)); + emit_rmro(as, XO_MOVZXb, tmp, obj, (int32_t) offsetof(GChead, gcflags)); } - emit_sjcc(as, CC_Z, l_end); - emit_i8(as, LJ_GC_BLACK); - emit_rmro(as, XO_GROUP3b, XOg_TEST, obj, - (int32_t)offsetof(GCupval, marked)-(int32_t)offsetof(GCupval, tv)); + emit_sjcc(as, CC_NZ, l_end); + emit_opgl(as, XO_CMPb, tmp | REX_GC64, gc.currentblack); + emit_gri(as, XG_ARITHi(XOg_AND), tmp, LJ_GC_COLORS); + emit_rmro(as, XO_MOVZXb, tmp, obj, + (int32_t)offsetof(GCupval, gcflags) - + (int32_t)offsetof(GCupval, tv)); } /* -- FP/int arithmetic and logic operations ------------------------------ */ diff --git a/src/lj_cdata.c b/src/lj_cdata.c index 77d9730f..4f2965c8 100644 --- a/src/lj_cdata.c +++ b/src/lj_cdata.c @@ -42,8 +42,8 @@ GCcdata *lj_cdata_newv(lua_State *L, CTypeID id, CTSize sz, CTSize align) g = G(L); setgcrefr(cd->nextgc, g->gc.root); setgcref(g->gc.root, obj2gco(cd)); - newwhite(g, obj2gco(cd)); - cd->marked |= 0x80; + newwhite(obj2gco(cd)); + cd->gcflags |= 0x80; cd->gct = ~LJ_TCDATA; cd->ctypeid = id; return cd; @@ -61,9 +61,8 @@ GCcdata *lj_cdata_newx(CTState *cts, CTypeID id, CTSize sz, CTInfo info) /* Free a C data object. */ void LJ_FASTCALL lj_cdata_free(global_State *g, GCcdata *cd) { - if (LJ_UNLIKELY(cd->marked & LJ_GC_CDATA_FIN)) { + if (LJ_UNLIKELY(cd->gcflags & LJ_GC_CDATA_FIN)) { GCobj *root; - makewhite(g, obj2gco(cd)); markfinalized(obj2gco(cd)); if ((root = gcref(g->gc.mmudata)) != NULL) { setgcrefr(cd->nextgc, root->gch.nextgc); @@ -95,10 +94,10 @@ void lj_cdata_setfin(lua_State *L, GCcdata *cd, GCobj *obj, uint32_t it) tv = lj_tab_set(L, t, &tmp); if (it == LJ_TNIL) { setnilV(tv); - cd->marked &= ~LJ_GC_CDATA_FIN; + cd->gcflags &= ~LJ_GC_CDATA_FIN; } else { setgcV(L, tv, obj, it); - cd->marked |= LJ_GC_CDATA_FIN; + cd->gcflags |= LJ_GC_CDATA_FIN; } } } diff --git a/src/lj_clib.c b/src/lj_clib.c index 513528ce..ff4b9cc7 100644 --- a/src/lj_clib.c +++ b/src/lj_clib.c @@ -17,6 +17,7 @@ #include "lj_cdata.h" #include "lj_clib.h" #include "lj_strfmt.h" +#include "lj_meta.h" /* -- OS-specific functions ----------------------------------------------- */ @@ -405,6 +406,8 @@ static CLibrary *clib_new(lua_State *L, GCtab *mt) ud->udtype = UDTYPE_FFI_CLIB; /* NOBARRIER: The GCudata is new (marked white). */ setgcref(ud->metatable, obj2gco(mt)); + if (lj_meta_fastg(G(L), tabref(ud->metatable), MM_gc)) + lj_mem_registergc_udata(L, ud); setudataV(L, L->top++, ud); return cl; } diff --git a/src/lj_crecord.c b/src/lj_crecord.c index 55d0b3ef..ec80274a 100644 --- a/src/lj_crecord.c +++ b/src/lj_crecord.c @@ -623,7 +623,7 @@ static TRef crec_ct_tv(jit_State *J, CType *d, TRef dp, TRef sp, cTValue *sval) ud->udtype == UDTYPE_IO_FILE ? IRFL_UDATA_FILE : IRFL_SBUF_R); } else { - sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCudata))); + sp = emitir(IRT(IR_FLOAD, IRT_PTR), sp, IRFL_UDATA_PAYLOAD); } } else if (tref_isstr(sp)) { if (ctype_isenum(d->info)) { /* Match string against enum constant. */ diff --git a/src/lj_debug.c b/src/lj_debug.c index fa189b6e..ae801df5 100644 --- a/src/lj_debug.c +++ b/src/lj_debug.c @@ -240,7 +240,7 @@ const char *lj_debug_uvnamev(cTValue *o, uint32_t idx, TValue **tvp, GCobj **op) } } else { if (idx < fn->c.nupvalues) { - *tvp = &fn->c.upvalue[idx]; + *tvp = &fn->c.data->upvalue[idx]; *op = obj2gco(fn); return ""; } @@ -632,7 +632,7 @@ void lj_debug_dumpstack(lua_State *L, SBuf *sb, const char *fmt, int depth) lj_buf_putb(sb, ']'); } else { /* Dump C function address. */ lj_buf_putb(sb, '@'); - lj_strfmt_putptr(sb, fn->c.f); + lj_strfmt_putptr(sb, fn->c.data->f); } break; case 'Z': /* Zap trailing separator. */ @@ -694,7 +694,7 @@ LUALIB_API void luaL_traceback (lua_State *L, lua_State *L1, const char *msg, if (*ar.what == 'm') { lua_pushliteral(L, " in main chunk"); } else if (*ar.what == 'C') { - lua_pushfstring(L, " at %p", fn->c.f); + lua_pushfstring(L, " at %p", fn->c.data->f); } else { lua_pushfstring(L, " in function <%s:%d>", ar.short_src, ar.linedefined); diff --git a/src/lj_def.h b/src/lj_def.h index 2a1d7b56..5bb7173a 100644 --- a/src/lj_def.h +++ b/src/lj_def.h @@ -59,7 +59,8 @@ typedef unsigned int uintptr_t; #define LJ_MAX_HBITS 26 /* Max. hash bits. */ #define LJ_MAX_ABITS 28 /* Max. bits of array key. */ #define LJ_MAX_ASIZE ((1<<(LJ_MAX_ABITS-1))+1) /* Max. array part size. */ -#define LJ_MAX_COLOSIZE 16 /* Max. elems for colocated array. */ +#define LJ_MAX_COLOSIZE 0 /* Max. elems for colocated array. */ +#define LJ_COLO_ENABLED 1 #define LJ_MAX_LINE LJ_MAX_MEM32 /* Max. source code line number. */ #define LJ_MAX_XLEVEL 200 /* Max. syntactic nesting level. */ diff --git a/src/lj_err.c b/src/lj_err.c index 7b11e4d0..26f68c06 100644 --- a/src/lj_err.c +++ b/src/lj_err.c @@ -1077,7 +1077,7 @@ LJ_NOINLINE void lj_err_argtype(lua_State *L, int narg, const char *xname) GCfunc *fn = curr_func(L); int idx = LUA_GLOBALSINDEX - narg; if (idx <= fn->c.nupvalues) - tname = lj_typename(&fn->c.upvalue[idx-1]); + tname = lj_typename(&fn->c.data->upvalue[idx-1]); else tname = lj_obj_typename[0]; } diff --git a/src/lj_ffrecord.c b/src/lj_ffrecord.c index 30dc6bfc..d3029da0 100644 --- a/src/lj_ffrecord.c +++ b/src/lj_ffrecord.c @@ -214,7 +214,7 @@ static void LJ_FASTCALL recff_type(jit_State *J, RecordFFData *rd) t = ~LJ_TLIGHTUD; else t = ~itype(&rd->argv[0]); - J->base[0] = lj_ir_kstr(J, strV(&J->fn->c.upvalue[t])); + J->base[0] = lj_ir_kstr(J, strV(&J->fn->c.data->upvalue[t])); UNUSED(rd); } @@ -452,7 +452,7 @@ static void LJ_FASTCALL recff_xpairs(jit_State *J, RecordFFData *rd) if (!((LJ_52 || (LJ_HASFFI && tref_iscdata(tr))) && recff_metacall(J, rd, MM_pairs + rd->data))) { if (tref_istab(tr)) { - J->base[0] = lj_ir_kfunc(J, funcV(&J->fn->c.upvalue[0])); + J->base[0] = lj_ir_kfunc(J, funcV(&J->fn->c.data->upvalue[0])); J->base[1] = tr; J->base[2] = rd->data ? lj_ir_kint(J, 0) : TREF_NIL; rd->nres = 3; @@ -663,7 +663,7 @@ static void LJ_FASTCALL recff_math_minmax(jit_State *J, RecordFFData *rd) static void LJ_FASTCALL recff_math_random(jit_State *J, RecordFFData *rd) { - GCudata *ud = udataV(&J->fn->c.upvalue[0]); + GCudata *ud = udataV(&J->fn->c.data->upvalue[0]); TRef tr, one; lj_ir_kgc(J, obj2gco(ud), IRT_UDATA); /* Prevent collection. */ tr = lj_ir_call(J, IRCALL_lj_prng_u64d, lj_ir_kptr(J, uddata(ud))); diff --git a/src/lj_func.c b/src/lj_func.c index 6a537649..f264a3a2 100644 --- a/src/lj_func.c +++ b/src/lj_func.c @@ -24,15 +24,6 @@ void LJ_FASTCALL lj_func_freeproto(global_State *g, GCproto *pt) /* -- Upvalues ------------------------------------------------------------ */ -static void unlinkuv(global_State *g, GCupval *uv) -{ - UNUSED(g); - lj_assertG(uvprev(uvnext(uv)) == uv && uvnext(uvprev(uv)) == uv, - "broken upvalue chain"); - setgcrefr(uvnext(uv)->prev, uv->prev); - setgcrefr(uvprev(uv)->next, uv->next); -} - /* Find existing open upvalue for a stack slot or create a new one. */ static GCupval *func_finduv(lua_State *L, TValue *slot) { @@ -44,34 +35,27 @@ static GCupval *func_finduv(lua_State *L, TValue *slot) while (gcref(*pp) != NULL && uvval((p = gco2uv(gcref(*pp)))) >= slot) { lj_assertG(!p->closed && uvval(p) != &p->tv, "closed upvalue in chain"); if (uvval(p) == slot) { /* Found open upvalue pointing to same slot? */ - if (isdead(g, obj2gco(p))) /* Resurrect it, if it's dead. */ - flipwhite(obj2gco(p)); return p; } - pp = &p->nextgc; + pp = &p->next; } /* No matching upvalue found. Create a new one. */ - uv = lj_mem_newt(L, sizeof(GCupval), GCupval); - newwhite(g, uv); + uv = lj_mem_allocuv(L); + newwhite(uv); uv->gct = ~LJ_TUPVAL; uv->closed = 0; /* Still open. */ setmref(uv->v, slot); /* Pointing to the stack slot. */ /* NOBARRIER: The GCupval is new (marked white) and open. */ - setgcrefr(uv->nextgc, *pp); /* Insert into sorted list of open upvalues. */ + setgcrefr(uv->next, *pp); /* Insert into sorted list of open upvalues. */ setgcref(*pp, obj2gco(uv)); - setgcref(uv->prev, obj2gco(&g->uvhead)); /* Insert into GC list, too. */ - setgcrefr(uv->next, g->uvhead.next); - setgcref(uvnext(uv)->prev, obj2gco(uv)); - setgcref(g->uvhead.next, obj2gco(uv)); - lj_assertG(uvprev(uvnext(uv)) == uv && uvnext(uvprev(uv)) == uv, - "broken upvalue chain"); return uv; } /* Create an empty and closed upvalue. */ static GCupval *func_emptyuv(lua_State *L) { - GCupval *uv = (GCupval *)lj_mem_newgco(L, sizeof(GCupval)); + GCupval *uv = lj_mem_allocuv(L); + newwhite(uv); uv->gct = ~LJ_TUPVAL; uv->closed = 1; setnilV(&uv->tv); @@ -86,32 +70,21 @@ void LJ_FASTCALL lj_func_closeuv(lua_State *L, TValue *level) global_State *g = G(L); while (gcref(L->openupval) != NULL && uvval((uv = gco2uv(gcref(L->openupval)))) >= level) { - GCobj *o = obj2gco(uv); - lj_assertG(!isblack(o), "bad black upvalue"); + lj_assertG(gcref(L->openupval) == obj2gco(uv), "bad upvalue list"); lj_assertG(!uv->closed && uvval(uv) != &uv->tv, "closed upvalue in chain"); - setgcrefr(L->openupval, uv->nextgc); /* No longer in open list. */ - if (isdead(g, o)) { - lj_func_freeuv(g, uv); - } else { - unlinkuv(g, uv); - lj_gc_closeuv(g, uv); - } - } -} + setgcrefr(L->openupval, uv->next); /* No longer in open list. */ -void LJ_FASTCALL lj_func_freeuv(global_State *g, GCupval *uv) -{ - if (!uv->closed) - unlinkuv(g, uv); - lj_mem_freet(g, uv); + copyTV(mainthread(g), &uv->tv, uvval(uv)); + setmref(uv->v, &uv->tv); + uv->closed = 1; + } } /* -- Functions (closures) ------------------------------------------------ */ GCfunc *lj_func_newC(lua_State *L, MSize nelems, GCtab *env) { - GCfunc *fn = (GCfunc *)lj_mem_newgco(L, sizeCfunc(nelems)); - fn->c.gct = ~LJ_TFUNC; + GCfunc *fn = lj_mem_allocfunc(L, sizeCfunc(nelems)); fn->c.ffid = FF_C; fn->c.nupvalues = (uint8_t)nelems; /* NOBARRIER: The GCfunc is new (marked white). */ @@ -123,8 +96,7 @@ GCfunc *lj_func_newC(lua_State *L, MSize nelems, GCtab *env) static GCfunc *func_newL(lua_State *L, GCproto *pt, GCtab *env) { uint32_t count; - GCfunc *fn = (GCfunc *)lj_mem_newgco(L, sizeLfunc((MSize)pt->sizeuv)); - fn->l.gct = ~LJ_TFUNC; + GCfunc *fn = lj_mem_allocfunc(L, sizeLfunc((MSize)pt->sizeuv)); fn->l.ffid = FF_LUA; fn->l.nupvalues = 0; /* Set to zero until upvalues are initialized. */ /* NOBARRIER: Really a setgcref. But the GCfunc is new (marked white). */ @@ -181,11 +153,3 @@ GCfunc *lj_func_newL_gc(lua_State *L, GCproto *pt, GCfuncL *parent) fn->l.nupvalues = (uint8_t)nuv; return fn; } - -void LJ_FASTCALL lj_func_free(global_State *g, GCfunc *fn) -{ - MSize size = isluafunc(fn) ? sizeLfunc((MSize)fn->l.nupvalues) : - sizeCfunc((MSize)fn->c.nupvalues); - lj_mem_free(g, fn, size); -} - diff --git a/src/lj_func.h b/src/lj_func.h index 96994ebf..ec277ed4 100644 --- a/src/lj_func.h +++ b/src/lj_func.h @@ -13,12 +13,10 @@ LJ_FUNC void LJ_FASTCALL lj_func_freeproto(global_State *g, GCproto *pt); /* Upvalues. */ LJ_FUNCA void LJ_FASTCALL lj_func_closeuv(lua_State *L, TValue *level); -LJ_FUNC void LJ_FASTCALL lj_func_freeuv(global_State *g, GCupval *uv); /* Functions (closures). */ LJ_FUNC GCfunc *lj_func_newC(lua_State *L, MSize nelems, GCtab *env); LJ_FUNC GCfunc *lj_func_newL_empty(lua_State *L, GCproto *pt, GCtab *env); LJ_FUNCA GCfunc *lj_func_newL_gc(lua_State *L, GCproto *pt, GCfuncL *parent); -LJ_FUNC void LJ_FASTCALL lj_func_free(global_State *g, GCfunc *c); #endif diff --git a/src/lj_gc.c b/src/lj_gc.c index c3a0c258..31675377 100644 --- a/src/lj_gc.c +++ b/src/lj_gc.c @@ -28,6 +28,7 @@ #include "lj_dispatch.h" #include "lj_vm.h" #include "lj_vmevent.h" +#include "lj_intrin.h" #define GCSTEPSIZE 1024u #define GCSWEEPMAX 40 @@ -35,60 +36,178 @@ #define GCFINALIZECOST 100 /* Macros to set GCobj colors and flags. */ -#define white2gray(x) ((x)->gch.marked &= (uint8_t)~LJ_GC_WHITES) -#define gray2black(x) ((x)->gch.marked |= LJ_GC_BLACK) -#define isfinalized(u) ((u)->marked & LJ_GC_FINALIZED) +#define white2gray(x) ((x)->gch.gcflags |= (uint8_t)LJ_GC_GRAY) +#define gray2black(g, x) \ + ((x)->gch.gcflags = (((x)->gch.gcflags & (uint8_t)~LJ_GC_COLORS) | (g)->gc.currentblack)) +#define isfinalized(u) ((u)->gcflags & LJ_GC_FINALIZED) /* -- Mark phase ---------------------------------------------------------- */ +#define gray_enq(a, g) \ + do { \ + a->hdr.gray = NULL; \ + if (LJ_LIKELY(g->gc.gray_head)) { \ + g->gc.gray_tail->gray = &a->hdr; \ + } else { \ + g->gc.gray_head = &a->hdr; \ + } \ + g->gc.gray_tail = &a->hdr; \ + } while (0) + /* Mark a TValue (if needed). */ #define gc_marktv(g, tv) \ { lj_assertG(!tvisgcv(tv) || (~itype(tv) == gcval(tv)->gch.gct), \ - "TValue and GC type mismatch"); \ - if (tviswhite(tv)) gc_mark(g, gcV(tv)); } + "TValue and GC type mismatch %d vs %d", ~itype(tv), \ + gcval(tv)->gch.gct); \ + if (tviswhite(g, tv)) gc_mark_type(g, gcV(tv), ~itype(tv)); } /* Mark a GCobj (if needed). */ #define gc_markobj(g, o) \ - { if (iswhite(obj2gco(o))) gc_mark(g, obj2gco(o)); } + { if (iswhite(g, obj2gco(o))) gc_mark_type(g, obj2gco(o), obj2gco(o)->gch.gct); } /* Mark a string object. */ -#define gc_mark_str(s) ((s)->marked &= (uint8_t)~LJ_GC_WHITES) +#define gc_mark_str(g, s) ((s)->gcflags |= (g)->gc.currentblack) + +static void *lj_mem_newblob_g(global_State *g, MSize sz); +static void gc_presweep_udata(global_State *g, GCAudata *a); + +static LJ_NOINLINE uintptr_t move_blob(global_State *g, uintptr_t src, MSize sz) +{ + void *newp = lj_mem_newblob_g(g, sz); + g->gc.bloblist_usage[gcablob(newp)->id] += sz; + memcpy(newp, (void *)src, sz); + return (uintptr_t)newp; +} + +#define mark_blob(g, b, sz) \ + do { \ + GCAblob *a = gcablob(b); \ + if (LJ_UNLIKELY((a->flags & GCA_BLOB_REAP))) { \ + b = move_blob(g, b, sz); \ + } else { \ + g->gc.bloblist_usage[a->id] += sz; \ + } \ + } while (0) + +#define maybe_mark_blob(g, b, sz) \ + if (sz > 0) { \ + mark_blob(g, b, sz); \ + } + +/* We only need to divide a small range and never need to divide + * anything with a remainder. An assert checks this is correct. + * Shifts of 16 to 52 work. Shift of 16 results in smaller + * constants. At 32 x86 might schedule edx:eax mul for + * shift-free access + */ +#define MULTIPLICATIVE_INVERSE_SHIFT 32 +#define MULTIPLICATIVE_INVERSE(x) \ + (uint32_t)(1 + (1ull << MULTIPLICATIVE_INVERSE_SHIFT) / x) + +/* ORDER LJ_T */ +const uint32_t kInverseDividers[~LJ_TNUMX] = { + 0, 0, 0, 0, 0, + MULTIPLICATIVE_INVERSE(sizeof(GCupval)), + 0, 0, + MULTIPLICATIVE_INVERSE(sizeof(GCfunc)), + 0, 0, + MULTIPLICATIVE_INVERSE(sizeof(GCtab)), + MULTIPLICATIVE_INVERSE(sizeof(GCudata)), +}; +const uint32_t kDividers[~LJ_TNUMX] = { + 0, 0, 0, 0, 0, + sizeof(GCupval), + 0, 0, + sizeof(GCfunc), + 0, 0, + sizeof(GCtab), + sizeof(GCudata), +}; + +#define is_arena_obj(t) (kInverseDividers[t] != 0) /* Mark a white GCobj. */ -static void gc_mark(global_State *g, GCobj *o) +static void gc_mark_type(global_State *g, GCobj *o, int gct) { - int gct = o->gch.gct; - lj_assertG(iswhite(o), "mark of non-white object"); - lj_assertG(!isdead(g, o), "mark of dead object"); - white2gray(o); - if (LJ_UNLIKELY(gct == ~LJ_TUDATA)) { - GCtab *mt = tabref(gco2ud(o)->metatable); - gray2black(o); /* Userdata are never gray. */ - if (mt) gc_markobj(g, mt); - gc_markobj(g, tabref(gco2ud(o)->env)); - if (LJ_HASBUFFER && gco2ud(o)->udtype == UDTYPE_BUFFER) { - SBufExt *sbx = (SBufExt *)uddata(gco2ud(o)); - if (sbufiscow(sbx) && gcref(sbx->cowref)) - gc_markobj(g, gcref(sbx->cowref)); - if (gcref(sbx->dict_str)) - gc_markobj(g, gcref(sbx->dict_str)); - if (gcref(sbx->dict_mt)) - gc_markobj(g, gcref(sbx->dict_mt)); + lj_assertG(gct == o->gch.gct, "GC type mismatch obj %d / param %d", + o->gch.gct, gct); + if (LJ_LIKELY(kInverseDividers[gct])) { + /* Generic arena marking */ + GCAcommon *a = arena(o); + /* mul + shift should be much faster than div on every CPU */ + uint32_t idx = (uint32_t)((objmask(o) * (uintptr_t)kInverseDividers[gct]) >> + MULTIPLICATIVE_INVERSE_SHIFT); + uint32_t h = aidxh(idx); + uint64_t bit = 1ull << aidxl(idx); + lj_assertG(idx <= ARENA_SIZE / 16, "index out of range"); + lj_assertG(idx == objmask(o) / kDividers[gct], "bad divider!"); + lj_assertG(!((((uintptr_t)(o)&ARENA_OMASK) % (uintptr_t)(kDividers[gct]))), + "index not multiple of divisor!"); + lj_assertG(gct == ~LJ_TUPVAL || gct == ~LJ_TSTR || gct == ~LJ_TFUNC || + gct == ~LJ_TTAB || gct == ~LJ_TUDATA, + "bad GC type %d", gct); + if (!(a->mark[h] & bit)) { + if (!a->gray_h) { + gray_enq(a, g); + } + a->gray_h |= 1ull << h; + a->mark[h] |= bit; + a->gray[h] |= bit; } - } else if (LJ_UNLIKELY(gct == ~LJ_TUPVAL)) { - GCupval *uv = gco2uv(o); - gc_marktv(g, uvval(uv)); - if (uv->closed) - gray2black(o); /* Closed upvalues are never gray. */ - } else if (gct != ~LJ_TSTR && gct != ~LJ_TCDATA) { - lj_assertG(gct == ~LJ_TFUNC || gct == ~LJ_TTAB || - gct == ~LJ_TTHREAD || gct == ~LJ_TPROTO || gct == ~LJ_TTRACE, - "bad GC type %d", gct); + return; + } + lj_assertG(iswhite(g, o), "mark of non-white object"); + lj_assertG(!checkdead(g, o), "mark of dead object"); + white2gray(o); + if (gct != ~LJ_TSTR && gct != ~LJ_TCDATA) { + lj_assertG(gct == ~LJ_TTHREAD || gct == ~LJ_TPROTO || + gct == ~LJ_TTRACE, "bad GC type %d", gct); + lj_assertG(o->gch.gcflags & LJ_GC_GRAY, "not gray?"); setgcrefr(o->gch.gclist, g->gc.gray); setgcref(g->gc.gray, o); } } +static void gc_mark_uv(global_State *g, GCupval *o) +{ + GCAupval *a = gcat(o, GCAupval); + uint32_t idx = aidx(o); + uint32_t h = aidxh(idx); + uint64_t bit = 1ull << aidxl(idx); + lj_assertG(idx >= ELEMENTS_OCCUPIED(GCAupval, GCupval) && + idx < ELEMENTS_MAX(GCupval), + "bad obj pointer"); + lj_assertG(~LJ_TUPVAL == o->gct, "not a upval"); + if (!(a->mark[h] & bit)) { + if (!a->gray_h) { + gray_enq(a, g); + } + a->gray_h |= 1ull << h; + a->mark[h] |= bit; + a->gray[h] |= bit; + } +} + +static void gc_mark_tab(global_State *g, GCtab *o) +{ + GCAtab *a = gcat(o, GCAtab); + uint32_t idx = aidx(o); + uint32_t h = aidxh(idx); + uint64_t bit = 1ull << aidxl(idx); + lj_assertG(idx >= ELEMENTS_OCCUPIED(GCAtab, GCtab) && + idx < ELEMENTS_MAX(GCtab), + "bad obj pointer"); + lj_assertG(~LJ_TTAB == o->gct, "not a table"); + if (!(a->mark[h] & bit)) { + if (!a->gray_h) { + gray_enq(a, g); + } + a->gray_h |= 1ull << h; + a->mark[h] |= bit; + a->gray[h] |= bit; + } +} + /* Mark GC roots. */ static void gc_mark_gcroot(global_State *g) { @@ -105,78 +224,62 @@ static void gc_mark_start(global_State *g) setgcrefnull(g->gc.grayagain); setgcrefnull(g->gc.weak); gc_markobj(g, mainthread(g)); - gc_markobj(g, tabref(mainthread(g)->env)); + gc_mark_tab(g, tabref(mainthread(g)->env)); gc_marktv(g, &g->registrytv); gc_mark_gcroot(g); g->gc.state = GCSpropagate; -} - -/* Mark open upvalues. */ -static void gc_mark_uv(global_State *g) -{ - GCupval *uv; - for (uv = uvnext(&g->uvhead); uv != &g->uvhead; uv = uvnext(uv)) { - lj_assertG(uvprev(uvnext(uv)) == uv && uvnext(uvprev(uv)) == uv, - "broken upvalue chain"); - if (isgray(obj2gco(uv))) - gc_marktv(g, uvval(uv)); - } -} - -/* Mark userdata in mmudata list. */ -static void gc_mark_mmudata(global_State *g) -{ - GCobj *root = gcref(g->gc.mmudata); - GCobj *u = root; - if (u) { - do { - u = gcnext(u); - makewhite(g, u); /* Could be from previous GC. */ - gc_mark(g, u); - } while (u != root); - } + g->gc.accum = 0; } /* Separate userdata objects to be finalized to mmudata list. */ -size_t lj_gc_separateudata(global_State *g, int all) +void lj_gc_separateudata(global_State *g) { - size_t m = 0; - GCRef *p = &mainthread(g)->nextgc; - GCobj *o; - while ((o = gcref(*p)) != NULL) { - if (!(iswhite(o) || all) || isfinalized(gco2ud(o))) { - p = &o->gch.nextgc; /* Nothing to do. */ - } else if (!lj_meta_fastg(g, tabref(gco2ud(o)->metatable), MM_gc)) { - markfinalized(o); /* Done, as there's no __gc metamethod. */ - p = &o->gch.nextgc; - } else { /* Otherwise move userdata to be finalized to mmudata list. */ - m += sizeudata(gco2ud(o)); - markfinalized(o); - *p = o->gch.nextgc; - if (gcref(g->gc.mmudata)) { /* Link to end of mmudata list. */ - GCobj *root = gcref(g->gc.mmudata); - setgcrefr(o->gch.nextgc, root->gch.nextgc); - setgcref(root->gch.nextgc, o); - setgcref(g->gc.mmudata, o); - } else { /* Create circular list. */ - setgcref(o->gch.nextgc, o); - setgcref(g->gc.mmudata, o); - } - } + GCArenaHdr *a; + uint32_t i; + setgcrefnull(g->gc.fin_list); + for (a = g->gc.udata; a; a = a->next) { + GCAudata *ud = (GCAudata *)a; + for (i = 0; i < WORDS_FOR_TYPE_UNROUNDED(GCudata); i++) + ud->mark[i] &= ~ud->fin_req[i]; } - return m; + gc_presweep_udata(g, (GCAudata *)g->gc.udata); } /* -- Propagation phase --------------------------------------------------- */ /* Traverse a table. */ +static void gc_mark_tab_hash(global_State *g, GCtab *t) +{ + MSize hmask = t->hmask; + MSize size = (hmask + 1) * sizeof(Node); + GCAblob *a = gcablob(mrefu(t->node)); + if (LJ_UNLIKELY(((a->flags & GCA_BLOB_REAP) != 0 && !mrefu(g->jit_base)))) { + /* Rewrite everything to account for the new location */ + ptrdiff_t old_addr = mrefu(t->node); + ptrdiff_t new_addr = move_blob(g, old_addr, size); + ptrdiff_t diff = new_addr - old_addr; + setmrefu(t->node, new_addr); + for (uint32_t i = 0; i <= hmask; i++) { + Node *n = mref(t->node, Node) + i; + if (mrefu(n->next)) { + setmrefu(n->next, mrefu(n->next) + diff); + } + } + if (mref(t->freetop, Node) != &g->nilnode) { + setmrefu(t->freetop, mrefu(t->freetop) + diff); + } + } else { + g->gc.bloblist_usage[a->id] += size; + } +} + static int gc_traverse_tab(global_State *g, GCtab *t) { int weak = 0; cTValue *mode; GCtab *mt = tabref(t->metatable); if (mt) - gc_markobj(g, mt); + gc_mark_tab(g, mt); mode = lj_meta_fastg(g, mt, MM_mode); if (mode && tvisstr(mode)) { /* Valid __mode field? */ const char *modestr = strVdata(mode); @@ -193,28 +296,40 @@ static int gc_traverse_tab(global_State *g, GCtab *t) } else #endif { - t->marked = (uint8_t)((t->marked & ~LJ_GC_WEAK) | weak); - setgcrefr(t->gclist, g->gc.weak); - setgcref(g->gc.weak, obj2gco(t)); + t->gcflags = (uint8_t)((t->gcflags & ~LJ_GC_WEAK) | weak); } } } - if (weak == LJ_GC_WEAK) /* Nothing to mark if both keys/values are weak. */ - return 1; + if (!(t->gcflags & LJ_GC_MARK_MASK) && mrefu(t->array)) { + GCAblob *a = gcablob(mref(t->array, void)); + if (LJ_UNLIKELY(((a->flags & GCA_BLOB_REAP) != 0 && !mrefu(g->jit_base)))) { + setmrefu(t->array, move_blob(g, mrefu(t->array), t->asize * sizeof(TValue))); + } else { + g->gc.bloblist_usage[a->id] += t->asize * sizeof(TValue); + } + } + /* Nothing to mark if both keys/values are weak or ephemeron. */ + if (weak > LJ_GC_WEAKVAL) + return weak; + /* We can't move table data while on a trace */ if (!(weak & LJ_GC_WEAKVAL)) { /* Mark array part. */ MSize i, asize = t->asize; for (i = 0; i < asize; i++) gc_marktv(g, arrayslot(t, i)); } if (t->hmask > 0) { /* Mark hash part. */ - Node *node = noderef(t->node); + Node *node; MSize i, hmask = t->hmask; + gc_mark_tab_hash(g, t); + node = noderef(t->node); + for (i = 0; i <= hmask; i++) { Node *n = &node[i]; - if (!tvisnil(&n->val)) { /* Mark non-empty slot. */ - lj_assertG(!tvisnil(&n->key), "mark of nil key in non-empty slot"); - if (!(weak & LJ_GC_WEAKKEY)) gc_marktv(g, &n->key); - if (!(weak & LJ_GC_WEAKVAL)) gc_marktv(g, &n->val); + if (!tvisnil(&n->val)) { /* Mark non-empty slot. */ + lj_assertG(!tvisnil(&n->key), "mark of nil key in non-empty slot"); + /* TODO this is *only* required for FFI finalizer table */ + if (!(weak & LJ_GC_WEAKKEY)) gc_marktv(g, &n->key); + if (!(weak & LJ_GC_WEAKVAL)) gc_marktv(g, &n->val); } } } @@ -224,18 +339,18 @@ static int gc_traverse_tab(global_State *g, GCtab *t) /* Traverse a function. */ static void gc_traverse_func(global_State *g, GCfunc *fn) { - gc_markobj(g, tabref(fn->c.env)); + gc_mark_tab(g, tabref(fn->c.env)); if (isluafunc(fn)) { uint32_t i; lj_assertG(fn->l.nupvalues <= funcproto(fn)->sizeuv, "function upvalues out of range"); gc_markobj(g, funcproto(fn)); for (i = 0; i < fn->l.nupvalues; i++) /* Mark Lua function upvalues. */ - gc_markobj(g, &gcref(fn->l.uvptr[i])->uv); + gc_mark_uv(g, gco2uv(gcref(fn->l.uvptr[i]))); } else { uint32_t i; for (i = 0; i < fn->c.nupvalues; i++) /* Mark C function upvalues. */ - gc_marktv(g, &fn->c.upvalue[i]); + gc_marktv(g, &fn->c.data->upvalue[i]); } } @@ -245,8 +360,9 @@ static void gc_marktrace(global_State *g, TraceNo traceno) { GCobj *o = obj2gco(traceref(G2J(g), traceno)); lj_assertG(traceno != G2J(g)->cur.traceno, "active trace escaped"); - if (iswhite(o)) { + if (iswhite(g, o)) { white2gray(o); + lj_assertG(o->gch.gcflags & LJ_GC_GRAY, "not gray?"); setgcrefr(o->gch.gclist, g->gc.gray); setgcref(g->gc.gray, o); } @@ -280,7 +396,7 @@ static void gc_traverse_trace(global_State *g, GCtrace *T) static void gc_traverse_proto(global_State *g, GCproto *pt) { ptrdiff_t i; - gc_mark_str(proto_chunkname(pt)); + gc_mark_str(g, proto_chunkname(pt)); for (i = -(ptrdiff_t)pt->sizekgc; i < 0; i++) /* Mark collectable consts. */ gc_markobj(g, proto_kgc(pt, i)); #if LJ_HASJIT @@ -316,37 +432,180 @@ static void gc_traverse_thread(global_State *g, lua_State *th) for (; o < top; o++) /* Clear unmarked slots. */ setnilV(o); } - gc_markobj(g, tabref(th->env)); + gc_mark_tab(g, tabref(th->env)); lj_state_shrinkstack(th, gc_traverse_frames(g, th)); } +static size_t traverse_upvals(global_State *g, GCAupval *a, size_t threshold) +{ + size_t ret = 0; + for (uint32_t i = tzcount64(a->gray_h); a->gray_h; + a->gray_h = reset_lowest64(a->gray_h), i = tzcount64(a->gray_h)) { + /* It is not allowed to synchronously change a->gray[i] */ + uint64_t v = a->gray[i]; + while(v) { + GCupval *uv = aobj(a, GCupval, (i << 6) + tzcount64(v)); + uv->gcflags = g->gc.currentblack; + v = reset_lowest64(v); + ret += sizeof(GCupval); + gc_marktv(g, uvval(uv)); + if (ret >= threshold) { + a->gray[i] = v; + return ret; + } + } + a->gray[i] = 0; + } + g->gc.gray_head = a->hdr.gray; + return ret; +} + +static size_t traverse_funcs(global_State *g, GCAfunc *a, size_t threshold) +{ + size_t ret = 0; + for (uint32_t i = tzcount64(a->gray_h); a->gray_h; + a->gray_h = reset_lowest64(a->gray_h), i = tzcount64(a->gray_h)) { + /* It is not allowed to synchronously change a->gray[i] */ + uint64_t v = a->gray[i]; + while (v) { + uint32_t j = tzcount64(v); + GCfunc *fn = aobj(a, GCfunc, (i << 6) + j); + MSize size = isluafunc(fn) ? sizeLfunc((MSize)fn->l.nupvalues) + : sizeCfunc((MSize)fn->c.nupvalues); + gray2black(g, obj2gco(fn)); + if (!(fn->gen.gcflags & LJ_GC_MARK_MASK)) { + maybe_mark_blob(g, mrefu(fn->gen.data), size); + } + v = reset_lowest64(v); + ret += sizeof(GCfunc) + size; + a->mark[i] |= flags2bitmask(obj2gco(fn), j); + gc_traverse_func(g, fn); + if (ret >= threshold) { + a->gray[i] = v; + return ret; + } + } + a->gray[i] = 0; + } + g->gc.gray_head = a->hdr.gray; + return ret; +} + +static size_t traverse_tables(global_State *g, GCAtab *a, size_t threshold) +{ + size_t ret = 0; + /* Tables could contain other table refs and those refs could be to + * this arena, so we must handle cases where gc_traverse_tab sets + * bits in the current or previous words. */ + while (a->gray_h) { + uint32_t i = tzcount64(a->gray_h); + for (uint32_t j = tzcount64(a->gray[i]); a->gray[i]; + j = tzcount64(a->gray[i])) { + GCtab *t = aobj(a, GCtab, (i << 6) + j); + gray2black(g, obj2gco(t)); + a->gray[i] = reset_lowest64(a->gray[i]); + + ret += sizeof(GCtab) + sizeof(TValue) * t->asize + + (t->hmask ? sizeof(Node) * (t->hmask + 1) : 0); + a->mark[i] |= flags2bitmask(obj2gco(t), j); + if (gc_traverse_tab(g, t) > 0) { + /* Weak tables go onto the grayagain list */ + t->gcflags |= LJ_GC_GRAY; + setgcrefr(t->gclist, g->gc.grayagain); + setgcref(g->gc.grayagain, obj2gco(t)); + } + if (ret >= threshold) + return ret; + } + a->gray_h ^= 1ull << i; + } + g->gc.gray_head = a->hdr.gray; + return ret; +} + +static size_t traverse_udata(global_State *g, GCAudata *a, size_t threshold) +{ + size_t ret = 0; + for (uint32_t i = tzcount64(a->gray_h); a->gray_h; + a->gray_h = reset_lowest64(a->gray_h), i = tzcount64(a->gray_h)) { + /* It is not allowed to synchronously change a->gray[i] */ + uint64_t v = a->gray[i]; + while (v) { + uint32_t j = tzcount64(v); + GCudata *ud = aobj(a, GCudata, (i << 6) + j); + GCtab *mt = tabref(ud->metatable); + v = reset_lowest64(v); + gray2black(g, obj2gco(ud)); + a->gray[i] = reset_lowest64(a->gray[i]); + /* If this occupies multiple slots mark them all */ + + a->mark[i] |= flags2bitmask(obj2gco(ud), j); + if (mt) + gc_mark_tab(g, mt); + gc_mark_tab(g, tabref(ud->env)); + if (LJ_HASBUFFER && ud->udtype == UDTYPE_BUFFER) { + SBufExt *sbx = (SBufExt *)uddata(ud); + if (sbufiscow(sbx) && gcref(sbx->cowref)) + gc_markobj(g, gcref(sbx->cowref)); + if (gcref(sbx->dict_str)) + gc_mark_tab(g, tabref(sbx->dict_str)); + if (gcref(sbx->dict_mt)) + gc_mark_tab(g, tabref(sbx->dict_mt)); + } + ret += sizeof(GCudata); + if (ret >= threshold) { + a->gray[i] = v; + return ret; + } + } + a->gray[i] = 0; + } + g->gc.gray_head = a->hdr.gray; + return ret; +} + +/* Propagate arena objects */ +static size_t propagatemark_arena(global_State *g, size_t threshold) +{ + GCArenaHdr *a = g->gc.gray_head; + size_t ret = 0; + switch (a->obj_type) { + case ~LJ_TUPVAL: + ret = traverse_upvals(g, (GCAupval *)a, threshold); + break; + case ~LJ_TFUNC: + ret = traverse_funcs(g, (GCAfunc *)a, threshold); + break; + case ~LJ_TTAB: + ret = traverse_tables(g, (GCAtab *)a, threshold); + break; + case ~LJ_TUDATA: + ret = traverse_udata(g, (GCAudata *)a, threshold); + break; + default: + lj_assertG(0, "bad arena type"); + break; + } + g->gc.accum += ret; + return ret; +} + /* Propagate one gray object. Traverse it and turn it black. */ static size_t propagatemark(global_State *g) { GCobj *o = gcref(g->gc.gray); int gct = o->gch.gct; lj_assertG(isgray(o), "propagation of non-gray object"); - gray2black(o); - setgcrefr(g->gc.gray, o->gch.gclist); /* Remove from gray list. */ - if (LJ_LIKELY(gct == ~LJ_TTAB)) { - GCtab *t = gco2tab(o); - if (gc_traverse_tab(g, t) > 0) - black2gray(o); /* Keep weak tables gray. */ - return sizeof(GCtab) + sizeof(TValue) * t->asize + - (t->hmask ? sizeof(Node) * (t->hmask + 1) : 0); - } else if (LJ_LIKELY(gct == ~LJ_TFUNC)) { - GCfunc *fn = gco2func(o); - gc_traverse_func(g, fn); - return isluafunc(fn) ? sizeLfunc((MSize)fn->l.nupvalues) : - sizeCfunc((MSize)fn->c.nupvalues); - } else if (LJ_LIKELY(gct == ~LJ_TPROTO)) { + gray2black(g, o); + setgcrefr(g->gc.gray, o->gch.gclist); /* Remove from gray list. */ + if (LJ_LIKELY(gct == ~LJ_TPROTO)) { GCproto *pt = gco2pt(o); gc_traverse_proto(g, pt); return pt->sizept; } else if (LJ_LIKELY(gct == ~LJ_TTHREAD)) { lua_State *th = gco2th(o); - setgcrefr(th->gclist, g->gc.grayagain); - setgcref(g->gc.grayagain, o); + setgcrefr(th->gclist, g->gc.grayagain_th); + setgcref(g->gc.grayagain_th, o); black2gray(o); /* Threads are never black. */ gc_traverse_thread(g, th); return sizeof(lua_State) + sizeof(TValue) * th->stacksize; @@ -354,8 +613,8 @@ static size_t propagatemark(global_State *g) #if LJ_HASJIT GCtrace *T = gco2trace(o); gc_traverse_trace(g, T); - return ((sizeof(GCtrace)+7)&~7) + (T->nins-T->nk)*sizeof(IRIns) + - T->nsnap*sizeof(SnapShot) + T->nsnapmap*sizeof(SnapEntry); + return ((sizeof(GCtrace) + 7) & ~7) + (T->nins - T->nk) * sizeof(IRIns) + + T->nsnap * sizeof(SnapShot) + T->nsnapmap * sizeof(SnapEntry); #else lj_assertG(0, "bad GC type %d", gct); return 0; @@ -363,15 +622,93 @@ static size_t propagatemark(global_State *g) } } +static void sweep_upvals(global_State *g) +{ + int c = g->gc.currentblack; + for (GCobj *o = gcref(g->gc.grayagain_th); o; o = gcref(o->gch.gclist)) { + GCobj *uvo; + GCRef *uvp = &gco2th(o)->openupval; + GCupval *uv; + /* Need to sweep dead upvals */ + while ((uvo = gcref(*uvp)) != NULL) { + uv = &uvo->uv; + if (uv->gcflags & c) { + uvp = &uv->next; + } else { + setgcrefr(*uvp, uv->next); + } + } + } +} + +static void propagatemark_again(global_State *g) +{ + for (GCobj *o = gcref(g->gc.grayagain); o;) { + GCobj *n = gcref(o->gch.gclist); + int x; + lj_assertG(isgray(o), "propagation of non-gray object"); + gray2black(g, o); + x = gc_traverse_tab(g, gco2tab(o)); + if(x > 0) { + lj_assertG(o->gch.gcflags & LJ_GC_WEAK, "no weak flags"); + if (x == LJ_GC_WEAKKEY) { + setgcrefr(o->gch.gclist, g->gc.ephemeron); + setgcref(g->gc.ephemeron, o); + } else { + setgcrefr(o->gch.gclist, g->gc.weak); + setgcref(g->gc.weak, o); + } + } + o = n; + } + + for (GCobj *o = gcref(g->gc.grayagain_th); o; o = gcref(o->gch.gclist)) { + gray2black(g, o); + gc_traverse_thread(g, gco2th(o)); + } +} + /* Propagate all gray objects. */ static size_t gc_propagate_gray(global_State *g) { size_t m = 0; - while (gcref(g->gc.gray) != NULL) - m += propagatemark(g); + while (gcref(g->gc.gray) != NULL || g->gc.gray_head != NULL) { + while (gcref(g->gc.gray) != NULL) + m += propagatemark(g); + while (g->gc.gray_head != NULL) + m += propagatemark_arena(g, UINT_MAX); + } return m; } +static int traverse_ephemeron(global_State *g, GCtab *t) +{ + int ret = 0; + MSize i, hmask; + Node *node = mref(t->node, Node); + hmask = t->hmask; + for (i = 0; i <= hmask; i++) { + Node *n = &node[i]; + if (!tvisnil(&n->val) && tviswhite(g, &n->val) && !tviswhite(g, &n->key)) { + gc_marktv(g, &n->val); + ret = 1; + } + } + return ret; +} + +static void process_ephemerons(global_State *g) +{ + int changed; + do { + gc_propagate_gray(g); + changed = 0; + for (GCtab *t = tabref(g->gc.ephemeron); t; t = tabref(t->gclist)) { + changed |= traverse_ephemeron(g, t); + } + } while (changed); +} + /* -- Sweep phase --------------------------------------------------------- */ /* Type of GC free functions. */ @@ -380,10 +717,10 @@ typedef void (LJ_FASTCALL *GCFreeFunc)(global_State *g, GCobj *o); /* GC free functions for LJ_TSTR .. LJ_TUDATA. ORDER LJ_T */ static const GCFreeFunc gc_freefunc[] = { (GCFreeFunc)lj_str_free, - (GCFreeFunc)lj_func_freeuv, + (GCFreeFunc)0, (GCFreeFunc)lj_state_free, (GCFreeFunc)lj_func_freeproto, - (GCFreeFunc)lj_func_free, + (GCFreeFunc)0, #if LJ_HASJIT (GCFreeFunc)lj_trace_free, #else @@ -394,30 +731,571 @@ static const GCFreeFunc gc_freefunc[] = { #else (GCFreeFunc)0, #endif - (GCFreeFunc)lj_tab_free, - (GCFreeFunc)lj_udata_free + (GCFreeFunc)0, + (GCFreeFunc)0 }; /* Full sweep of a GC list. */ #define gc_fullsweep(g, p) gc_sweep(g, (p), ~(uint32_t)0) +#ifdef LUA_USE_ASSERT +static int check_not_gray(global_State *g, GCArenaHdr *a) +{ + for (GCArenaHdr *h = g->gc.gray_head; h; h = h->gray) + if (h == a) + return 0; + return 1; +} +#endif + +static void gc_free_arena(global_State *g, GCArenaHdr *a) +{ + lj_assertG(check_not_gray(g, a), "arena in gray list while being freed"); + lj_assertG(a->prev != NULL, "freeing list head"); + lj_assertG(a->prev->next == a, "freeing broken chain"); + lj_assertG(!a->freeprev || a->freenext != a->freeprev, "broken freelist"); + + a->prev->next = a->next; + if (a->next) { + lj_assertG(a->next->prev == a, "freeing broken chain"); + a->next->prev = a->prev; + } + + if (a->freeprev) + a->freeprev->freenext = a->freenext; + if (a->freenext) + a->freenext->freeprev = a->freeprev; + lj_arena_free(&g->gc.ctx, a); +} + +/* Fixups are required for the first & last words */ +#define sweep_fixup(atype, otype) \ + free &= FREE_MASK(otype); \ + a->free[0] &= FREE_LOW(atype, otype); \ + if (!a->free[0]) \ + free &= ~1ull; \ + if (HIGH_ELEMENTS_OCCUPIED(otype) != 0) { \ + a->free[FREE_HIGH_INDEX(otype)] &= FREE_HIGH(otype); \ + if (!a->free[FREE_HIGH_INDEX(otype)]) \ + free &= ~(1ull << FREE_HIGH_INDEX(otype)); \ + } + +/* The first arena in the list is the primary one. It is being allocated out of + * and can never be put on the freelist or released */ +#define sweep_free(atype, src, freevar) \ + if (LJ_LIKELY(g->gc.src != &a->hdr)) { \ + if (LJ_UNLIKELY(I256_EQ_64_MASK(any, zero) == 0xF)) { \ + GCArenaHdr *x = &a->hdr; \ + a = (atype *)a->hdr.next; \ + if (x == g->gc.freevar) { \ + g->gc.freevar = x->freenext; \ + } \ + gc_free_arena(g, x); \ + continue; \ + } \ + if (LJ_UNLIKELY(free && !a->free_h)) { \ + free_enq(&a->hdr, g->gc.freevar); \ + } \ + } + +/* This is pipelined, while a is processing the branchy fixup & free code + * b is running the SIMD crunchy code. */ +static void *gc_sweep_tab_i256(global_State *g, GCAtab *a, uint32_t lim) +{ + GCAtab *b; + I256 v, v2; + I256 any, any2; + I256 zero; + I256 ones; + I256_ZERO(any); + I256_ZERO(zero); + I256_ONES(ones); + uint64_t free = ~0ull; + uint64_t free2; + + if (!a) + return NULL; + b = (GCAtab *)a->hdr.next; + + for (uint32_t i = 0; i < SIMD_WORDS_FOR_TYPE(GCtab); i++) { + I256_LOADA(v, &a->mark[i * SIMD_MULTIPLIER]); + I256_OR(any, any, v); + if (!isminor(g)) + I256_STOREA(&a->mark[i * SIMD_MULTIPLIER], zero); + I256_XOR(v, v, ones); + I256_STOREA(&a->free[i * SIMD_MULTIPLIER], v); + free ^= I256_EQ_64_MASK(v, zero) << (SIMD_MULTIPLIER * i); + } + for (; b && lim; lim--, a = b, b = (GCAtab*)b->hdr.next, any = any2, v = v2, free = free2) { + lj_assertG((a->hdr.flags & LJ_GC_SWEEPS) != LJ_GC_SWEEPS, + "both bits cannot be set!"); + + lj_assertG(!(a->hdr.flags & g->gc.currentsweep), "sweeping swept arena"); + + I256_ZERO(any2); + free2 = ~0ull; + + a->hdr.flags ^= LJ_GC_SWEEPS; + + for (uint32_t i = 0; i < SIMD_WORDS_FOR_TYPE(GCtab); i++) { + I256_LOADA(v2, &b->mark[i * SIMD_MULTIPLIER]); + I256_OR(any2, any2, v2); + if (!isminor(g)) + I256_STOREA(&b->mark[i * SIMD_MULTIPLIER], zero); + I256_XOR(v2, v2, ones); + I256_STOREA(&b->free[i * SIMD_MULTIPLIER], v2); + free2 ^= I256_EQ_64_MASK(v2, zero) << (SIMD_MULTIPLIER * i); + } + + sweep_fixup(GCAtab, GCtab); + + if (LJ_UNLIKELY(I256_EQ_64_MASK(any, zero) == 0xF)) { + GCArenaHdr *x = &a->hdr; + a = (GCAtab *)a->hdr.next; + if (x == g->gc.free_tab) { + g->gc.free_tab = x->freenext; + } + gc_free_arena(g, x); + continue; + } + if (LJ_UNLIKELY(free && !a->free_h)) { + free_enq(&a->hdr, g->gc.free_tab); + } + + a->free_h = free; + } + + sweep_fixup(GCAtab, GCtab); + a->hdr.flags ^= LJ_GC_SWEEPS; + if (LJ_UNLIKELY(I256_EQ_64_MASK(any, zero) == 0xF)) { + GCArenaHdr *x = &a->hdr; + if (x == g->gc.free_tab) { + g->gc.free_tab = x->freenext; + if (a->hdr.freenext) + a->hdr.freenext->freeprev = NULL; + } + gc_free_arena(g, x); + return b; + } + if (LJ_UNLIKELY(free && !a->free_h)) { + free_enq(&a->hdr, g->gc.free_tab); + } + + a->free_h = free; + + return b; +} + +static void *gc_sweep_tab1_i256(global_State *g, GCAtab *a) +{ + I256 v; + I256 any; + I256 zero; + I256 ones; + I256_ZERO(any); + I256_ZERO(zero); + I256_ONES(ones); + do { + uint64_t free = ~0ull; + lj_assertG((a->hdr.flags & LJ_GC_SWEEPS) != LJ_GC_SWEEPS, + "both bits cannot be set!"); + + lj_assertG(!(a->hdr.flags & g->gc.currentsweep), "sweeping swept arena"); + + a->hdr.flags ^= LJ_GC_SWEEPS; + /* free = ~mark + * mark = 0 (if major collection) + */ + for (uint32_t i = 0; i < SIMD_WORDS_FOR_TYPE(GCtab); i++) { + I256_LOADA(v, &a->mark[i * SIMD_MULTIPLIER]); + I256_OR(any, any, v); + if (!isminor(g)) + I256_STOREA(&a->mark[i * SIMD_MULTIPLIER], zero); + I256_XOR(v, v, ones); + I256_STOREA(&a->free[i * SIMD_MULTIPLIER], v); + free ^= I256_EQ_64_MASK(v, zero) << (SIMD_MULTIPLIER * i); + } + + sweep_fixup(GCAtab, GCtab); + + sweep_free(GCAtab, tab, free_tab); + + a->free_h = free; + a = (GCAtab *)a->hdr.next; + } while (0); + return a; +} + +static void *gc_sweep_fintab1_i256(global_State *g, GCAtab *a) +{ + I256 v, f; + I256 any; + I256 zero; + I256 ones; + I256_ZERO(any); + I256_ZERO(zero); + I256_ONES(ones); + do { + uint64_t free = ~0ull; + lj_assertG((a->hdr.flags & LJ_GC_SWEEPS) != LJ_GC_SWEEPS, + "both bits cannot be set!"); + + lj_assertG(!(a->hdr.flags & g->gc.currentsweep), "sweeping swept arena"); + + a->hdr.flags ^= LJ_GC_SWEEPS; + /* free = ~mark + * fin = fin & mark + * mark = 0 (if major collection) + */ + for (uint32_t i = 0; i < SIMD_WORDS_FOR_TYPE(GCtab); i++) { + I256_LOADA(v, &a->mark[i * SIMD_MULTIPLIER]); + I256_LOADA(f, &a->fin[i * SIMD_MULTIPLIER]); + I256_OR(any, any, v); + if (!isminor(g)) + I256_STOREA(&a->mark[i * SIMD_MULTIPLIER], zero); + I256_AND(f, f, v); + I256_XOR(v, v, ones); + I256_STOREA(&a->free[i * SIMD_MULTIPLIER], v); + I256_STOREA(&a->fin[i * SIMD_MULTIPLIER], f); + free ^= I256_EQ_64_MASK(v, zero) << (SIMD_MULTIPLIER * i); + } + + sweep_fixup(GCAtab, GCtab); + + sweep_free(GCAtab, fintab, free_fintab); + + a->free_h = free; + a = (GCAtab *)a->hdr.next; + } while (0); + return a; +} + +static void gc_presweep_process(global_State *g, GCAtab *a, uint32_t i, + bitmap_t f) +{ + for (uint32_t j = tzcount64(f); f; f = reset_lowest64(f), j = tzcount64(f)) { + GCtab *t = aobj(a, GCtab, (i << 6) + j); + setgcrefr(t->gclist, g->gc.fin_list); + setgcref(g->gc.fin_list, obj2gco(t)); + } +} + +static void gc_presweep_process_ud(global_State *g, GCAudata *a, uint32_t i, bitmap_t f) +{ + for (uint32_t j = tzcount64(f); f; f = reset_lowest64(f), j = tzcount64(f)) { + GCudata *t = aobj(a, GCudata, (i << 6) + j); + setgcrefr(t->gclist, g->gc.fin_list); + setgcref(g->gc.fin_list, obj2gco(t)); + } +} + +static void gc_presweep_fintab(global_State *g, GCAtab *a) +{ + /* We could set fin to f directly as f represents "new" finalized objects + * and this would skip the step of clearing fin bits in sweeping, + * however this would cause a re-run of a finalizer for an object that had + * previously been finalized but were referenced by a dead finalized object + * in the next cycle, because it's fin bit would get cleared here since it + * still isn't marked. + * While it's very unlikely this will ever happen in a real program this + * matches PUC Lua behaviour. + */ + for (; a; a = (GCAtab *)a->hdr.next) { + uint32_t i; + bitmap_t gray_h = 0; + bitmap_t f = + ~(a->free[0] | a->fin[0] | a->mark[0]) & FREE_LOW(GCAtab, GCtab); + a->fin[0] |= f; + a->gray[0] = f; + a->mark[0] |= f; + if (f) { + gray_h |= 1; + gc_presweep_process(g, a, 0, f); + } + for (i = 1; i < WORDS_FOR_TYPE_UNROUNDED(GCtab) - 1; i++) { + f = ~(a->free[i] | a->fin[i] | a->mark[i]); + a->fin[i] |= f; + a->gray[i] = f; + a->mark[i] |= f; + if (f) { + gray_h |= abit(i); + gc_presweep_process(g, a, i, f); + } + } + f = ~(a->free[i] | a->fin[i] | a->mark[i]); + if (HIGH_ELEMENTS_OCCUPIED(GCtab) != 0) { + f &= FREE_HIGH(GCtab); + } + a->fin[i] |= f; + a->gray[i] = f; + a->mark[i] |= f; + if (f) { + gray_h |= abit(i); + gc_presweep_process(g, a, i, f); + } + + a->gray_h = gray_h; + if (gray_h) + gray_enq(a, g); + } +} + +static void gc_presweep_udata(global_State *g, GCAudata *a) +{ + for (; a; a = (GCAudata *)a->hdr.next) { + uint32_t i; + bitmap_t gray_h = 0; + bitmap_t f = a->fin_req[0] & ~(a->free[0] | a->fin[0] | a->mark[0]) & + FREE_LOW(GCAudata, GCudata); + a->fin[0] |= f; + a->gray[0] = f; + a->mark[0] |= f; + if (f) { + gray_h |= 1; + gc_presweep_process_ud(g, a, 0, f); + } + for (i = 1; i < WORDS_FOR_TYPE_UNROUNDED(GCudata) - 1; i++) { + f = a->fin_req[i] & ~(a->free[i] | a->fin[i] | a->mark[i]); + a->fin[i] |= f; + a->gray[i] = f; + a->mark[i] |= f; + if (f) { + gray_h |= abit(i); + gc_presweep_process_ud(g, a, i, f); + } + } + f = a->fin_req[i] & ~(a->free[i] | a->fin[i] | a->mark[i]); + if (HIGH_ELEMENTS_OCCUPIED(GCudata) != 0) { + f &= FREE_HIGH(GCudata); + } + a->fin[i] |= f; + a->gray[i] = f; + a->mark[i] |= f; + if (f) { + gray_h |= abit(i); + gc_presweep_process_ud(g, a, i, f); + } + + a->gray_h = gray_h; + if (gray_h) + gray_enq(a, g); + } +} + +static void *gc_sweep_func_i256(global_State *g, GCAfunc *a, uint32_t lim) +{ + I256 v; + I256 any; + I256 zero; + I256 ones; + I256_ZERO(zero); + I256_ONES(ones); + for (; a && lim; lim--) { + uint64_t free = ~0ull; + I256_ZERO(any); + + lj_assertG((a->hdr.flags & LJ_GC_SWEEPS) != LJ_GC_SWEEPS, + "both bits cannot be set!"); + + lj_assertG(!(a->hdr.flags & g->gc.currentsweep), "sweeping swept arena"); + a->hdr.flags ^= LJ_GC_SWEEPS; + + for (uint32_t i = 0; i < SIMD_WORDS_FOR_TYPE(GCfunc); i++) { + /* free = ~mark; mark = 0*/ + I256_LOADA(v, &a->mark[i * SIMD_MULTIPLIER]); + I256_OR(any, any, v); + if (!isminor(g)) + I256_STOREA(&a->mark[i * SIMD_MULTIPLIER], zero); + I256_XOR(v, v, ones); + I256_STOREA(&a->free[i * SIMD_MULTIPLIER], v); + free ^= I256_EQ_64_MASK(v, zero) << (SIMD_MULTIPLIER * i); + } + + sweep_fixup(GCAfunc, GCfunc); + + sweep_free(GCAfunc, func, free_func); + + a->free_h = free; + a = (GCAfunc *)a->hdr.next; + } + return a; +} + +static void *gc_sweep_uv_i256(global_State *g, GCAupval *a, uint32_t lim) +{ + I256 v; + I256 any; + I256 zero; + I256 ones; + I256_ZERO(zero); + I256_ONES(ones); + for (; a && lim; lim--) { + uint64_t free = ~0ull; + I256_ZERO(any); + + lj_assertG((a->hdr.flags & LJ_GC_SWEEPS) != LJ_GC_SWEEPS, "both bits cannot be set!"); + + lj_assertG(!(a->hdr.flags & g->gc.currentsweep), "sweeping swept arena"); + a->hdr.flags ^= LJ_GC_SWEEPS; + + for (uint32_t i = 0; i < SIMD_WORDS_FOR_TYPE(GCupval); i++) { + /* free = ~mark; mark = 0*/ + I256_LOADA(v, &a->mark[i * SIMD_MULTIPLIER]); + I256_OR(any, any, v); + if (!isminor(g)) + I256_STOREA(&a->mark[i * SIMD_MULTIPLIER], zero); + I256_XOR(v, v, ones); + I256_STOREA(&a->free[i * SIMD_MULTIPLIER], v); + free ^= I256_EQ_64_MASK(v, zero) << (SIMD_MULTIPLIER * i); + } + + sweep_fixup(GCAupval, GCupval); + + sweep_free(GCAupval, uv, free_uv); + + a->free_h = free; + a = (GCAupval *)a->hdr.next; + } + return a; +} + +static void *gc_sweep_tab(global_State *g, GCAtab *a, uint32_t lim) +{ + return gc_sweep_tab_i256(g, a, lim); +} +static void *gc_sweep_tab1(global_State *g, GCAtab *a) +{ + return gc_sweep_tab1_i256(g, a); +} + +static void *gc_sweep_fintab(global_State *g, GCAtab *a, uint32_t lim) +{ + return gc_sweep_fintab1_i256(g, a); +} +static void *gc_sweep_fintab1(global_State *g, GCAtab *a) +{ + return gc_sweep_fintab1_i256(g, a); +} + +static void *gc_sweep_func(global_State *g, GCAfunc *a, uint32_t lim) +{ + return gc_sweep_func_i256(g, a, lim); +} +static void *gc_sweep_func1(global_State *g, GCAfunc *a) +{ + return gc_sweep_func_i256(g, a, 1); +} + +static void *gc_sweep_uv(global_State *g, GCAupval *a, uint32_t lim) +{ + return gc_sweep_uv_i256(g, a, lim); +} +static void *gc_sweep_uv1(global_State *g, GCAupval *a) +{ + return gc_sweep_uv_i256(g, a, 1); +} + +static void gc_sweep_udata_obj(global_State *g, GCAudata *a, uint32_t i, + bitmap_t f) +{ + GCudata *base = aobj(a, GCudata, i << 6); + for (uint32_t j = tzcount64(f); f; f = reset_lowest64(f), j = tzcount64(f)) { + GCudata *ud = &base[j]; + if (!(ud->gcflags & LJ_GC_MARK_MASK) && ud->len > 0) { + g->gc.malloc -= ud->len; + g->allocf(g->allocd, uddata(ud), ud->len, 0); + } + } +} + +/* Because a lot of these will require individual traversal anyway, + * it's probably best to do this as scalar code */ +static void *gc_sweep_udata1(global_State *g, GCAudata *a) +{ + uint32_t free = 0; + bitmap_t m; + bitmap_t f; + bitmap_t any = 0; + uint32_t i = 0; + + lj_assertG((a->hdr.flags & LJ_GC_SWEEPS) != LJ_GC_SWEEPS, + "both bits cannot be set!"); + + lj_assertG(!(a->hdr.flags & g->gc.currentsweep), "sweeping swept arena"); + a->hdr.flags ^= LJ_GC_SWEEPS; + + m = a->mark[i]; + any |= m; + f = ~m & ~a->free[i] & FREE_LOW(GCAudata, GCudata); + a->free[i] |= f; + if (!isminor(g)) + a->mark[i] = 0; + a->fin[i] &= m; + a->fin_req[i] &= m; + gc_sweep_udata_obj(g, a, 0, f); + if (f) + free = 1; + + for (i = 1; i < WORDS_FOR_TYPE_UNROUNDED(GCudata) - 1; i++) { + m = a->mark[i]; + any |= m; + f = ~m & ~a->free[i]; + a->free[i] |= f; + if (!isminor(g)) + a->mark[i] = 0; + a->fin[i] &= m; + a->fin_req[i] &= m; + gc_sweep_udata_obj(g, a, 0, f); + if (f) + free |= 1u << i; + } + + m = a->mark[i]; + any |= m; + f = ~m & ~a->free[i]; + if (HIGH_ELEMENTS_OCCUPIED(GCudata) != 0) { + f &= FREE_HIGH(GCudata); + } + a->free[i] |= f; + if (!isminor(g)) + a->mark[i] = 0; + a->fin[i] &= m; + a->fin_req[i] &= m; + gc_sweep_udata_obj(g, a, 0, f); + if (f) + free |= 1u << i; + + if (&a->hdr != g->gc.udata) { + if (!any) { + GCArenaHdr *x = &a->hdr; + a = (GCAudata *)a->hdr.next; + if (x == g->gc.free_udata) { + g->gc.free_udata = x->freenext; + } + gc_free_arena(g, x); + return a; + } + + if (free && !a->free_h) { + free_enq(&a->hdr, g->gc.free_udata); + } + } + + a->free_h |= free; + return a->hdr.next; +} + /* Partial sweep of a GC list. */ static GCRef *gc_sweep(global_State *g, GCRef *p, uint32_t lim) { /* Mask with other white and LJ_GC_FIXED. Or LJ_GC_SFIXED on shutdown. */ - int ow = otherwhite(g); + int safe = g->gc.safecolor; GCobj *o; while ((o = gcref(*p)) != NULL && lim-- > 0) { - if (o->gch.gct == ~LJ_TTHREAD) /* Need to sweep open upvalues, too. */ - gc_fullsweep(g, &gco2th(o)->openupval); - if (((o->gch.marked ^ LJ_GC_WHITES) & ow)) { /* Black or current white? */ - lj_assertG(!isdead(g, o) || (o->gch.marked & LJ_GC_FIXED), - "sweep of undead object"); - makewhite(g, o); /* Value is alive, change to the current white. */ + if (o->gch.gcflags & safe) { /* Black or current white? */ p = &o->gch.nextgc; + makewhite(o); } else { /* Otherwise value is dead, free it. */ - lj_assertG(isdead(g, o) || ow == LJ_GC_SFIXED, - "sweep of unlive object"); setgcrefr(*p, o->gch.nextgc); if (o == gcref(g->gc.root)) setgcrefr(g->gc.root, o->gch.nextgc); /* Adjust list anchor. */ @@ -431,21 +1309,18 @@ static GCRef *gc_sweep(global_State *g, GCRef *p, uint32_t lim) static void gc_sweepstr(global_State *g, GCRef *chain) { /* Mask with other white and LJ_GC_FIXED. Or LJ_GC_SFIXED on shutdown. */ - int ow = otherwhite(g); + int sweep = g->gc.safecolor; + uint8_t mask = isminor(g) ? 0xFF : ~LJ_GC_COLORS; uintptr_t u = gcrefu(*chain); GCRef q; GCRef *p = &q; GCobj *o; setgcrefp(q, (u & ~(uintptr_t)1)); while ((o = gcref(*p)) != NULL) { - if (((o->gch.marked ^ LJ_GC_WHITES) & ow)) { /* Black or current white? */ - lj_assertG(!isdead(g, o) || (o->gch.marked & LJ_GC_FIXED), - "sweep of undead string"); - makewhite(g, o); /* String is alive, change to the current white. */ + if ((o->gch.gcflags & sweep)) { /* Black or current white? */ + o->gch.gcflags &= mask; /* String is alive. */ p = &o->gch.nextgc; } else { /* Otherwise string is dead, free it. */ - lj_assertG(isdead(g, o) || ow == LJ_GC_SFIXED, - "sweep of unlive string"); setgcrefr(*p, o->gch.nextgc); lj_str_free(g, gco2str(o)); } @@ -454,14 +1329,14 @@ static void gc_sweepstr(global_State *g, GCRef *chain) } /* Check whether we can clear a key or a value slot from a table. */ -static int gc_mayclear(cTValue *o, int val) +static int gc_mayclear(global_State *g, cTValue *o, int val) { if (tvisgcv(o)) { /* Only collectable objects can be weak references. */ if (tvisstr(o)) { /* But strings cannot be used as weak references. */ - gc_mark_str(strV(o)); /* And need to be marked. */ + gc_mark_str(g, strV(o)); /* And need to be marked. */ return 0; } - if (iswhite(gcV(o))) + if (iswhite(g, gcV(o))) return 1; /* Object is about to be collected. */ if (tvisudata(o) && val && isfinalized(udataV(o))) return 1; /* Finalized userdata is dropped only from values. */ @@ -475,13 +1350,17 @@ static void gc_clearweak(global_State *g, GCobj *o) UNUSED(g); while (o) { GCtab *t = gco2tab(o); - lj_assertG((t->marked & LJ_GC_WEAK), "clear of non-weak table"); - if ((t->marked & LJ_GC_WEAKVAL)) { + if ((t->gcflags & LJ_GC_WEAK) != LJ_GC_WEAKVAL) { + /* Need to mark & relocate hash part */ + gc_mark_tab_hash(g, t); + } + lj_assertG((t->gcflags & LJ_GC_WEAK), "clear of non-weak table"); + if ((t->gcflags & LJ_GC_WEAKVAL)) { MSize i, asize = t->asize; for (i = 0; i < asize; i++) { /* Clear array slot when value is about to be collected. */ TValue *tv = arrayslot(t, i); - if (gc_mayclear(tv, 1)) + if (gc_mayclear(g, tv, 1)) setnilV(tv); } } @@ -491,8 +1370,8 @@ static void gc_clearweak(global_State *g, GCobj *o) for (i = 0; i <= hmask; i++) { Node *n = &node[i]; /* Clear hash slot when key or value is about to be collected. */ - if (!tvisnil(&n->val) && (gc_mayclear(&n->key, 0) || - gc_mayclear(&n->val, 1))) + if (!tvisnil(&n->val) && (gc_mayclear(g, &n->key, 0) || + gc_mayclear(g, &n->val, 1))) setnilV(&n->val); } } @@ -531,6 +1410,17 @@ static void gc_call_finalizer(global_State *g, lua_State *L, } } +static GCobj *gc_finalize_obj(lua_State *L, GCobj *o) +{ + global_State *g = G(L); + cTValue *mo; + lj_assertG(tvref(g->jit_base) == NULL, "finalizer called on trace"); + mo = lj_meta_fastg(g, tabref(o->gch.metatable), MM_gc); + if (mo) + gc_call_finalizer(g, L, mo, o); + return gcref(o->gch.gclist); +} + /* Finalize one userdata or cdata object from the mmudata list. */ static void gc_finalize(lua_State *L) { @@ -549,8 +1439,7 @@ static void gc_finalize(lua_State *L) /* Add cdata back to the GC list and make it white. */ setgcrefr(o->gch.nextgc, g->gc.root); setgcref(g->gc.root, o); - makewhite(g, o); - o->gch.marked &= (uint8_t)~LJ_GC_CDATA_FIN; + o->gch.gcflags &= (uint8_t)~LJ_GC_CDATA_FIN; /* Resolve finalizer. */ setcdataV(L, &tmp, gco2cd(o)); tv = lj_tab_set(L, ctype_ctsG(g)->finalizer, &tmp); @@ -566,7 +1455,6 @@ static void gc_finalize(lua_State *L) /* Add userdata back to the main userdata list and make it white. */ setgcrefr(o->gch.nextgc, mainthread(g)->nextgc); setgcref(mainthread(g)->nextgc, o); - makewhite(g, o); /* Resolve the __gc metamethod. */ mo = lj_meta_fastg(g, tabref(gco2ud(o)->metatable), MM_gc); if (mo) @@ -576,8 +1464,9 @@ static void gc_finalize(lua_State *L) /* Finalize all userdata objects from mmudata list. */ void lj_gc_finalize_udata(lua_State *L) { - while (gcref(G(L)->gc.mmudata) != NULL) - gc_finalize(L); + global_State *g = G(L); + while (gcref(g->gc.fin_list) != NULL) + setgcref(g->gc.fin_list, gc_finalize_obj(L, gcref(g->gc.fin_list))); } #if LJ_HASFFI @@ -595,8 +1484,7 @@ void lj_gc_finalize_cdata(lua_State *L) if (!tvisnil(&node[i].val) && tviscdata(&node[i].key)) { GCobj *o = gcV(&node[i].key); TValue tmp; - makewhite(g, o); - o->gch.marked &= (uint8_t)~LJ_GC_CDATA_FIN; + o->gch.gcflags &= (uint8_t)~LJ_GC_CDATA_FIN; copyTV(L, &tmp, &node[i].val); setnilV(&node[i].val); gc_call_finalizer(g, L, &tmp, o); @@ -608,13 +1496,25 @@ void lj_gc_finalize_cdata(lua_State *L) /* Free all remaining GC objects. */ void lj_gc_freeall(global_State *g) { + GCArenaHdr *a; MSize i, strmask; /* Free everything, except super-fixed objects (the main thread). */ - g->gc.currentwhite = LJ_GC_WHITES | LJ_GC_SFIXED; + g->gc.safecolor = LJ_GC_SFIXED; gc_fullsweep(g, &g->gc.root); strmask = g->str.mask; for (i = 0; i <= strmask; i++) /* Free all string hash chains. */ gc_sweepstr(g, &g->str.tab[i]); + /* Only track malloced data from this point. */ + g->gc.total = g->gc.malloc; + + g->gc.currentsweep ^= LJ_GC_SWEEPS; + + /* Some objects may contain malloced data and may not get collected. */ + for (a = g->gc.udata; a; a = a->next) { + GCAudata *ud = (GCAudata *)a; + memset(ud->mark, 0, sizeof(ud->mark)); + gc_sweep_udata1(g, ud); + } } /* -- Collector ----------------------------------------------------------- */ @@ -624,35 +1524,101 @@ static void atomic(global_State *g, lua_State *L) { size_t udsize; - gc_mark_uv(g); /* Need to remark open upvalues (the thread may be dead). */ - gc_propagate_gray(g); /* Propagate any left-overs. */ - - setgcrefr(g->gc.gray, g->gc.weak); /* Empty the list of weak tables. */ setgcrefnull(g->gc.weak); - lj_assertG(!iswhite(obj2gco(mainthread(g))), "main thread turned white"); + setgcrefnull(g->gc.ephemeron); + lj_assertG(!iswhite(g, obj2gco(mainthread(g))), "main thread turned white"); gc_markobj(g, L); /* Mark running thread. */ gc_traverse_curtrace(g); /* Traverse current trace. */ gc_mark_gcroot(g); /* Mark GC roots (again). */ - gc_propagate_gray(g); /* Propagate all of the above. */ - setgcrefr(g->gc.gray, g->gc.grayagain); /* Empty the 2nd chance list. */ + /* Empty the 2nd chance list. */ + propagatemark_again(g); + /* Propagate any leftovers. Ephemeron processing clears the gray queue */ + process_ephemerons(g); + + sweep_upvals(g); + setgcrefnull(g->gc.grayagain); - gc_propagate_gray(g); /* Propagate it. */ - udsize = lj_gc_separateudata(g, 0); /* Separate userdata to be finalized. */ - gc_mark_mmudata(g); /* Mark them. */ - udsize += gc_propagate_gray(g); /* And propagate the marks. */ + setgcrefnull(g->gc.fin_list); + gc_presweep_fintab(g, (GCAtab*)g->gc.fintab); + gc_presweep_udata(g, (GCAudata *)g->gc.udata); + udsize = gc_propagate_gray(g); /* All marking done, clear weak tables. */ gc_clearweak(g, gcref(g->gc.weak)); + gc_clearweak(g, gcref(g->gc.ephemeron)); lj_buf_shrink(L, &g->tmpbuf); /* Shrink temp buffer. */ /* Prepare for sweep phase. */ - g->gc.currentwhite = (uint8_t)otherwhite(g); /* Flip current white. */ - g->strempty.marked = g->gc.currentwhite; + /* Gray is for strings which are gray while sweeping */ + g->gc.safecolor = g->gc.currentblack | LJ_GC_GRAY | LJ_GC_FIXED | LJ_GC_SFIXED; + if (!isminor(g)) { + /* Need to keep the thread list around */ + setgcrefnull(g->gc.grayagain_th); + g->gc.currentblack ^= LJ_GC_BLACKS; + g->gc.currentblackgray ^= LJ_GC_BLACKS; + } + g->gc.currentsweep ^= LJ_GC_SWEEPS; setmref(g->gc.sweep, &g->gc.root); + + /* Expected memory consumption is everything that has been malloced + + * everything that arena traversal found as by definition we only keep + * things that traversal found. This can be inaccurate if object vectors + * have been resized post-marking but that's fine, it will get corrected + * next cycle anyway. + * This is also why we cannot just assert that total >= malloc + accum + * even though in practice that will almost always hold. + */ + g->gc.total = g->gc.malloc + g->gc.accum; g->gc.estimate = g->gc.total - (GCSize)udsize; /* Initial estimate. */ + + /* We must clear the first arena of each type in here as the allocator + * only checks when a new arena is acquired. Alternately a new arena + * can be assigned. This is because new objects will not have the mark bit set + * and would mistakenly get swept. They will also have incorrect object bits + * but those don't matter. + */ + gc_sweep_tab1(g, (GCAtab *)g->gc.tab); + gc_sweep_fintab1(g, (GCAtab *)g->gc.fintab); + gc_sweep_func1(g, (GCAfunc *)g->gc.func); + gc_sweep_uv1(g, (GCAupval *)g->gc.uv); + gc_sweep_udata1(g, (GCAudata *)g->gc.udata); + + lj_assertG(g->gc.bloblist_wr > 0, "no blobs?"); + g->gc.bloblist_sweep = g->gc.bloblist_wr - 2; + if (!isminor(g)) + g->gc.bloblist_usage[g->gc.bloblist_wr - 1] = 0; +} + +static void gc_sweepblobs(global_State *g) +{ + GCAblob **list = g->gc.bloblist; + uint32_t *usage = g->gc.bloblist_usage; + for (int32_t i = g->gc.bloblist_sweep; i >= 0; i--) { + lj_assertG(list[i]->id == i, "id invariant violated"); + if (!usage[i]) { + GCAblob *a = list[i]; + list[i] = list[--g->gc.bloblist_wr]; + list[i]->id = i; + if (a->flags & GCA_BLOB_HUGE) + lj_arena_freehuge(&g->gc.ctx, a, a->alloc); + else + lj_arena_free(&g->gc.ctx, a); + } else if (usage[i] < BLOB_REAP_THRESHOLD) { + list[i]->flags |= GCA_BLOB_REAP; + } + if (!isminor(g)) + usage[i] = 0; + } +} + +static void *find_unswept(global_State *g, GCArenaHdr *a) +{ + while (a && (a->flags & LJ_GC_SWEEPS) == g->gc.currentsweep) + a = a->next; + return a; } /* GC state machine. Returns a cost estimate for each step performed. */ @@ -666,7 +1632,9 @@ static size_t gc_onestep(lua_State *L) case GCSpropagate: if (gcref(g->gc.gray) != NULL) return propagatemark(g); /* Propagate one gray object. */ - g->gc.state = GCSatomic; /* End of mark phase. */ + if (g->gc.gray_head != NULL) + return propagatemark_arena(g, GCSTEPSIZE); + g->gc.state = GCSatomic; /* End of mark phase. */ return 0; case GCSatomic: if (tvref(g->jit_base)) /* Don't run atomic phase on trace. */ @@ -682,7 +1650,7 @@ static size_t gc_onestep(lua_State *L) g->gc.state = GCSsweep; /* All string hash chains sweeped. */ lj_assertG(old >= g->gc.total, "sweep increased memory"); g->gc.estimate -= old - g->gc.total; - return GCSWEEPCOST; + return 0; } case GCSsweep: { GCSize old = g->gc.total; @@ -690,20 +1658,78 @@ static size_t gc_onestep(lua_State *L) lj_assertG(old >= g->gc.total, "sweep increased memory"); g->gc.estimate -= old - g->gc.total; if (gcref(*mref(g->gc.sweep, GCRef)) == NULL) { - if (g->str.num <= (g->str.mask >> 2) && g->str.mask > LJ_MIN_STRTAB*2-1) - lj_str_resize(L, g->str.mask >> 1); /* Shrink string table. */ - if (gcref(g->gc.mmudata)) { /* Need any finalizations? */ - g->gc.state = GCSfinalize; + if (g->str.num <= (g->str.mask >> 2) && + g->str.mask > LJ_MIN_STRTAB * 2 - 1) { + lj_str_resize(L, g->str.mask >> 1); /* Shrink string table. */ + } + g->gc.state = GCSsweep_blob; + } + /* TODO: make this non-atomic again */ + return 0; + } + case GCSsweep_blob: { + if (~g->gc.bloblist_sweep) + gc_sweepblobs(g); + g->gc.state = GCSsweep_func; + setmref(g->gc.sweep, find_unswept(g, g->gc.func->next)); + return GCSWEEPCOST; + } + case GCSsweep_func: + if (mrefu(g->gc.sweep)) { + setmref(g->gc.sweep, gc_sweep_func(g, mref(g->gc.sweep, GCAfunc), 10)); + } else { + g->gc.state = GCSsweep_tab; + setmref(g->gc.sweep, find_unswept(g, g->gc.tab->next)); + } + return GCSWEEPCOST; + case GCSsweep_tab: + if (mrefu(g->gc.sweep)) { + setmref(g->gc.sweep, gc_sweep_tab(g, mref(g->gc.sweep, GCAtab), 10)); + } else { + g->gc.state = GCSsweep_fintab; + setmref(g->gc.sweep, find_unswept(g, g->gc.fintab->next)); + } + return GCSWEEPCOST; + case GCSsweep_fintab: + if (mrefu(g->gc.sweep)) { + setmref(g->gc.sweep, gc_sweep_fintab(g, mref(g->gc.sweep, GCAtab), 10)); + } else { + g->gc.state = GCSsweep_uv; + setmref(g->gc.sweep, find_unswept(g, g->gc.uv->next)); + } + return GCSWEEPCOST; + case GCSsweep_uv: + if (mrefu(g->gc.sweep)) { + setmref(g->gc.sweep, gc_sweep_uv(g, mref(g->gc.sweep, GCAupval), 10)); + } else { + setmref(g->gc.sweep, find_unswept(g, g->gc.udata->next)); + g->gc.state = GCSsweep_udata; + } + return GCSWEEPCOST; + case GCSsweep_udata: + if (mrefu(g->gc.sweep)) { + setmref(g->gc.sweep, gc_sweep_udata1(g, mref(g->gc.sweep, GCAudata))); + } else { + g->gc.state = GCSfinalize_arena; + } + return GCSWEEPCOST; + case GCSfinalize_arena: + if (gcrefu(g->gc.fin_list)) { + if (tvref(g->jit_base)) /* Don't call finalizers on trace. */ + return LJ_MAX_MEM; + setgcref(g->gc.fin_list, gc_finalize_obj(L, gcref(g->gc.fin_list))); + } else { + if (gcref(g->gc.mmudata)) { /* Need any finalizations? */ + g->gc.state = GCSfinalize; #if LJ_HASFFI - g->gc.nocdatafin = 1; + g->gc.nocdatafin = 1; #endif } else { /* Otherwise skip this phase to help the JIT. */ - g->gc.state = GCSpause; /* End of GC cycle. */ - g->gc.debt = 0; + g->gc.state = GCSpause; /* End of GC cycle. */ + g->gc.debt = 0; } } - return GCSWEEPMAX*GCSWEEPCOST; - } + return GCSWEEPCOST; case GCSfinalize: if (gcref(g->gc.mmudata) != NULL) { GCSize old = g->gc.total; @@ -782,25 +1808,16 @@ int LJ_FASTCALL lj_gc_step_jit(global_State *g, MSize steps) #endif /* Perform a full GC cycle. */ -void lj_gc_fullgc(lua_State *L) +void lj_gc_fullgc(lua_State *L, int maximal) { global_State *g = G(L); int32_t ostate = g->vmstate; setvmstate(g, GC); - if (g->gc.state <= GCSatomic) { /* Caught somewhere in the middle. */ - setmref(g->gc.sweep, &g->gc.root); /* Sweep everything (preserving it). */ - setgcrefnull(g->gc.gray); /* Reset lists from partial propagation. */ - setgcrefnull(g->gc.grayagain); - setgcrefnull(g->gc.weak); - g->gc.state = GCSsweepstring; /* Fast forward to the sweep phase. */ - g->gc.sweepstr = 0; + /* Finish any previous cycle or sweep in progress. */ + if (g->gc.state > (maximal ? GCSpause : GCSatomic)) { + do { gc_onestep(L); } while (g->gc.state != GCSpause); } - while (g->gc.state == GCSsweepstring || g->gc.state == GCSsweep) - gc_onestep(L); /* Finish sweep. */ - lj_assertG(g->gc.state == GCSfinalize || g->gc.state == GCSpause, - "bad GC state"); /* Now perform a full GC. */ - g->gc.state = GCSpause; do { gc_onestep(L); } while (g->gc.state != GCSpause); g->gc.threshold = (g->gc.estimate/100) * g->gc.pause; g->vmstate = ostate; @@ -811,51 +1828,23 @@ void lj_gc_fullgc(lua_State *L) /* Move the GC propagation frontier forward. */ void lj_gc_barrierf(global_State *g, GCobj *o, GCobj *v) { - lj_assertG(isblack(o) && iswhite(v) && !isdead(g, v) && !isdead(g, o), + lj_assertG(isblack(g, o) && iswhite(g, v) && !checkdead(g, v) && !checkdead(g, o), "bad object states for forward barrier"); lj_assertG(g->gc.state != GCSfinalize && g->gc.state != GCSpause, "bad GC state"); lj_assertG(o->gch.gct != ~LJ_TTAB, "barrier object is not a table"); /* Preserve invariant during propagation. Otherwise it doesn't matter. */ - if (g->gc.state == GCSpropagate || g->gc.state == GCSatomic) - gc_mark(g, v); /* Move frontier forward. */ - else - makewhite(g, o); /* Make it white to avoid the following barrier. */ + if (g->gc.state == GCSpropagate || g->gc.state == GCSatomic) { + gc_markobj(g, v); /* Move frontier forward. */ + } else { + makewhite(o); /* Make it white to avoid the following barrier. */ + } } /* Specialized barrier for closed upvalue. Pass &uv->tv. */ void LJ_FASTCALL lj_gc_barrieruv(global_State *g, TValue *tv) { -#define TV2MARKED(x) \ - (*((uint8_t *)(x) - offsetof(GCupval, tv) + offsetof(GCupval, marked))) - if (g->gc.state == GCSpropagate || g->gc.state == GCSatomic) - gc_mark(g, gcV(tv)); - else - TV2MARKED(tv) = (TV2MARKED(tv) & (uint8_t)~LJ_GC_COLORS) | curwhite(g); -#undef TV2MARKED -} - -/* Close upvalue. Also needs a write barrier. */ -void lj_gc_closeuv(global_State *g, GCupval *uv) -{ - GCobj *o = obj2gco(uv); - /* Copy stack slot to upvalue itself and point to the copy. */ - copyTV(mainthread(g), &uv->tv, uvval(uv)); - setmref(uv->v, &uv->tv); - uv->closed = 1; - setgcrefr(o->gch.nextgc, g->gc.root); - setgcref(g->gc.root, o); - if (isgray(o)) { /* A closed upvalue is never gray, so fix this. */ - if (g->gc.state == GCSpropagate || g->gc.state == GCSatomic) { - gray2black(o); /* Make it black and preserve invariant. */ - if (tviswhite(&uv->tv)) - lj_gc_barrierf(g, o, gcV(&uv->tv)); - } else { - makewhite(g, o); /* Make it white, i.e. sweep the upvalue. */ - lj_assertG(g->gc.state != GCSfinalize && g->gc.state != GCSpause, - "bad GC state"); - } - } + gc_marktv(g, tv); } #if LJ_HASJIT @@ -881,6 +1870,7 @@ void *lj_mem_realloc(lua_State *L, void *p, GCSize osz, GCSize nsz) lj_assertG(checkptrGC(p), "allocated memory address %p outside required range", p); g->gc.total = (g->gc.total - osz) + nsz; + g->gc.malloc = (g->gc.malloc - osz) + nsz; return p; } @@ -894,9 +1884,10 @@ void * LJ_FASTCALL lj_mem_newgco(lua_State *L, GCSize size) lj_assertG(checkptrGC(o), "allocated memory address %p outside required range", o); g->gc.total += size; + g->gc.malloc += size; setgcrefr(o->gch.nextgc, g->gc.root); setgcref(g->gc.root, o); - newwhite(g, o); + newwhite(o); return o; } @@ -913,3 +1904,432 @@ void *lj_mem_grow(lua_State *L, void *p, MSize *szp, MSize lim, MSize esz) return p; } + +int checkdead(global_State *g, GCobj *o) +{ + if (g->gc.state <= GCSatomic) + return 0; /* Nothing can be dead before atomic finishes */ + if (is_arena_obj(o->gch.gct)) { + /* The top 3 bits for arena types have different meanings */ + if ((g->gc.safecolor & o->gch.gcflags & ~LJ_GC_MARK_MASK)) + return 0; /* Anything marked with the safe colour is live */ + /* Anything living in a swept arena is live */ + return !(arena(o)->hdr.flags & g->gc.currentsweep); + } else { + if (g->gc.safecolor & o->gch.gcflags) + return 0; /* Anything marked with the safe colour is live */ + /* Anything past sweep is live */ + return !(g->gc.state > GCSsweep); + } +} + + + +/* Arena allocator */ + + +#define relink(freehead, head) \ + { \ + GCArenaHdr *fh = freehead; \ + GCArenaHdr *fn = fh->freenext; \ + if (fn) \ + fn->freeprev = NULL; \ + freehead = fn; \ + fh->prev->next = fh->next; \ + if (fh->next) \ + fh->next->prev = fh->prev; \ + head->prev = fh; \ + fh->prev = NULL; \ + fh->next = head; \ + } + +/* All bitmap arenas are fundamentally the same so we can macro all of these. + * Note that each struct has a different layout. + * Everything other than the free bitmap can be zeroed. + * If we are reusing an arena we need to move it to the front of the queue for + * the type and possibly sweep it + */ +#define NEW_ARENA(fn, atype, otype, idtype, var, freevar, sweepfn, ...) \ + static atype *fn(global_State *g) \ + { \ + atype *o; \ + if (LJ_LIKELY(g->gc.freevar)) { \ + o = (atype *)g->gc.freevar; \ + lj_assertG(o->free_h != 0, "no free elements in freelist?"); \ + relink(g->gc.freevar, g->gc.var); \ + g->gc.var = &o->hdr; \ + o->hdr.freenext = o->hdr.freeprev = NULL; \ + if (LJ_UNLIKELY(!(g->gc.currentsweep & o->hdr.flags))) { \ + if (LJ_UNLIKELY(mref(g->gc.sweep, atype) == o)) { \ + setmref(g->gc.sweep, o->hdr.next); \ + } \ + sweepfn##1(g, o); \ + } \ + return o; \ + } \ + o = (atype *)lj_arena_alloc(&g->gc.ctx); \ + if (LJ_UNLIKELY(!o)) \ + lj_err_mem(&gcref(g->cur_L)->th); \ + do_arena_init(o, g, idtype, atype, otype); \ + g->gc.var->prev = &o->hdr; \ + o->hdr.next = g->gc.var; \ + g->gc.var = &o->hdr; \ + __VA_ARGS__ \ + return o; \ + } + +/* All bitmap allocators are basically the same */ +#define BM_ALLOC(type, arena, newfn, otype) \ + global_State *g = G(L); \ + uint32_t i, j; \ + uint64_t f; \ + otype *x; \ + type *o = (type *)g->gc.arena; \ + if (LJ_UNLIKELY(!o->free_h)) { \ + o = newfn(g); \ + } \ + i = tzcount64(o->free_h); \ + lj_assertG(o->free[i] != 0, "no free elemnts"); \ + j = tzcount64(o->free[i]); \ + lj_assertG((i << 6) + j >= ELEMENTS_OCCUPIED(type, otype), "bad arena"); \ + f = reset_lowest64(o->free[i]); \ + o->free[i] = f; \ + if (!f) \ + o->free_h = reset_lowest64(o->free_h); \ + x = &((otype *)o)[(i << 6) + j]; \ + lj_assertG((char *)x + sizeof(otype) - (char *)o <= ARENA_SIZE, "out of bounds") + +NEW_ARENA(lj_arena_tab, GCAtab, GCtab, ~LJ_TTAB, tab, free_tab, gc_sweep_tab) +NEW_ARENA(lj_arena_fintab, GCAtab, GCtab, ~LJ_TTAB, fintab, free_fintab, + gc_sweep_fintab) +NEW_ARENA(lj_arena_uv, GCAupval, GCupval, ~LJ_TUPVAL, uv, free_uv, gc_sweep_uv) +NEW_ARENA(lj_arena_func, GCAfunc, GCfunc, ~LJ_TFUNC, func, free_func, + gc_sweep_func) +NEW_ARENA(lj_arena_udata, GCAudata, GCudata, ~LJ_TUDATA, udata, free_udata, + gc_sweep_udata, o->free4_h = o->free_h;) + +static void lj_arena_newblobspace(global_State *g) +{ + if (LJ_UNLIKELY(g->gc.bloblist_wr == g->gc.bloblist_alloc)) { + uint32_t old = g->gc.bloblist_alloc; + g->gc.bloblist_alloc *= 2; + g->gc.bloblist = + (GCAblob **)g->allocf(g->allocd, g->gc.bloblist, old * sizeof(void *), + g->gc.bloblist_alloc * sizeof(void *)); + g->gc.bloblist_usage = (uint32_t *)g->allocf( + g->allocd, g->gc.bloblist_usage, old * sizeof(uint32_t), + g->gc.bloblist_alloc * sizeof(uint32_t)); + } +} + +static GCAblob *lj_arena_blob(global_State *g) +{ + uint32_t id; + GCAblob *o = (GCAblob *)lj_arena_alloc(&g->gc.ctx); + o->alloc = sizeof(GCAblob); + o->flags = 0; + g->gc.blob_generic = o; + id = g->gc.bloblist_wr++; + o->id = id; + g->gc.bloblist[id] = o; + g->gc.bloblist_usage[id] = 0; + return o; +} + +GCtab *lj_mem_alloctab(lua_State *L, uint32_t asize) +{ + global_State *g = G(L); + uint32_t i, j; + uint64_t f; + GCtab *x; + GCAtab *o = (GCAtab *)g->gc.tab; + void *blob = NULL; + uint8_t newf = 0; + uint32_t n = (asize * sizeof(TValue) + sizeof(GCtab) - 1) / sizeof(GCtab); + if (LJ_UNLIKELY(!o->free_h)) { + o = lj_arena_tab(g); + } + i = tzcount64(o->free_h); + lj_assertG(o->free[i] != 0, "no free elemnts"); + j = tzcount64(o->free[i]); + f = reset_lowest64(o->free[i]); + if (n > 0 && n <= 3) { + uint64_t k = o->free[i]; + /* Shift 1 if n is 1 or 2, 2 if n is 3*/ + k &= k >> ((n >> 1) + (n & 1)); + /* Shift 1 if n is 2 or 3 */ + k &= k >> (n >> 1); + if (k) { + j = tzcount64(k); + f = o->free[i] ^ (((1ull << (n + 1)) - 1) << j); + newf = size2flags(n + 1); + blob = &((GCtab *)o)[(i << 6) + j + 1]; + } + } + + lj_assertG((i << 6) + j >= ELEMENTS_OCCUPIED(GCAtab, GCtab), "bad arena"); + o->free[i] = f; + if (!f) + o->free_h = reset_lowest64(o->free_h); + x = &((GCtab *)o)[(i << 6) + j]; + lj_assertG((char *)x + sizeof(GCtab) - (char *)o <= ARENA_SIZE, + "out of bounds"); + + x->gcflags = newf; + x->gct = ~LJ_TTAB; + x->nomm = (uint8_t)~0; + x->colo = blob ? (int8_t)asize : 0; + x->asize = asize; + x->hmask = 0; + setgcrefnull(x->metatable); + if (!blob && asize > 0) { + if (asize > LJ_MAX_ASIZE) + lj_err_msg(L, LJ_ERR_TABOV); + blob = lj_mem_newv(L, asize, TValue); + } + setmref(x->array, blob); + g->gc.total += sizeof(GCtab) + sizeof(TValue) * asize; + return x; +} + +GCtab *lj_mem_alloctabempty_gc(lua_State *L) +{ + BM_ALLOC(GCAtab, fintab, lj_arena_fintab, GCtab); + + x->gcflags = 0; + x->gct = ~LJ_TTAB; + x->nomm = (uint8_t)~0; + x->colo = 0; + setmref(x->array, NULL); + setgcrefnull(x->metatable); + x->asize = 0; + g->gc.total += sizeof(GCtab); + return x; +} + +GCstr *lj_mem_allocstr(lua_State *L, MSize len) +{ + GCstr *str = lj_mem_newt(L, lj_str_size(len), GCstr); + return str; +} + +GCupval *lj_mem_allocuv(lua_State *L) +{ + BM_ALLOC(GCAupval, uv, lj_arena_uv, GCupval); + g->gc.total += sizeof(GCupval); + return x; +} + +static GCudata *lj_mem_allocudatamerged(lua_State *L, uint32_t n, GCAudata *a) +{ + GCudata *ud; + while (1) { + uint32_t i = tzcount64(a->free4_h); + uint64_t q = a->free[i]; + q &= q >> 2; + q &= q >> 1; + if (!q) { + a->free4_h = reset_lowest32(a->free4_h); + if (!a->free4_h) { + return NULL; + } + continue; + } + uint32_t j = tzcount64(q); + + a->free[i] ^= ((1ull << n) - 1) << j; + if (!a->free[i]) + a->free_h ^= 1ull << i; + + ud = aobj(a, GCudata, (i << 6) + j); + /* newwhite(ud); */ /* Not finalized. */ + ud->gct = ~LJ_TUDATA; + ud->gcflags = size2flags(n); + setmref(ud->payload, ud + 1); + G(L)->gc.total += sizeof(GCudata); + return ud; + } +} + +GCudata *lj_mem_allocudata(lua_State *L, MSize bytes) +{ + uint32_t n = (bytes + 2 * sizeof(GCudata) - 1) / sizeof(GCudata); + global_State *g = G(L); + GCAudata *o = (GCAudata *)g->gc.udata; + GCudata *ud; + if (!o->free_h) { + o = lj_arena_udata(g); + } + if (n > 1 && n <= 4) { + ud = lj_mem_allocudatamerged(L, n, o); + if (ud) + return ud; + o = (GCAudata *)g->gc.free_udata; + for (; o; o = (GCAudata*)o->hdr.freenext) { + ud = lj_mem_allocudatamerged(L, n, o); + if (ud) { + if (!o->free_h) { + /* If we allocate the last free slots in an arena + * we have to remove it from the freelist */ + if (&o->hdr == g->gc.free_udata) { + g->gc.free_udata = o->hdr.freenext; + if (o->hdr.freenext) + o->hdr.freeprev = NULL; + } else { + o->hdr.freeprev->freenext = o->hdr.freenext; + if (o->hdr.freenext) + o->hdr.freenext->freeprev = o->hdr.freeprev; + } + } + return ud; + } + } + o = lj_arena_udata(g); + ud = lj_mem_allocudatamerged(L, n, o); + return ud; + } + uint32_t i = tzcount64(o->free_h); + uint32_t j = tzcount64(o->free[i]); + uint64_t x = reset_lowest64(o->free[i]); + o->free[i] = x; + if (!x) { + o->free_h &= ~abit(i); + o->free4_h &= ~abit(i); + } + ud = aobj(o, GCudata, (i << 6) + j); + g->gc.malloc += bytes; + setmref(ud->payload, (bytes > 0) ? g->allocf(g->allocd, NULL, 0, bytes) : NULL); + newwhite(ud); /* Not finalized. */ + ud->gct = ~LJ_TUDATA; + return ud; +} + +GCfunc *lj_mem_allocfunc(lua_State *L, MSize bytes) +{ + global_State *g = G(L); + uint32_t i, j; + uint64_t f; + GCfunc *x; + GCAfunc *o = (GCAfunc *)g->gc.func; + void *blob = NULL; + uint8_t newf = 0; + uint32_t n = (bytes + sizeof(GCfunc) - 1) / sizeof(GCfunc); + if (LJ_UNLIKELY(!o->free_h)) { + o = lj_arena_func(g); + } + i = tzcount64(o->free_h); + lj_assertG(o->free[i] != 0, "no free elemnts"); + j = tzcount64(o->free[i]); + f = reset_lowest64(o->free[i]); + if (n > 0 && n <= 3) { + uint64_t k = o->free[i]; + /* Shift 1 if n is 1 or 2, 2 if n is 3*/ + k &= k >> ((n >> 1) + (n & 1)); + /* Shift 1 if n is 2 or 3 */ + k &= k >> (n >> 1); + if (k) { + j = tzcount64(k); + f = o->free[i] ^ (((1ull << (n + 1)) - 1) << j); + newf = size2flags(n + 1); + blob = &((GCfunc *)o)[(i << 6) + j + 1]; + } + } + + lj_assertG((i << 6) + j >= ELEMENTS_OCCUPIED(GCAfunc, GCfunc), "bad arena"); + o->free[i] = f; + if (!f) + o->free_h = reset_lowest64(o->free_h); + x = &((GCfunc *)o)[(i << 6) + j]; + lj_assertG((char *)x + sizeof(GCfunc) - (char *)o <= ARENA_SIZE, + "out of bounds"); + + setmref(x->gen.data, blob ? blob : lj_mem_newblob_g(G(L), bytes)); + x->gen.gcflags = newf; + x->gen.gct = ~LJ_TFUNC; + g->gc.total += bytes + sizeof(GCfunc); + return x; +} + +static void *lj_mem_newblob_g(global_State *g, MSize sz) +{ + GCAblob *a = g->gc.blob_generic; + void *ret; + sz = (sz + 15) & ~15u; + if (LJ_UNLIKELY(sz > ARENA_HUGE_THRESHOLD)) { + uint32_t id; + lj_arena_newblobspace(g); + id = g->gc.bloblist_wr++; + a = (GCAblob *)lj_arena_allochuge(&g->gc.ctx, sz + sizeof(GCAblob)); + a->alloc = sizeof(GCAblob); + a->flags = GCA_BLOB_HUGE; + /* The current blob must always be the last one so we have to shift it */ + a->id = id - 1; + g->gc.bloblist[id - 1]->id = id; + g->gc.bloblist[id] = g->gc.bloblist[id - 1]; + g->gc.bloblist[id - 1] = a; + g->gc.bloblist_usage[id] = g->gc.bloblist_usage[id - 1]; + g->gc.bloblist_usage[id - 1] = 0; + } else if (a->alloc + sz > ARENA_SIZE) { + lj_arena_newblobspace(g); + a = lj_arena_blob(g); + } + + ret = (char *)a + a->alloc; + a->alloc += sz; + return ret; +} + +void *lj_mem_newblob(lua_State *L, MSize sz) +{ + G(L)->gc.total += sz; + return lj_mem_newblob_g(G(L), sz); +} + +void *lj_mem_reallocblob(lua_State *L, void *p, MSize osz, MSize nsz) +{ + global_State *g = G(L); + GCAblob *a; + g->gc.total = (g->gc.total - osz) + nsz; + if (!osz) + return lj_mem_newblob_g(g, nsz); + osz = (osz + 15) & ~15u; + nsz = (nsz + 15) & ~15u; + if (nsz <= osz) { + if (!nsz) + return NULL; + return p; + } + if (((char *)g->gc.blob_generic + g->gc.blob_generic->alloc - osz) == p) { + /* We *can* resize if no more allocations have occurred */ + MSize d = nsz - osz; + if (g->gc.blob_generic->alloc + d <= ARENA_SIZE) { + g->gc.blob_generic->alloc += (uint32_t)d; + return p; + } + } + + a = gcablob(p); + if (a->flags & GCA_BLOB_HUGE) { + GCAblob *newp = (GCAblob *)lj_arena_reallochuge( + &g->gc.ctx, a, osz + sizeof(GCAblob), nsz + sizeof(GCAblob)); + if (!newp) + lj_err_mem(L); + + g->gc.bloblist[newp->id] = newp; + newp->alloc = sizeof(GCAblob) + nsz; + return newp + 1; + } + + void *r = lj_mem_newblob_g(g, nsz); + if (!r) + lj_err_mem(L); + memcpy(r, p, osz); + return r; +} + +void lj_mem_registergc_udata(lua_State *L, GCudata *ud) +{ + GCAudata *a = gcat(ud, GCAudata); + uint32_t idx = aidx(ud); + a->fin_req[aidxh(idx)] |= abit(aidxl(idx)); +} diff --git a/src/lj_gc.h b/src/lj_gc.h index 3ed790ca..0e0721c3 100644 --- a/src/lj_gc.h +++ b/src/lj_gc.h @@ -7,46 +7,71 @@ #define _LJ_GC_H #include "lj_obj.h" +#include "lj_jit.h" +#include "lj_intrin.h" /* Garbage collector states. Order matters. */ enum { - GCSpause, GCSpropagate, GCSatomic, GCSsweepstring, GCSsweep, GCSfinalize + GCSpause, GCSpropagate, GCSatomic, GCSsweepstring, GCSsweep, + GCSsweep_blob, GCSsweep_func, GCSsweep_tab, GCSsweep_fintab, + GCSsweep_uv, GCSsweep_udata, GCSfinalize_arena, GCSfinalize }; /* Bitmasks for marked field of GCobj. */ -#define LJ_GC_WHITE0 0x01 -#define LJ_GC_WHITE1 0x02 -#define LJ_GC_BLACK 0x04 +#define LJ_GC_BLACK0 0x01 +#define LJ_GC_BLACK1 0x02 +#define LJ_GC_GRAY 0x04 #define LJ_GC_FINALIZED 0x08 -#define LJ_GC_WEAKKEY 0x08 -#define LJ_GC_WEAKVAL 0x10 +#define LJ_GC_WEAKKEY 0x10 +#define LJ_GC_WEAKVAL 0x08 #define LJ_GC_CDATA_FIN 0x10 #define LJ_GC_FIXED 0x20 #define LJ_GC_SFIXED 0x40 -#define LJ_GC_WHITES (LJ_GC_WHITE0 | LJ_GC_WHITE1) -#define LJ_GC_COLORS (LJ_GC_WHITES | LJ_GC_BLACK) +#define LJ_GC_MARK_MASK 0xE0 + +#define size2flags(n) (((1u << (n - 1)) - 1) << 5) +#define flags2bitmask(o, bit) (((bitmap_t)(o)->gch.gcflags >> 5) << ((bit) + 1)) + +#define LJ_GCMODE_MINORSWEEP 0x01 +#define LJ_GCMODE_ENABLE_MINORSWEEP 0x02 + +#define LJ_GC_BLACKS (LJ_GC_BLACK0 | LJ_GC_BLACK1) +#define LJ_GC_COLORS (LJ_GC_BLACKS | LJ_GC_GRAY) #define LJ_GC_WEAK (LJ_GC_WEAKKEY | LJ_GC_WEAKVAL) +#define LJ_GC_SWEEP0 0x01 +#define LJ_GC_SWEEP1 0x02 +#define LJ_GC_SWEEPS (LJ_GC_SWEEP0 | LJ_GC_SWEEP1) + /* Macros to test and set GCobj colors. */ -#define iswhite(x) ((x)->gch.marked & LJ_GC_WHITES) -#define isblack(x) ((x)->gch.marked & LJ_GC_BLACK) -#define isgray(x) (!((x)->gch.marked & (LJ_GC_BLACK|LJ_GC_WHITES))) -#define tviswhite(x) (tvisgcv(x) && iswhite(gcV(x))) -#define otherwhite(g) (g->gc.currentwhite ^ LJ_GC_WHITES) -#define isdead(g, v) ((v)->gch.marked & otherwhite(g) & LJ_GC_WHITES) +#define iswhite(g, x) (!((x)->gch.gcflags & (g)->gc.currentblackgray)) +#define isblack(g, x) (((x)->gch.gcflags & LJ_GC_COLORS) == (g)->gc.currentblack) +#define isgray(x) ((x)->gch.gcflags & LJ_GC_GRAY) +#define tviswhite(g, x) (tvisgcv(x) && iswhite(g, gcV(x))) + +/* Death checking should only be done in asserts */ +LJ_FUNC int checkdead(global_State *g, GCobj *o); #define curwhite(g) ((g)->gc.currentwhite & LJ_GC_WHITES) -#define newwhite(g, x) (obj2gco(x)->gch.marked = (uint8_t)curwhite(g)) -#define makewhite(g, x) \ - ((x)->gch.marked = ((x)->gch.marked & (uint8_t)~LJ_GC_COLORS) | curwhite(g)) -#define flipwhite(x) ((x)->gch.marked ^= LJ_GC_WHITES) -#define black2gray(x) ((x)->gch.marked &= (uint8_t)~LJ_GC_BLACK) -#define fixstring(s) ((s)->marked |= LJ_GC_FIXED) -#define markfinalized(x) ((x)->gch.marked |= LJ_GC_FINALIZED) +#define newwhite(x) (obj2gco(x)->gch.gcflags = 0) +#define makewhite(x) \ + ((x)->gch.gcflags = ((x)->gch.gcflags & (uint8_t)~LJ_GC_COLORS)) +#define black2gray(x) ((x)->gch.gcflags |= (uint8_t)LJ_GC_GRAY) +#define fixstring(s) ((s)->gcflags |= LJ_GC_FIXED) +#define markfinalized(x) ((x)->gch.gcflags |= LJ_GC_FINALIZED) + +#define maybe_resurrect_str(g, s) \ + if (LJ_UNLIKELY(iswhite(g, obj2gco(s)) && (g)->gc.state == GCSsweepstring && \ + ((s)->hash & (g)->str.mask) >= (g)->gc.sweepstr)) \ + { \ + (s)->gcflags |= (g)->gc.currentblack; \ + } + +#define isminor(g) (g->gc.gcmode & LJ_GCMODE_MINORSWEEP) /* Collector. */ -LJ_FUNC size_t lj_gc_separateudata(global_State *g, int all); +LJ_FUNC void lj_gc_separateudata(global_State *g); LJ_FUNC void lj_gc_finalize_udata(lua_State *L); #if LJ_HASFFI LJ_FUNC void lj_gc_finalize_cdata(lua_State *L); @@ -59,7 +84,7 @@ LJ_FUNCA void LJ_FASTCALL lj_gc_step_fixtop(lua_State *L); #if LJ_HASJIT LJ_FUNC int LJ_FASTCALL lj_gc_step_jit(global_State *g, MSize steps); #endif -LJ_FUNC void lj_gc_fullgc(lua_State *L); +LJ_FUNC void lj_gc_fullgc(lua_State *L, int maximal); /* GC check: drive collector forward if the GC threshold has been reached. */ #define lj_gc_check(L) \ @@ -72,7 +97,6 @@ LJ_FUNC void lj_gc_fullgc(lua_State *L); /* Write barriers. */ LJ_FUNC void lj_gc_barrierf(global_State *g, GCobj *o, GCobj *v); LJ_FUNCA void LJ_FASTCALL lj_gc_barrieruv(global_State *g, TValue *tv); -LJ_FUNC void lj_gc_closeuv(global_State *g, GCupval *uv); #if LJ_HASJIT LJ_FUNC void lj_gc_barriertrace(global_State *g, uint32_t traceno); #endif @@ -81,7 +105,7 @@ LJ_FUNC void lj_gc_barriertrace(global_State *g, uint32_t traceno); static LJ_AINLINE void lj_gc_barrierback(global_State *g, GCtab *t) { GCobj *o = obj2gco(t); - lj_assertG(isblack(o) && !isdead(g, o), + lj_assertG(isblack(g, o) && !checkdead(g, o), "bad object states for backward barrier"); lj_assertG(g->gc.state != GCSfinalize && g->gc.state != GCSpause, "bad GC state"); @@ -92,20 +116,20 @@ static LJ_AINLINE void lj_gc_barrierback(global_State *g, GCtab *t) /* Barrier for stores to table objects. TValue and GCobj variant. */ #define lj_gc_anybarriert(L, t) \ - { if (LJ_UNLIKELY(isblack(obj2gco(t)))) lj_gc_barrierback(G(L), (t)); } + { if (LJ_UNLIKELY(isblack(G(L), obj2gco(t)))) lj_gc_barrierback(G(L), (t)); } #define lj_gc_barriert(L, t, tv) \ - { if (tviswhite(tv) && isblack(obj2gco(t))) \ + { if (tviswhite(G(L), tv) && isblack(G(L), obj2gco(t))) \ lj_gc_barrierback(G(L), (t)); } #define lj_gc_objbarriert(L, t, o) \ - { if (iswhite(obj2gco(o)) && isblack(obj2gco(t))) \ + { if (iswhite(G(L), obj2gco(o)) && isblack(G(L), obj2gco(t))) \ lj_gc_barrierback(G(L), (t)); } /* Barrier for stores to any other object. TValue and GCobj variant. */ #define lj_gc_barrier(L, p, tv) \ - { if (tviswhite(tv) && isblack(obj2gco(p))) \ + { if (tviswhite(G(L), tv) && isblack(G(L), obj2gco(p))) \ lj_gc_barrierf(G(L), obj2gco(p), gcV(tv)); } #define lj_gc_objbarrier(L, p, o) \ - { if (iswhite(obj2gco(o)) && isblack(obj2gco(p))) \ + { if (iswhite(G(L), obj2gco(o)) && isblack(G(L), obj2gco(p))) \ lj_gc_barrierf(G(L), obj2gco(p), obj2gco(o)); } /* Allocator. */ @@ -119,18 +143,203 @@ LJ_FUNC void *lj_mem_grow(lua_State *L, void *p, static LJ_AINLINE void lj_mem_free(global_State *g, void *p, size_t osize) { g->gc.total -= (GCSize)osize; + g->gc.malloc -= (GCSize)osize; g->allocf(g->allocd, p, osize, 0); } -#define lj_mem_newvec(L, n, t) ((t *)lj_mem_new(L, (GCSize)((n)*sizeof(t)))) -#define lj_mem_reallocvec(L, p, on, n, t) \ - ((p) = (t *)lj_mem_realloc(L, p, (on)*sizeof(t), (GCSize)((n)*sizeof(t)))) -#define lj_mem_growvec(L, p, n, m, t) \ +#define lj_mem_newvec(L, n, t) ((t *)lj_mem_new(L, (GCSize)((n) * sizeof(t)))) +#define lj_mem_reallocvec(L, p, on, n, t) \ + ((p) = (t *)lj_mem_realloc(L, p, (on) * sizeof(t), (GCSize)((n) * sizeof(t)))) +#define lj_mem_growvec(L, p, n, m, t) \ ((p) = (t *)lj_mem_grow(L, (p), &(n), (m), (MSize)sizeof(t))) -#define lj_mem_freevec(g, p, n, t) lj_mem_free(g, (p), (n)*sizeof(t)) +#define lj_mem_freevec(g, p, n, t) lj_mem_free(g, (p), (n) * sizeof(t)) -#define lj_mem_newobj(L, t) ((t *)lj_mem_newgco(L, sizeof(t))) +#define lj_mem_newv(L, n, t) ((t *)lj_mem_newblob(L, (MSize)((n) * sizeof(t)))) +#define lj_mem_reallocv(L, p, on, n, t) \ + (t *)lj_mem_reallocblob(L, p, (on) * sizeof(t), \ + (MSize)((n) * sizeof(t))) + +#define lj_mem_newobj(L, t) ((t *)lj_mem_newgco(L, sizeof(t))) #define lj_mem_newt(L, s, t) ((t *)lj_mem_new(L, (s))) #define lj_mem_freet(g, p) lj_mem_free(g, (p), sizeof(*(p))) +/* New GC */ + +LJ_FUNC GCtab *lj_mem_alloctab(lua_State *L, uint32_t asize); +LJ_FUNC GCtab *lj_mem_alloctabempty_gc(lua_State *L); +LJ_FUNC GCstr *lj_mem_allocstr(lua_State *L, MSize len); + +LJ_FUNC GCupval *lj_mem_allocuv(lua_State *L); +LJ_FUNC GCudata *lj_mem_allocudata(lua_State *L, MSize bytes); +LJ_FUNC GCfunc *lj_mem_allocfunc(lua_State *L, MSize bytes); + + +LJ_FUNC void *lj_mem_newblob(lua_State *L, MSize sz); +LJ_FUNC void *lj_mem_reallocblob(lua_State *L, void *p, MSize osz, MSize nsz); + +LJ_FUNC void lj_mem_registergc_udata(lua_State *L, GCudata *ud); + +#define lj_gc_markblob(L, blob, size) \ + G(L)->gc.bloblist_usage[gcablob(blob)->id] += size + +typedef uint64_t bitmap_t; +#define WORD_BITS 64 +#define SIMD_BITS 256 +#define BLOB_REAP_THRESHOLD (ARENA_SIZE / 3) + +/* Basic requirement is that each bitmap have an increment of 256 and that each + * area be 32-byte aligned. ELEMENTS_MAX is the maximum index allowed. + * WORDS_FOR_TYPE is the number of 64-bit words in each header bitmap + * WORDS_FOR_TYPE_UNROUNDED is WORDS_FOR_TYPE but not rounded up to 256 bits + * ELEMENTS_OCCUPIED is the number of elements occupied by the header + * HIGH_ELEMENTS_OCCUPIED is the number of elements in the last word + */ +#define ELEMENTS_MAX(type) (ARENA_SIZE / sizeof(type)) +#define WORDS_FOR_TYPE(type) \ + (((ELEMENTS_MAX(type) + SIMD_BITS - 1) / SIMD_BITS) * (SIMD_BITS / WORD_BITS)) +#define WORDS_FOR_SZ(sz) ((((ARENA_SIZE / sz) + SIMD_BITS - 1) / SIMD_BITS) * 4) +#define SIMD_WORDS_FOR_TYPE(type) \ + ((ELEMENTS_MAX(type) + SIMD_BITS - 1) / SIMD_BITS) +#define SIMD_MULTIPLIER (sizeof(I256) / sizeof(bitmap_t)) +#define WORDS_FOR_TYPE_UNROUNDED(type) \ + ((ELEMENTS_MAX(type) + WORD_BITS - 1) / WORD_BITS) +#define HIGH_ELEMENTS_OCCUPIED(type) \ + ((ELEMENTS_MAX(type)) & (WORD_BITS - 1)) +#define ELEMENTS_OCCUPIED(hdr, type) \ + ((sizeof(hdr) + sizeof(type) - 1) / sizeof(type)) +#define ELEMENTS_AVAILABLE(atype, otype) (ELEMENTS_MAX(otype) - ELEMENTS_OCCUPIED(atype, otype)) + +#define FREE_EXTRA_MASK(type) (~0ull >> (WORD_BITS - WORDS_FOR_TYPE(type))) +#define FREE_MASK(type) (~0ull >> (WORD_BITS - WORDS_FOR_TYPE_UNROUNDED(type))) +#define FREE_LOW(atype, type) ~0ull << ELEMENTS_OCCUPIED(atype, type) +/* The else branch of the ternary is incorrect and must be guarded against, + * but it eliminates UB an a warning. It should be resolved at compile time */ +#define FREE_HIGH(type) \ + (~0ull >> \ + HIGH_ELEMENTS_OCCUPIED(type) ? (WORD_BITS - HIGH_ELEMENTS_OCCUPIED(type)) : 1) +#define FREE_HIGH_INDEX(type) (WORDS_FOR_TYPE_UNROUNDED(type) - 1) + +#define MAX_BMARRAY_SIZE (ARENA_SIZE / 16 / WORD_BITS) +LJ_STATIC_ASSERT(MAX_BMARRAY_SIZE <= WORD_BITS); + +#define arena(p) ((GCAcommon *)((uintptr_t)(p)&ARENA_MASK)) +#define gcat(p, t) ((t *)((uintptr_t)(p)&ARENA_MASK)) +#define gcablob(p) gcat(p, GCAblob) +#define aobj(base, type, index) ((type *)(base) + (index)) +#define objmask(p) ((uintptr_t)(p)&ARENA_OMASK) +/* Any good compiler should be able to turn this divide into a multiply */ +#define aidx(p) (objmask(p) / sizeof(*p)) +#define aidxl(i) ((i)&63) +#define aidxh(i) ((i) >> 6) +#define abit(i) (1ull << (i)) + +#define free_enq(a, h) \ + do { \ + (a)->freenext = h; \ + (a)->freeprev = NULL; \ + if (h) \ + (h)->freeprev = a; \ + h = a; \ + } while (0) + +#define do_arena_init(a, g, id, atype, otype) \ + memset(a, 0, sizeof(atype)); \ + a->hdr.obj_type = id; \ + a->hdr.flags = g->gc.currentsweep; \ + a->free_h = FREE_MASK(otype); \ + for (uint32_t i = 0; i < WORDS_FOR_TYPE_UNROUNDED(otype); i++) { \ + a->free[i] = ~0ull; \ + } \ + a->free[0] = FREE_LOW(atype, otype); \ + if (HIGH_ELEMENTS_OCCUPIED(otype) != 0) \ + a->free[FREE_HIGH_INDEX(otype)] = FREE_HIGH(otype) + +typedef struct GCAcommon { + GCArenaHdr hdr; + bitmap_t unspecified; + bitmap_t gray_h; + bitmap_t mark[MAX_BMARRAY_SIZE]; + bitmap_t gray[MAX_BMARRAY_SIZE]; +} GCAcommon; + +/* The requirement is that the offset between gray and mark is identical, + * but the extra space is unused so we can stuff the other arrays in there. + * This works because tables are relatively large objects. + */ +typedef struct GCAtab { + GCArenaHdr hdr; + bitmap_t free_h; + bitmap_t gray_h; + bitmap_t mark[WORDS_FOR_TYPE(GCtab)]; + bitmap_t free[WORDS_FOR_TYPE(GCtab)]; + /*bitmap_t padding[MAX_BMARRAY_SIZE - 4 * WORDS_FOR_TYPE(GCtab)];*/ + bitmap_t weak[WORDS_FOR_TYPE(GCtab)]; + bitmap_t fin[WORDS_FOR_TYPE(GCtab)]; + bitmap_t gray[WORDS_FOR_TYPE(GCtab)]; + struct GCAtab *fin_next; +} GCAtab; + +LJ_STATIC_ASSERT(sizeof(GCtab) == 64); + +typedef struct GCAudata { + GCArenaHdr hdr; + /* This works because we cannot have a bit set in free4 if that same bit is + * cleared in free, so the general case will stop at the set bit in free + * first. */ + LJ_ENDIAN_LOHI(uint32_t free_h;, uint32_t free4_h); + bitmap_t gray_h; + bitmap_t mark[WORDS_FOR_TYPE(GCudata)]; + bitmap_t free[WORDS_FOR_TYPE(GCudata)]; + bitmap_t padding[MAX_BMARRAY_SIZE - 2 * WORDS_FOR_TYPE(GCudata)]; + bitmap_t gray[WORDS_FOR_TYPE(GCudata)]; + bitmap_t fin[WORDS_FOR_TYPE(GCudata)]; + bitmap_t fin_req[WORDS_FOR_TYPE(GCudata)]; +} GCAudata; + +LJ_STATIC_ASSERT(sizeof(GCudata) >= sizeof(PRNGState)); + +typedef struct GCAupval { + GCArenaHdr hdr; + bitmap_t free_h; + bitmap_t gray_h; + bitmap_t mark[WORDS_FOR_TYPE(GCupval)]; + bitmap_t padding[MAX_BMARRAY_SIZE - WORDS_FOR_TYPE(GCupval)]; + bitmap_t gray[WORDS_FOR_TYPE(GCupval)]; + bitmap_t free[WORDS_FOR_TYPE(GCupval)]; +} GCAupval; + +typedef struct GCAfunc { + GCArenaHdr hdr; + bitmap_t free_h; + bitmap_t gray_h; + bitmap_t mark[WORDS_FOR_TYPE(GCfunc)]; + bitmap_t free[WORDS_FOR_TYPE(GCfunc)]; + /*bitmap_t padding[MAX_BMARRAY_SIZE - 2 * WORDS_FOR_TYPE(GCfunc)];*/ + bitmap_t gray[WORDS_FOR_TYPE(GCfunc)]; +} GCAfunc; + +/* All offsets must match the common arena */ +LJ_STATIC_ASSERT(offsetof(GCAtab, gray) == offsetof(GCAcommon, gray)); +LJ_STATIC_ASSERT(offsetof(GCAtab, mark) == offsetof(GCAcommon, mark)); +LJ_STATIC_ASSERT(offsetof(GCAudata, gray) == offsetof(GCAcommon, gray)); +LJ_STATIC_ASSERT(offsetof(GCAudata, mark) == offsetof(GCAcommon, mark)); +LJ_STATIC_ASSERT(offsetof(GCAupval, gray) == offsetof(GCAcommon, gray)); +LJ_STATIC_ASSERT(offsetof(GCAupval, mark) == offsetof(GCAcommon, mark)); +LJ_STATIC_ASSERT(offsetof(GCAfunc, gray) == offsetof(GCAcommon, gray)); +LJ_STATIC_ASSERT(offsetof(GCAfunc, mark) == offsetof(GCAcommon, mark)); + +#if LJ_HASJIT +typedef struct GCAtrace { + GCArenaHdr hdr; + bitmap_t free_h; + bitmap_t gray_h; + bitmap_t mark[WORDS_FOR_TYPE(GCtrace)]; + bitmap_t free[WORDS_FOR_TYPE(GCtrace)]; + bitmap_t padding[MAX_BMARRAY_SIZE - 2 * WORDS_FOR_TYPE(GCtrace)]; + bitmap_t gray[WORDS_FOR_TYPE(GCtrace)]; +} GCAtrace; +LJ_STATIC_ASSERT(offsetof(GCAtrace, gray) == offsetof(GCAcommon, gray)); +LJ_STATIC_ASSERT(offsetof(GCAtrace, mark) == offsetof(GCAcommon, mark)); +#endif + #endif diff --git a/src/lj_intrin.h b/src/lj_intrin.h new file mode 100644 index 00000000..d0c9a5a5 --- /dev/null +++ b/src/lj_intrin.h @@ -0,0 +1,38 @@ +#ifndef _LJ_INTRIN_H_ +#define _LJ_INTRIN_H_ + +#include "lj_arch.h" + +#if LUAJIT_TARGET == LUAJIT_ARCH_X64 +#if LJ_TARGET_WINDOWS +#include +#else +#include +#endif + +/* # of contiguous low 0 bits */ +#define tzcount32(x) (unsigned)_tzcnt_u32(x) +#define tzcount64(x) (unsigned)_tzcnt_u64(x) + +/* x & (x - 1) */ +#define reset_lowest32(x) _blsr_u32(x) +#define reset_lowest64(x) _blsr_u64(x) + +/* 256 bit SIMD */ +#define LJ_SIMD_256 1 +#define I256 __m256i +#define I256_ZERO(o) o = _mm256_setzero_si256() +/* vpxor a, a, a; vpcmpeqq a, a, a sets all bits to 1 */ +#define I256_ONES(o) o = _mm256_cmpeq_epi64(_mm256_setzero_si256(), _mm256_setzero_si256()) +#define I256_EQ_64_MASK(x, y) (uint64_t)_mm256_movemask_pd(_mm256_castsi256_pd(_mm256_cmpeq_epi64(x, y))) +#define I256_AND(o, x, y) o = _mm256_and_si256(x, y) +#define I256_XOR(o, x, y) o = _mm256_xor_si256(x, y) +#define I256_OR(o, x, y) o = _mm256_or_si256(x, y) +#define I256_LOADA(o, ptr) o = _mm256_load_si256((__m256i *)ptr) +#define I256_STOREA(ptr, v) _mm256_store_si256((__m256i *)ptr, v) + +#else +#error "No intrinsics defined for arch" +#endif + +#endif diff --git a/src/lj_ir.c b/src/lj_ir.c index 631b7cbc..4445816c 100644 --- a/src/lj_ir.c +++ b/src/lj_ir.c @@ -280,7 +280,7 @@ TRef lj_ir_kgc(jit_State *J, GCobj *o, IRType t) { IRIns *ir, *cir = J->cur.ir; IRRef ref; - lj_assertJ(!isdead(J2G(J), o), "interning of dead GC object"); + lj_assertJ(!checkdead(J2G(J), o), "interning of dead GC object"); for (ref = J->chain[IR_KGC]; ref; ref = cir[ref].prev) if (ir_kgc(&cir[ref]) == o) goto found; diff --git a/src/lj_ir.h b/src/lj_ir.h index dbfba258..2baf3b4e 100644 --- a/src/lj_ir.h +++ b/src/lj_ir.h @@ -204,7 +204,8 @@ IRFPMDEF(FPMENUM) _(TAB_NOMM, offsetof(GCtab, nomm)) \ _(UDATA_META, offsetof(GCudata, metatable)) \ _(UDATA_UDTYPE, offsetof(GCudata, udtype)) \ - _(UDATA_FILE, sizeof(GCudata)) \ + _(UDATA_FILE, sizeof(GCudata)) \ + _(UDATA_PAYLOAD, offsetof(GCudata, payload)) \ _(SBUF_W, sizeof(GCudata) + offsetof(SBufExt, w)) \ _(SBUF_E, sizeof(GCudata) + offsetof(SBufExt, e)) \ _(SBUF_B, sizeof(GCudata) + offsetof(SBufExt, b)) \ diff --git a/src/lj_jit.h b/src/lj_jit.h index 6902fba3..a90ce9c7 100644 --- a/src/lj_jit.h +++ b/src/lj_jit.h @@ -259,11 +259,12 @@ typedef struct GCtrace { GCHeader; uint16_t nsnap; /* Number of snapshots. */ IRRef nins; /* Next IR instruction. Biased with REF_BIAS. */ + GCRef nextgc; #if LJ_GC64 uint32_t unused_gc64; #endif - GCRef gclist; IRIns *ir; /* IR instructions/constants. Biased with REF_BIAS. */ + GCRef gclist; IRRef nk; /* Lowest IR constant. Biased with REF_BIAS. */ uint32_t nsnapmap; /* Number of snapshot map elements. */ SnapShot *snap; /* Snapshot array. */ @@ -298,6 +299,7 @@ typedef struct GCtrace { check_exp((n)>0 && (MSize)(n)sizetrace, (GCtrace *)gcref(J->trace[(n)])) LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCtrace, gclist)); +LJ_STATIC_ASSERT(offsetof(GChead, nextgc) == offsetof(GCtrace, nextgc)); static LJ_AINLINE MSize snap_nextofs(GCtrace *T, SnapShot *snap) { diff --git a/src/lj_lib.c b/src/lj_lib.c index ebe0dc78..1565b583 100644 --- a/src/lj_lib.c +++ b/src/lj_lib.c @@ -95,7 +95,7 @@ void lj_lib_register(lua_State *L, const char *libname, GCfunc *fn = lj_func_newC(L, nuv, env); if (nuv) { L->top = L->base + tpos; - memcpy(fn->c.upvalue, L->top, sizeof(TValue)*nuv); + memcpy(fn->c.data->upvalue, L->top, sizeof(TValue)*nuv); } fn->c.ffid = (uint8_t)(ffid++); name = (const char *)p; @@ -105,9 +105,9 @@ void lj_lib_register(lua_State *L, const char *libname, else setmref(fn->c.pc, bcff++); if (tag == LIBINIT_ASM_) - fn->c.f = ofn->c.f; /* Copy handler from previous function. */ + fn->c.data->f = ofn->c.data->f; /* Copy handler from previous function. */ else - fn->c.f = *cf++; /* Get cf or handler from C function table. */ + fn->c.data->f = *cf++; /* Get cf or handler from C function table. */ if (len) { /* NOBARRIER: See above for common barrier. */ setfuncV(L, lj_tab_setstr(L, tab, lj_str_new(L, name, len)), fn); diff --git a/src/lj_lib.h b/src/lj_lib.h index 6c3a1c83..3c722d1f 100644 --- a/src/lj_lib.h +++ b/src/lj_lib.h @@ -55,7 +55,7 @@ LJ_FUNC int32_t lj_lib_checkintrange(lua_State *L, int narg, /* Avoid including lj_frame.h. */ #if LJ_GC64 #define lj_lib_upvalue(L, n) \ - (&gcval(L->base-2)->fn.c.upvalue[(n)-1]) + (&gcval(L->base-2)->fn.c.data->upvalue[(n)-1]) #elif LJ_FR2 #define lj_lib_upvalue(L, n) \ (&gcref((L->base-2)->gcr)->fn.c.upvalue[(n)-1]) diff --git a/src/lj_obj.h b/src/lj_obj.h index 2d4386e1..ee537583 100644 --- a/src/lj_obj.h +++ b/src/lj_obj.h @@ -12,6 +12,7 @@ #include "lua.h" #include "lj_def.h" #include "lj_arch.h" +#include "lj_arena.h" /* -- Memory references --------------------------------------------------- */ @@ -60,7 +61,7 @@ typedef struct GCRef { } GCRef; /* Common GC header for all collectable objects. */ -#define GCHeader GCRef nextgc; uint8_t marked; uint8_t gct +#define GCHeader uint8_t gcflags; uint8_t gct /* This occupies 6 bytes, so use the next 2 bytes for non-32 bit fields. */ #if LJ_GC64 @@ -307,7 +308,8 @@ typedef struct GCstr { GCHeader; uint8_t reserved; /* Used by lexer for fast lookup of reserved words. */ uint8_t hashalg; /* Hash algorithm. */ - StrID sid; /* Interned string ID. */ + GCRef nextgc; + StrID sid; /* Interned string ID. */ StrHash hash; /* Hash of string. */ MSize len; /* Size of string. */ } GCstr; @@ -319,15 +321,16 @@ typedef struct GCstr { /* -- Userdata object ----------------------------------------------------- */ -/* Userdata object. Payload follows. */ +/* Userdata object. */ typedef struct GCudata { GCHeader; uint8_t udtype; /* Userdata type. */ uint8_t unused2; - GCRef env; /* Should be at same offset in GCfunc. */ MSize len; /* Size of payload. */ + MRef payload; + GCRef env; /* Should be at same offset in GCfunc. */ GCRef metatable; /* Must be at same offset in GCtab. */ - uint32_t align1; /* To force 8 byte alignment of the payload. */ + GCRef gclist; } GCudata; /* Userdata types. */ @@ -339,7 +342,7 @@ enum { UDTYPE__MAX }; -#define uddata(u) ((void *)((u)+1)) +#define uddata(u) mref((u)->payload, void) #define sizeudata(u) (sizeof(struct GCudata)+(u)->len) /* -- C data object ------------------------------------------------------- */ @@ -348,6 +351,7 @@ enum { typedef struct GCcdata { GCHeader; uint16_t ctypeid; /* C type ID. */ + GCRef nextgc; } GCcdata; /* Prepended to variable-sized or realigned C data objects. */ @@ -358,7 +362,7 @@ typedef struct GCcdataVar { } GCcdataVar; #define cdataptr(cd) ((void *)((cd)+1)) -#define cdataisv(cd) ((cd)->marked & 0x80) +#define cdataisv(cd) ((cd)->gcflags & 0x80) #define cdatav(cd) ((GCcdataVar *)((char *)(cd) - sizeof(GCcdataVar))) #define cdatavlen(cd) check_exp(cdataisv(cd), cdatav(cd)->len) #define sizecdatav(cd) (cdatavlen(cd) + cdatav(cd)->extra) @@ -374,11 +378,12 @@ typedef struct GCproto { uint8_t numparams; /* Number of parameters. */ uint8_t framesize; /* Fixed frame size. */ MSize sizebc; /* Number of bytecode instructions. */ + GCRef nextgc; #if LJ_GC64 uint32_t unused_gc64; #endif - GCRef gclist; MRef k; /* Split constant array (points to the middle). */ + GCRef gclist; MRef uv; /* Upvalue list. local slot|0x8000 or parent uv idx. */ MSize sizekgc; /* Number of collectable constants. */ MSize sizekn; /* Number of lua_Number constants. */ @@ -433,19 +438,14 @@ typedef struct GCupval { GCHeader; uint8_t closed; /* Set if closed (i.e. uv->v == &uv->u.value). */ uint8_t immutable; /* Immutable value. */ + uint32_t dhash; /* Disambiguation hash: dh1 != dh2 => cannot alias. */ union { TValue tv; /* If closed: the value itself. */ - struct { /* If open: double linked list, anchored at thread. */ - GCRef prev; - GCRef next; - }; + GCRef next; /* If open: single linked list, anchored at thread. */ }; MRef v; /* Points to stack slot (open) or above (closed). */ - uint32_t dhash; /* Disambiguation hash: dh1 != dh2 => cannot alias. */ } GCupval; -#define uvprev(uv_) (&gcref((uv_)->prev)->uv) -#define uvnext(uv_) (&gcref((uv_)->next)->uv) #define uvval(uv_) (mref((uv_)->v, TValue)) /* -- Function object (closures) ------------------------------------------ */ @@ -453,22 +453,32 @@ typedef struct GCupval { /* Common header for functions. env should be at same offset in GCudata. */ #define GCfuncHeader \ GCHeader; uint8_t ffid; uint8_t nupvalues; \ - GCRef env; GCRef gclist; MRef pc + MRef pc; GCRef env + +typedef struct GCfuncCData { + lua_CFunction f; /* C function to be called. */ + TValue upvalue[1]; /* Array of upvalues (TValue). */ +} GCfuncCData; + +typedef struct GCfuncGen { + GCfuncHeader; + MRef data; +} GCfuncGen; typedef struct GCfuncC { GCfuncHeader; - lua_CFunction f; /* C function to be called. */ - TValue upvalue[1]; /* Array of upvalues (TValue). */ + GCfuncCData *data; } GCfuncC; typedef struct GCfuncL { GCfuncHeader; - GCRef uvptr[1]; /* Array of _pointers_ to upvalue objects (GCupval). */ + GCRef *uvptr; /* Pointer to array of _pointers_ to upvalue objects (GCupval). */ } GCfuncL; typedef union GCfunc { GCfuncC c; GCfuncL l; + GCfuncGen gen; } GCfunc; #define FF_LUA 0 @@ -478,8 +488,8 @@ typedef union GCfunc { #define isffunc(fn) ((fn)->c.ffid > FF_C) #define funcproto(fn) \ check_exp(isluafunc(fn), (GCproto *)(mref((fn)->l.pc, char)-sizeof(GCproto))) -#define sizeCfunc(n) (sizeof(GCfuncC)-sizeof(TValue)+sizeof(TValue)*(n)) -#define sizeLfunc(n) (sizeof(GCfuncL)-sizeof(GCRef)+sizeof(GCRef)*(n)) +#define sizeCfunc(n) (sizeof(TValue)+sizeof(TValue)*(n)) +#define sizeLfunc(n) (sizeof(GCRef)*(n)) /* -- Table object -------------------------------------------------------- */ @@ -498,15 +508,16 @@ LJ_STATIC_ASSERT(offsetof(Node, val) == 0); typedef struct GCtab { GCHeader; uint8_t nomm; /* Negative cache for fast metamethods. */ - int8_t colo; /* Array colocation. */ + uint8_t colo; /* Array colocation. */ MRef array; /* Array part. */ - GCRef gclist; - GCRef metatable; /* Must be at same offset in GCudata. */ MRef node; /* Hash part. */ + GCRef metatable; /* Must be at same offset in GCudata. */ + GCRef gclist; uint32_t asize; /* Size of array part (keys [0, asize-1]). */ uint32_t hmask; /* Hash part mask (size of hash part - 1). */ #if LJ_GC64 MRef freetop; /* Top of free elements. */ + uint64_t unused; #endif } GCtab; @@ -590,7 +601,11 @@ typedef enum { typedef struct GCState { GCSize total; /* Memory currently allocated. */ GCSize threshold; /* Memory threshold. */ - uint8_t currentwhite; /* Current white color. */ + uint8_t currentblack; + uint8_t currentblackgray; + uint8_t safecolor; + uint8_t currentsweep; + uint8_t gcmode; uint8_t state; /* GC state. */ uint8_t nocdatafin; /* No cdata finalizer called. */ #if LJ_64 @@ -602,16 +617,52 @@ typedef struct GCState { GCRef root; /* List of all collectable objects. */ MRef sweep; /* Sweep position in root list. */ GCRef gray; /* List of gray objects. */ - GCRef grayagain; /* List of objects for atomic traversal. */ + GCRef grayagain; /* List of tables for atomic traversal. */ + GCRef grayagain_th; /* List of threads for atomic traversal. */ GCRef weak; /* List of weak tables (to be cleared). */ + GCRef ephemeron; GCRef mmudata; /* List of userdata (to be finalized). */ + GCRef fin_list; GCSize debt; /* Debt (how much GC is behind schedule). */ GCSize estimate; /* Estimate of memory actually in use. */ + GCSize accum; /* Accumulated memory marked. */ + GCSize malloc; /* Malloced memory. */ MSize stepmul; /* Incremental GC step granularity. */ MSize pause; /* Pause between successive GC cycles. */ #if LJ_64 MRef lightudseg; /* Upper bits of lightuserdata segments. */ #endif + + /* New GC */ + arena_context ctx; + /* Arenas to allocate from. Never NULL. Chains with prev/next */ + GCArenaHdr *tab; + GCArenaHdr *fintab; + GCArenaHdr *uv; + GCArenaHdr *func; + GCArenaHdr *udata; + + /* This is the allocated-from blob arena. Never NULL */ + GCAblob *blob_generic; + + GCAblob **bloblist; + uint32_t *bloblist_usage; + uint32_t bloblist_alloc; + uint32_t bloblist_wr; + uint32_t bloblist_sweep; + + /* These are the arenas that have at least one gray object, may be NULL. + * Chains with 'gray' */ + GCArenaHdr *gray_head; + GCArenaHdr *gray_tail; + + /* Freelist, may be NULL. Contains at least one free object. Chains with + * 'freenext' & 'freeprev' */ + GCArenaHdr *free_tab; + GCArenaHdr *free_fintab; + GCArenaHdr *free_uv; + GCArenaHdr *free_func; + GCArenaHdr *free_udata; } GCState; /* String interning state. */ @@ -644,7 +695,6 @@ typedef struct global_State { TValue tmptv, tmptv2; /* Temporary TValues. */ Node nilnode; /* Fallback 1-element hash part (nil key and value). */ TValue registrytv; /* Anchor for registry. */ - GCupval uvhead; /* Head of double-linked list of all open upvalues. */ int32_t hookcount; /* Instruction hook countdown. */ int32_t hookcstart; /* Start count for instruction hook counter. */ lua_Hook hookf; /* Hook function. */ @@ -687,9 +737,10 @@ struct lua_State { GCHeader; uint8_t dummy_ffid; /* Fake FF_C for curr_funcisL() on dummy frames. */ uint8_t status; /* Thread status. */ - MRef glref; /* Link to global state. */ - GCRef gclist; /* GC chain. */ + GCRef nextgc; + MRef glref; /* Link to global state. */ TValue *base; /* Base of currently executing function. */ + GCRef gclist; /* GC chain. */ TValue *top; /* First free slot in the stack. */ MRef maxstack; /* Last free slot in the stack. */ MRef stack; /* Stack base. */ @@ -727,9 +778,10 @@ typedef struct GChead { GCHeader; uint8_t unused1; uint8_t unused2; + GCRef nextgc; GCRef env; - GCRef gclist; GCRef metatable; + GCRef gclist; } GChead; /* The env field SHOULD be at the same offset for all GC objects. */ @@ -742,9 +794,14 @@ LJ_STATIC_ASSERT(offsetof(GChead, metatable) == offsetof(GCudata, metatable)); /* The gclist field MUST be at the same offset for all GC objects. */ LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(lua_State, gclist)); -LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCproto, gclist)); -LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCfuncL, gclist)); LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCtab, gclist)); +LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCproto, gclist)); + +/* The nextgc field MUST be at the same offset for all LL GC objects. */ +LJ_STATIC_ASSERT(offsetof(GChead, nextgc) == offsetof(lua_State, nextgc)); +LJ_STATIC_ASSERT(offsetof(GChead, nextgc) == offsetof(GCproto, nextgc)); +LJ_STATIC_ASSERT(offsetof(GChead, nextgc) == offsetof(GCcdata, nextgc)); +LJ_STATIC_ASSERT(offsetof(GChead, nextgc) == offsetof(GCstr, nextgc)); typedef union GCobj { GChead gch; @@ -897,6 +954,9 @@ static LJ_AINLINE void setrawlightudV(TValue *o, void *p) ((o)->u64 = (uint64_t)(void *)(f) - (uint64_t)lj_vm_asm_begin) #endif +/* Copy from lj_gc.h */ +LJ_FUNC int checkdead(global_State *g, GCobj *o); + static LJ_AINLINE void checklivetv(lua_State *L, TValue *o, const char *msg) { UNUSED(L); UNUSED(o); UNUSED(msg); @@ -906,7 +966,7 @@ static LJ_AINLINE void checklivetv(lua_State *L, TValue *o, const char *msg) "mismatch of TValue type %d vs GC type %d", ~itype(o), gcval(o)->gch.gct); /* Copy of isdead check from lj_gc.h to avoid circular include. */ - lj_assertL(!(gcval(o)->gch.marked & (G(L)->gc.currentwhite ^ 3) & 3), msg); + lj_assertL(!checkdead(G(L), gcval(o)), msg); } #endif } diff --git a/src/lj_record.c b/src/lj_record.c index b7af5896..e3d94e9b 100644 --- a/src/lj_record.c +++ b/src/lj_record.c @@ -1727,7 +1727,7 @@ static int rec_upvalue_constify(jit_State *J, GCupval *uvp) #if LJ_HASFFI if (tviscdata(o)) { GCcdata *cd = cdataV(o); - if (!cdataisv(cd) && !(cd->marked & LJ_GC_CDATA_FIN)) { + if (!cdataisv(cd) && !(cd->gcflags & LJ_GC_CDATA_FIN)) { CType *ct = ctype_raw(ctype_ctsG(J2G(J)), cd->ctypeid); if (!ctype_hassize(ct->info) || ct->size <= 16) return 1; diff --git a/src/lj_snap.c b/src/lj_snap.c index 7d7347a1..70a801a4 100644 --- a/src/lj_snap.c +++ b/src/lj_snap.c @@ -224,7 +224,7 @@ static BCReg snap_usedef(jit_State *J, uint8_t *udf, while (o) { if (uvval(gco2uv(o)) < J->L->base) break; udf[uvval(gco2uv(o)) - J->L->base] = 0; - o = gcref(o->gch.nextgc); + o = gcref(o->uv.next); } #define USE_SLOT(s) udf[(s)] &= ~1 diff --git a/src/lj_state.c b/src/lj_state.c index 7e4961bd..c4676cff 100644 --- a/src/lj_state.c +++ b/src/lj_state.c @@ -28,6 +28,7 @@ #include "lj_prng.h" #include "lj_lex.h" #include "lj_alloc.h" +#include "lj_arena.h" #include "luajit.h" /* -- Stack handling ------------------------------------------------------ */ @@ -76,7 +77,7 @@ static void resizestack(lua_State *L, MSize n) setmref(G(L)->jit_base, mref(G(L)->jit_base, char) + delta); L->base = (TValue *)((char *)L->base + delta); L->top = (TValue *)((char *)L->top + delta); - for (up = gcref(L->openupval); up != NULL; up = gcnext(up)) + for (up = gcref(L->openupval); up != NULL; up = gcref(up->uv.next)) setmref(gco2uv(up)->v, (TValue *)((char *)uvval(gco2uv(up)) + delta)); } @@ -200,6 +201,11 @@ static void close_state(lua_State *L) lj_mem_freevec(g, mref(g->gc.lightudseg, uint32_t), segnum, uint32_t); } #endif + lj_arena_cleanup(g); + lj_assertG(g->gc.ctx.mem_commit == 0, "memory leak of %u arenas", + g->gc.ctx.mem_commit); + lj_assertG(g->gc.ctx.mem_huge == 0, "memory leak of %llu huge arena bytes", + g->gc.ctx.mem_huge); lj_assertG(g->gc.total == sizeof(GG_State), "memory leak of %lld bytes", (long long)(g->gc.total - sizeof(GG_State))); @@ -215,7 +221,16 @@ static void close_state(lua_State *L) lua_State *lj_state_newstate(lua_Alloc allocf, void *allocd) #else LUA_API lua_State *lua_newstate(lua_Alloc allocf, void *allocd) +{ + return lj_newstate(allocf, allocd, NULL, NULL, NULL, NULL); +} #endif + +lua_State *lj_newstate(lua_Alloc allocf, void *allocd, + luaJIT_allocpages allocp, + luaJIT_freepages freep, + luaJIT_reallochuge realloch, + void *page_ud) { PRNGState prng; GG_State *GG; @@ -239,15 +254,22 @@ LUA_API lua_State *lua_newstate(lua_Alloc allocf, void *allocd) memset(GG, 0, sizeof(GG_State)); L = &GG->L; g = &GG->g; - L->gct = ~LJ_TTHREAD; - L->marked = LJ_GC_WHITE0 | LJ_GC_FIXED | LJ_GC_SFIXED; /* Prevent free. */ - L->dummy_ffid = FF_C; - setmref(L->glref, g); - g->gc.currentwhite = LJ_GC_WHITE0 | LJ_GC_FIXED; - g->strempty.marked = LJ_GC_WHITE0; - g->strempty.gct = ~LJ_TSTR; g->allocf = allocf; g->allocd = allocd; + g->gc.currentsweep = LJ_GC_SWEEP0; + if (!lj_arena_init(g, allocp, freep, realloch, page_ud)) { + close_state(L); + return NULL; + } + L->gct = ~LJ_TTHREAD; + L->gcflags = LJ_GC_FIXED | LJ_GC_SFIXED; /* Prevent free. */ + L->dummy_ffid = FF_C; + setmref(L->glref, g); + g->gc.currentblack = LJ_GC_BLACK0; + g->gc.currentblackgray = LJ_GC_BLACK0 | LJ_GC_GRAY; + g->gc.safecolor = 0; + g->strempty.gcflags = 0; + g->strempty.gct = ~LJ_TSTR; g->prng = prng; #ifndef LUAJIT_USE_SYSMALLOC if (allocf == lj_alloc_f) { @@ -255,8 +277,6 @@ LUA_API lua_State *lua_newstate(lua_Alloc allocf, void *allocd) } #endif setgcref(g->mainthref, obj2gco(L)); - setgcref(g->uvhead.prev, obj2gco(&g->uvhead)); - setgcref(g->uvhead.next, obj2gco(&g->uvhead)); g->str.mask = ~(MSize)0; setnilV(registry(L)); setnilV(&g->nilnode.val); @@ -269,9 +289,10 @@ LUA_API lua_State *lua_newstate(lua_Alloc allocf, void *allocd) setgcref(g->gc.root, obj2gco(L)); setmref(g->gc.sweep, &g->gc.root); g->gc.total = sizeof(GG_State); + g->gc.malloc = g->gc.total; g->gc.pause = LUAI_GCPAUSE; g->gc.stepmul = LUAI_GCMUL; - lj_dispatch_init((GG_State *)L); + lj_dispatch_init(GG); L->status = LUA_ERRERR+1; /* Avoid touching the stack upon memory error. */ if (lj_vm_cpcall(L, NULL, NULL, cpluaopen) != 0) { /* Memory allocation error: free partial state. */ @@ -302,7 +323,7 @@ LUA_API void lua_close(lua_State *L) #endif setgcrefnull(g->cur_L); lj_func_closeuv(L, tvref(L->stack)); - lj_gc_separateudata(g, 1); /* Separate udata which have GC metamethods. */ + lj_gc_separateudata(g); /* Separate udata which have GC metamethods. */ #if LJ_HASJIT G2J(g)->flags &= ~JIT_F_ON; G2J(g)->state = LJ_TRACE_IDLE; @@ -315,8 +336,9 @@ LUA_API void lua_close(lua_State *L) L->cframe = NULL; if (lj_vm_cpcall(L, NULL, NULL, cpfinalize) == LUA_OK) { if (++i >= 10) break; - lj_gc_separateudata(g, 1); /* Separate udata again. */ - if (gcref(g->gc.mmudata) == NULL) /* Until nothing is left to do. */ + lj_gc_separateudata(g); /* Separate udata again. */ + if (gcref(g->gc.mmudata) == NULL && gcref(g->gc.fin_list) == NULL) + /* Until nothing is left to do. */ break; } } @@ -337,7 +359,7 @@ lua_State *lj_state_new(lua_State *L) setmrefr(L1->glref, L->glref); setgcrefr(L1->env, L->env); stack_init(L1, L); /* init stack */ - lj_assertL(iswhite(obj2gco(L1)), "new thread object is not white"); + lj_assertL(iswhite(G(L), obj2gco(L1)), "new thread object is not white"); return L1; } diff --git a/src/lj_state.h b/src/lj_state.h index 3850e5a1..b0b4dc15 100644 --- a/src/lj_state.h +++ b/src/lj_state.h @@ -35,4 +35,11 @@ LJ_FUNC lua_State *lj_state_newstate(lua_Alloc f, void *ud); #define LJ_ALLOCF_INTERNAL ((lua_Alloc)(void *)(uintptr_t)(1237<<4)) +LJ_FUNC lua_State *lj_newstate(lua_Alloc f, void *ud, + luaJIT_allocpages allocp, + luaJIT_freepages freep, + luaJIT_reallochuge realloch, + void *page_ud); + + #endif diff --git a/src/lj_str.c b/src/lj_str.c index cfdaec6f..f952850c 100644 --- a/src/lj_str.c +++ b/src/lj_str.c @@ -220,7 +220,9 @@ static LJ_NOINLINE GCstr *lj_str_rehash_chain(lua_State *L, StrHash hashc, const char *str, MSize len) { global_State *g = G(L); - int ow = g->gc.state == GCSsweepstring ? otherwhite(g) : 0; /* Sweeping? */ + int sweep = + g->gc.state == GCSsweepstring ? g->gc.safecolor : 0; /* Sweeping? */ + uint8_t mask = isminor(g) ? 0xFF : ~LJ_GC_COLORS; GCRef *strtab = g->str.tab; MSize strmask = g->str.mask; GCobj *o = gcref(strtab[hashc & strmask]); @@ -231,13 +233,13 @@ static LJ_NOINLINE GCstr *lj_str_rehash_chain(lua_State *L, StrHash hashc, GCobj *next = gcnext(o); GCstr *s = gco2str(o); StrHash hash; - if (ow) { /* Must sweep while rechaining. */ - if (((o->gch.marked ^ LJ_GC_WHITES) & ow)) { /* String alive? */ - lj_assertG(!isdead(g, o) || (o->gch.marked & LJ_GC_FIXED), + if (sweep) { /* Must sweep while rechaining. */ + if (o->gch.gcflags & sweep) { /* String alive? */ + lj_assertG(!checkdead(g, o) || (o->gch.gcflags & LJ_GC_FIXED), "sweep of undead string"); - makewhite(g, o); + o->gch.gcflags &= mask; } else { /* Free dead string. */ - lj_assertG(isdead(g, o) || ow == LJ_GC_SFIXED, + lj_assertG(checkdead(g, o) || (sweep & LJ_GC_SFIXED), "sweep of unlive string"); lj_str_free(g, s); o = next; @@ -275,10 +277,10 @@ static LJ_NOINLINE GCstr *lj_str_rehash_chain(lua_State *L, StrHash hashc, static GCstr *lj_str_alloc(lua_State *L, const char *str, MSize len, StrHash hash, int hashalg) { - GCstr *s = lj_mem_newt(L, lj_str_size(len), GCstr); + GCstr *s = lj_mem_allocstr(L, len); global_State *g = G(L); uintptr_t u; - newwhite(g, s); + newwhite(s); s->gct = ~LJ_TSTR; s->len = len; s->hash = hash; @@ -332,7 +334,7 @@ GCstr *lj_str_new(lua_State *L, const char *str, size_t lenx) GCstr *sx = gco2str(o); if (sx->hash == hash && sx->len == len) { if (memcmp(str, strdata(sx), len) == 0) { - if (isdead(g, o)) flipwhite(o); /* Resurrect if dead. */ + maybe_resurrect_str(g, sx); return sx; /* Return existing string. */ } coll++; diff --git a/src/lj_tab.c b/src/lj_tab.c index a9b02dcc..79035082 100644 --- a/src/lj_tab.c +++ b/src/lj_tab.c @@ -42,7 +42,7 @@ static LJ_AINLINE void newhpart(lua_State *L, GCtab *t, uint32_t hbits) if (hbits > LJ_MAX_HBITS) lj_err_msg(L, LJ_ERR_TABOV); hsize = 1u << hbits; - node = lj_mem_newvec(L, hsize, Node); + node = lj_mem_newv(L, hsize, Node); setmref(t->node, node); setfreetop(t, node, &node[hsize]); t->hmask = hsize-1; @@ -80,51 +80,40 @@ static LJ_AINLINE void clearapart(GCtab *t) /* Create a new table. Note: the slots are not initialized (yet). */ static GCtab *newtab(lua_State *L, uint32_t asize, uint32_t hbits) { - GCtab *t; - /* First try to colocate the array part. */ - if (LJ_MAX_COLOSIZE != 0 && asize > 0 && asize <= LJ_MAX_COLOSIZE) { - Node *nilnode; - lj_assertL((sizeof(GCtab) & 7) == 0, "bad GCtab size"); - t = (GCtab *)lj_mem_newgco(L, sizetabcolo(asize)); - t->gct = ~LJ_TTAB; - t->nomm = (uint8_t)~0; - t->colo = (int8_t)asize; - setmref(t->array, (TValue *)((char *)t + sizeof(GCtab))); - setgcrefnull(t->metatable); + GCtab *t = lj_mem_alloctab(L, asize); + Node *nilnode = &G(L)->nilnode; + setmref(t->node, nilnode); +#if LJ_GC64 + setmref(t->freetop, nilnode); +#endif + + if (hbits) + newhpart(L, t, hbits); + return t; +} + +GCtab *lj_tab_newgc(lua_State *L, uint32_t asize, uint32_t hbits) +{ + Node *nilnode; + GCtab *t = lj_mem_alloctabempty_gc(L); + t->hmask = 0; + nilnode = &G(L)->nilnode; + setmref(t->node, nilnode); +#if LJ_GC64 + setmref(t->freetop, nilnode); +#endif + if (asize > 0) { + if (asize > LJ_MAX_ASIZE) + lj_err_msg(L, LJ_ERR_TABOV); + setmref(t->array, lj_mem_newv(L, asize, TValue)); t->asize = asize; - t->hmask = 0; - nilnode = &G(L)->nilnode; - setmref(t->node, nilnode); -#if LJ_GC64 - setmref(t->freetop, nilnode); -#endif - } else { /* Otherwise separately allocate the array part. */ - Node *nilnode; - t = lj_mem_newobj(L, GCtab); - t->gct = ~LJ_TTAB; - t->nomm = (uint8_t)~0; - t->colo = 0; - setmref(t->array, NULL); - setgcrefnull(t->metatable); - t->asize = 0; /* In case the array allocation fails. */ - t->hmask = 0; - nilnode = &G(L)->nilnode; - setmref(t->node, nilnode); -#if LJ_GC64 - setmref(t->freetop, nilnode); -#endif - if (asize > 0) { - if (asize > LJ_MAX_ASIZE) - lj_err_msg(L, LJ_ERR_TABOV); - setmref(t->array, lj_mem_newvec(L, asize, TValue)); - t->asize = asize; - } } if (hbits) newhpart(L, t, hbits); return t; } + /* Create a new table. ** ** IMPORTANT NOTE: The API differs from lua_createtable()! @@ -211,19 +200,6 @@ void LJ_FASTCALL lj_tab_clear(GCtab *t) } } -/* Free a table. */ -void LJ_FASTCALL lj_tab_free(global_State *g, GCtab *t) -{ - if (t->hmask > 0) - lj_mem_freevec(g, noderef(t->node), t->hmask+1, Node); - if (t->asize > 0 && LJ_MAX_COLOSIZE != 0 && t->colo <= 0) - lj_mem_freevec(g, tvref(t->array), t->asize, TValue); - if (LJ_MAX_COLOSIZE != 0 && t->colo) - lj_mem_free(g, t, sizetabcolo((uint32_t)t->colo & 0x7f)); - else - lj_mem_freet(g, t); -} - /* -- Table resizing ------------------------------------------------------ */ /* Resize a table to fit the new array/hash part sizes. */ @@ -237,26 +213,34 @@ void lj_tab_resize(lua_State *L, GCtab *t, uint32_t asize, uint32_t hbits) uint32_t i; if (asize > LJ_MAX_ASIZE) lj_err_msg(L, LJ_ERR_TABOV); - if (LJ_MAX_COLOSIZE != 0 && t->colo > 0) { + if (LJ_COLO_ENABLED && t->colo > 0) { /* A colocated array must be separated and copied. */ TValue *oarray = tvref(t->array); - array = lj_mem_newvec(L, asize, TValue); - t->colo = (int8_t)(t->colo | 0x80); /* Mark as separated (colo < 0). */ + array = lj_mem_newv(L, asize, TValue); + t->colo = 0; /* Mark as separated (colo == 0). */ + t->gcflags &= (uint8_t)~LJ_GC_MARK_MASK; /* Release the slots. */ for (i = 0; i < oldasize; i++) copyTV(L, &array[i], &oarray[i]); } else { - array = (TValue *)lj_mem_realloc(L, tvref(t->array), - oldasize*sizeof(TValue), asize*sizeof(TValue)); + array = lj_mem_reallocv(L, tvref(t->array), oldasize, asize, TValue); } setmref(t->array, array); t->asize = asize; for (i = oldasize; i < asize; i++) /* Clear newly allocated slots. */ setnilV(&array[i]); + if (isblack(G(L), obj2gco(t))) { + /* If this is black then the array has been marked, + * so the new array needs marking too */ + lj_gc_markblob(L, mref(t->array, void), asize * sizeof(TValue)); + } } /* Create new (empty) hash part. */ if (hbits) { newhpart(L, t, hbits); clearhpart(t); + if (isblack(G(L), obj2gco(t))) { + lj_gc_markblob(L, mref(t->node, void), (t->hmask+1) * sizeof(Node)); + } } else { global_State *g = G(L); setmref(t->node, &g->nilnode); @@ -273,20 +257,21 @@ void lj_tab_resize(lua_State *L, GCtab *t, uint32_t asize, uint32_t hbits) if (!tvisnil(&array[i])) copyTV(L, lj_tab_setinth(L, t, (int32_t)i), &array[i]); /* Physically shrink only separated arrays. */ - if (LJ_MAX_COLOSIZE != 0 && t->colo <= 0) - setmref(t->array, lj_mem_realloc(L, array, - oldasize*sizeof(TValue), asize*sizeof(TValue))); + if (!LJ_COLO_ENABLED || t->colo == 0) { + setmref(t->array, + lj_mem_reallocv(L, tvref(t->array), oldasize, asize, TValue)); + if (isblack(G(L), obj2gco(t))) { + lj_gc_markblob(L, mref(t->array, void), asize * sizeof(TValue)); + } + } } if (oldhmask > 0) { /* Reinsert pairs from old hash part. */ - global_State *g; uint32_t i; for (i = 0; i <= oldhmask; i++) { Node *n = &oldnode[i]; if (!tvisnil(&n->val)) copyTV(L, lj_tab_set(L, t, &n->key), &n->val); } - g = G(L); - lj_mem_freevec(g, oldnode, oldhmask+1, Node); } } diff --git a/src/lj_tab.h b/src/lj_tab.h index cddc4765..8a2f7b3c 100644 --- a/src/lj_tab.h +++ b/src/lj_tab.h @@ -53,13 +53,13 @@ static LJ_AINLINE Node *hashmask(const GCtab *t, uint32_t hash) #define hsize2hbits(s) ((s) ? ((s)==1 ? 1 : 1+lj_fls((uint32_t)((s)-1))) : 0) LJ_FUNCA GCtab *lj_tab_new(lua_State *L, uint32_t asize, uint32_t hbits); +LJ_FUNCA GCtab *lj_tab_newgc(lua_State *L, uint32_t asize, uint32_t hbits); LJ_FUNC GCtab *lj_tab_new_ah(lua_State *L, int32_t a, int32_t h); #if LJ_HASJIT LJ_FUNC GCtab * LJ_FASTCALL lj_tab_new1(lua_State *L, uint32_t ahsize); #endif LJ_FUNCA GCtab * LJ_FASTCALL lj_tab_dup(lua_State *L, const GCtab *kt); LJ_FUNC void LJ_FASTCALL lj_tab_clear(GCtab *t); -LJ_FUNC void LJ_FASTCALL lj_tab_free(global_State *g, GCtab *t); #if LJ_HASFFI LJ_FUNC void lj_tab_rehash(lua_State *L, GCtab *t); #endif diff --git a/src/lj_target_x86.h b/src/lj_target_x86.h index 3482309b..e930efa4 100644 --- a/src/lj_target_x86.h +++ b/src/lj_target_x86.h @@ -261,6 +261,7 @@ typedef enum { XO_IMUL = XO_0f(af), XO_IMULi = XO_(69), XO_IMULi8 = XO_(6b), + XO_CMPb = XO_(38), XO_CMP = XO_(3b), XO_TESTb = XO_(84), XO_TEST = XO_(85), diff --git a/src/lj_trace.c b/src/lj_trace.c index fee78e2e..69009acc 100644 --- a/src/lj_trace.c +++ b/src/lj_trace.c @@ -130,7 +130,7 @@ GCtrace * LJ_FASTCALL lj_trace_alloc(lua_State *L, GCtrace *T) GCtrace *T2 = lj_mem_newt(L, (MSize)sz, GCtrace); char *p = (char *)T2 + sztr; T2->gct = ~LJ_TTRACE; - T2->marked = 0; + T2->gcflags = 0; T2->traceno = 0; T2->ir = (IRIns *)p - T->nk; T2->nins = T->nins; @@ -150,7 +150,7 @@ static void trace_save(jit_State *J, GCtrace *T) memcpy(T, &J->cur, sizeof(GCtrace)); setgcrefr(T->nextgc, J2G(J)->gc.root); setgcrefp(J2G(J)->gc.root, T); - newwhite(J2G(J), T); + newwhite(T); T->gct = ~LJ_TTRACE; T->ir = (IRIns *)p - J->cur.nk; /* The IR has already been copied above. */ #if LJ_ABI_PAUTH diff --git a/src/lj_udata.c b/src/lj_udata.c index 7acd9e32..21302c22 100644 --- a/src/lj_udata.c +++ b/src/lj_udata.c @@ -13,26 +13,15 @@ GCudata *lj_udata_new(lua_State *L, MSize sz, GCtab *env) { - GCudata *ud = lj_mem_newt(L, sizeof(GCudata) + sz, GCudata); - global_State *g = G(L); - newwhite(g, ud); /* Not finalized. */ - ud->gct = ~LJ_TUDATA; + GCudata *ud = lj_mem_allocudata(L, sz); ud->udtype = UDTYPE_USERDATA; ud->len = sz; /* NOBARRIER: The GCudata is new (marked white). */ setgcrefnull(ud->metatable); setgcref(ud->env, obj2gco(env)); - /* Chain to userdata list (after main thread). */ - setgcrefr(ud->nextgc, mainthread(g)->nextgc); - setgcref(mainthread(g)->nextgc, obj2gco(ud)); return ud; } -void LJ_FASTCALL lj_udata_free(global_State *g, GCudata *ud) -{ - lj_mem_free(g, ud, sizeudata(ud)); -} - #if LJ_64 void *lj_lightud_intern(lua_State *L, void *p) { diff --git a/src/lj_udata.h b/src/lj_udata.h index 143bf81d..cba2cde3 100644 --- a/src/lj_udata.h +++ b/src/lj_udata.h @@ -9,7 +9,6 @@ #include "lj_obj.h" LJ_FUNC GCudata *lj_udata_new(lua_State *L, MSize sz, GCtab *env); -LJ_FUNC void LJ_FASTCALL lj_udata_free(global_State *g, GCudata *ud); #if LJ_64 LJ_FUNC void * LJ_FASTCALL lj_lightud_intern(lua_State *L, void *p); #endif diff --git a/src/luajit_rolling.h b/src/luajit_rolling.h index e564477a..c0179ee5 100644 --- a/src/luajit_rolling.h +++ b/src/luajit_rolling.h @@ -73,6 +73,38 @@ LUA_API void luaJIT_profile_stop(lua_State *L); LUA_API const char *luaJIT_profile_dumpstack(lua_State *L, const char *fmt, int depth, size_t *len); +typedef unsigned (*luaJIT_allocpages)(void *ud, void **pages, unsigned n); +/* Free is called one last time with NULL, 0 to indicate any state should be released */ +typedef void (*luaJIT_freepages)(void *ud, void **pages, unsigned n); +typedef void* (*luaJIT_reallochuge)(void *ud, void *p, size_t osz, size_t nsz); + +/* This many bytes are reserved at the start of each huge arena for the allocator's use */ +#define LUAJIT_HUGE_RESERVED_SPACE 20 + +LUA_API size_t luaJIT_getpagesize(); + +/* Custom allocator support. + * Page allocators must yield naturally aligned regions in the size provided by + * luaJIT_getpagesize. Page allocators allocate and free N pages at once. Huge + * allocations are one at a time. + * The lua_Alloc allocator is used for allocations that require variable size + * and a fixed address. + * Allocators are permitted to fail. The huge page allocator can return NULL and + * the page allocator can return fewer than the requested amount. Allocators are not + * permitted to throw. + * Huge allocations have the same alignment requirements as normal arenas. + * Either the normal or page allocation can be NULL and the default will be used. + * Passing all NULL is functionally the same as luaL_newstate() + */ +LUA_API lua_State *luaJIT_newstate(lua_Alloc f, void *ud, + luaJIT_allocpages allocp, + luaJIT_freepages freep, + luaJIT_reallochuge realloch, + void *page_ud); + +/* As lua_createtable, but can be used with __gc */ +LUA_API void luaJIT_createtable(lua_State *L, int narray, int nrec); + /* Enforce (dynamic) linker error for version mismatches. Call from main. */ LUA_API void LUAJIT_VERSION_SYM(void); diff --git a/src/vm_x64.dasc b/src/vm_x64.dasc index 8c46ea59..010b6efc 100644 --- a/src/vm_x64.dasc +++ b/src/vm_x64.dasc @@ -53,8 +53,10 @@ |.define RDL, RCL |.define TMPR, r10 |.define TMPRd, r10d +|.define TMPRb, r10b |.define ITYPE, r11 |.define ITYPEd, r11d +|.define ITYPEb, r11b | |.if X64WIN |.define CARG1, rcx // x64/WIN64 C call arguments. @@ -89,6 +91,7 @@ |.type TAB, GCtab |.type LFUNC, GCfuncL |.type CFUNC, GCfuncC +|.type CFUNCD, GCfuncCData |.type PROTO, GCproto |.type UPVAL, GCupval |.type NODE, Node @@ -368,12 +371,26 @@ | |// Move table write barrier back. Overwrites reg. |.macro barrierback, tab, reg -| and byte tab->marked, (uint8_t)~LJ_GC_BLACK // black2gray(tab) +| or byte tab->gcflags, (uint8_t)LJ_GC_GRAY // black2gray(tab) | mov reg, [DISPATCH+DISPATCH_GL(gc.grayagain)] | mov [DISPATCH+DISPATCH_GL(gc.grayagain)], tab | mov tab->gclist, reg |.endmacro | +|// Generic isblack(o) macro +|.macro isblack, rD, rB, o, j +| movzx rD, byte o->gcflags +| and rD, LJ_GC_COLORS +| cmp rB, byte [DISPATCH+DISPATCH_GL(gc.currentblack)] +| jz j +|.endmacro +|.macro isnblack, rD, rB, o, j +| movzx rD, byte o->gcflags +| and rD, LJ_GC_COLORS +| cmp rB, byte [DISPATCH+DISPATCH_GL(gc.currentblack)] +| jnz j +|.endmacro +| |//----------------------------------------------------------------------- /* Generate subroutines used by opcodes and other parts of the VM. */ @@ -1205,7 +1222,8 @@ static void build_subroutines(BuildCtx *ctx) |2: | mov CFUNC:RB, [BASE-16] | cleartp CFUNC:RB - | mov STR:RC, [CFUNC:RB+RC*8+((char *)(&((GCfuncC *)0)->upvalue))] + | mov PC, CFUNC:RB->data + | mov STR:RC, [PC+RC*8+((char *)(&((GCfuncCData *)0)->upvalue))] | mov PC, [BASE-8] | settp STR:RC, LJ_TSTR | mov [BASE-16], STR:RC @@ -1265,8 +1283,7 @@ static void build_subroutines(BuildCtx *ctx) | mov TAB:RB->metatable, TAB:RA | mov PC, [BASE-8] | mov [BASE-16], TAB:TMPR // Return original table. - | test byte TAB:RB->marked, LJ_GC_BLACK // isblack(table) - | jz >1 + | isnblack RCd, RCL, TAB:RB, >1 | // Possible write barrier. Table is black, but skip iswhite(mt) check. | barrierback TAB:RB, RC |1: @@ -1375,7 +1392,8 @@ static void build_subroutines(BuildCtx *ctx) #endif | mov CFUNC:RD, [BASE-16] | cleartp CFUNC:RD - | mov CFUNC:RD, CFUNC:RD->upvalue[0] + | mov CFUNCD:RD, CFUNC:RD->data + | mov CFUNC:RD, CFUNCD:RD->upvalue[0] | settp CFUNC:RD, LJ_TFUNC | mov PC, [BASE-8] | mov [BASE-16], CFUNC:RD @@ -1446,7 +1464,8 @@ static void build_subroutines(BuildCtx *ctx) #endif | mov CFUNC:RD, [BASE-16] | cleartp CFUNC:RD - | mov CFUNC:RD, CFUNC:RD->upvalue[0] + | mov CFUNCD:RD, CFUNC:RD->data + | mov CFUNC:RD, CFUNCD:RD->upvalue[0] | settp CFUNC:RD, LJ_TFUNC | mov PC, [BASE-8] | mov [BASE-16], CFUNC:RD @@ -1508,7 +1527,8 @@ static void build_subroutines(BuildCtx *ctx) |.ffunc coroutine_wrap_aux | mov CFUNC:RB, [BASE-16] | cleartp CFUNC:RB - | mov L:RB, CFUNC:RB->upvalue[0].gcr + | mov CFUNCD:RB, CFUNC:RB->data + | mov L:RB, CFUNCD:RB->upvalue[0].gcr | cleartp L:RB |.endif | mov PC, [BASE-8] @@ -2143,7 +2163,8 @@ static void build_subroutines(BuildCtx *ctx) | cmp RA, L:RB->maxstack | ja >5 // Need to grow stack. | mov CARG1, L:RB - | call aword CFUNC:RD->f // (lua_State *L) + | mov CFUNCD:RD, CFUNC:RD->data + | call aword CFUNCD:RD->f // (lua_State *L) | mov BASE, L:RB->base | // Either throws an error, or recovers and returns -1, 0 or nresults+1. | test RDd, RDd; jg ->fff_res // Returned nresults+1? @@ -3445,7 +3466,8 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | ins_AD // RA = dst, RD = upvalue # | mov LFUNC:RB, [BASE-16] | cleartp LFUNC:RB - | mov UPVAL:RB, [LFUNC:RB+RD*8+offsetof(GCfuncL, uvptr)] + | mov RB, LFUNC:RB->uvptr + | mov UPVAL:RB, [RB+RD*8] | mov RB, UPVAL:RB->v | mov RD, [RB] | mov [BASE+RA*8], RD @@ -3453,18 +3475,20 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) break; case BC_USETV: #define TV2MARKOFS \ - ((int32_t)offsetof(GCupval, marked)-(int32_t)offsetof(GCupval, tv)) + ((int32_t)offsetof(GCupval, gcflags)-(int32_t)offsetof(GCupval, tv)) | ins_AD // RA = upvalue #, RD = src | mov LFUNC:RB, [BASE-16] | cleartp LFUNC:RB - | mov UPVAL:RB, [LFUNC:RB+RA*8+offsetof(GCfuncL, uvptr)] + | mov UPVAL:RB, LFUNC:RB->uvptr + | mov UPVAL:RB, [RB+RA*8] | cmp byte UPVAL:RB->closed, 0 | mov RB, UPVAL:RB->v | mov RA, [BASE+RD*8] | mov [RB], RA | jz >1 | // Check barrier for closed upvalue. - | test byte [RB+TV2MARKOFS], LJ_GC_BLACK // isblack(uv) + | movzx RDd, byte [DISPATCH+DISPATCH_GL(gc.currentblack)] + | test byte [RB+TV2MARKOFS], RDL // isblack(uv) | jnz >2 |1: | ins_next @@ -3476,8 +3500,9 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | cmp RDd, LJ_TNUMX - LJ_TISGCV // tvisgcv(v) | jbe <1 | cleartp GCOBJ:RA - | test byte GCOBJ:RA->gch.marked, LJ_GC_WHITES // iswhite(v) - | jz <1 + | movzx TMPRd, byte [DISPATCH+DISPATCH_GL(gc.currentblackgray)] + | test byte GCOBJ:RA->gch.gcflags, TMPRb // iswhite(v) + | jnz <1 | // Crossed a write barrier. Move the barrier forward. |.if not X64WIN | mov CARG2, RB @@ -3495,19 +3520,20 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | ins_AND // RA = upvalue #, RD = str const (~) | mov LFUNC:RB, [BASE-16] | cleartp LFUNC:RB - | mov UPVAL:RB, [LFUNC:RB+RA*8+offsetof(GCfuncL, uvptr)] + | mov UPVAL:RB, LFUNC:RB->uvptr + | mov UPVAL:RB, [RB+RA*8] | mov STR:RA, [KBASE+RD*8] | mov RD, UPVAL:RB->v | settp STR:ITYPE, STR:RA, LJ_TSTR | mov [RD], STR:ITYPE - | test byte UPVAL:RB->marked, LJ_GC_BLACK // isblack(uv) - | jnz >2 + | isblack TMPRd, TMPRb, UPVAL:RB, >2 // isblack(uv) |1: | ins_next | |2: // Check if string is white and ensure upvalue is closed. - | test byte GCOBJ:RA->gch.marked, LJ_GC_WHITES // iswhite(str) - | jz <1 + | movzx TMPRd, byte [DISPATCH+DISPATCH_GL(gc.currentblackgray)] + | test byte GCOBJ:RA->gch.gcflags, TMPRb // iswhite(str) + | jnz <1 | cmp byte UPVAL:RB->closed, 0 | jz <1 | // Crossed a write barrier. Move the barrier forward. @@ -3523,7 +3549,8 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | mov LFUNC:RB, [BASE-16] | cleartp LFUNC:RB | movsd xmm0, qword [KBASE+RD*8] - | mov UPVAL:RB, [LFUNC:RB+RA*8+offsetof(GCfuncL, uvptr)] + | mov UPVAL:RB, LFUNC:RB->uvptr + | mov UPVAL:RB, [RB+RA*8] | mov RA, UPVAL:RB->v | movsd qword [RA], xmm0 | ins_next @@ -3532,7 +3559,8 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | ins_AD // RA = upvalue #, RD = primitive type (~) | mov LFUNC:RB, [BASE-16] | cleartp LFUNC:RB - | mov UPVAL:RB, [LFUNC:RB+RA*8+offsetof(GCfuncL, uvptr)] + | mov UPVAL:RB, LFUNC:RB->uvptr + | mov UPVAL:RB, [RB+RA*8] | shl RD, 47 | not RD | mov RA, UPVAL:RB->v @@ -3802,8 +3830,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | cmp aword [RC], LJ_TNIL | je >3 // Previous value is nil? |1: - | test byte TAB:RB->marked, LJ_GC_BLACK // isblack(table) - | jnz >7 + | isblack TMPRd, TMPRb, TAB:RB, >7 // isblack(table) |2: // Set array slot. | mov RB, [BASE+RA*8] | mov [RC], RB @@ -3846,8 +3873,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | cmp aword [TMPR], LJ_TNIL | je >4 // Previous value is nil? |2: - | test byte TAB:RB->marked, LJ_GC_BLACK // isblack(table) - | jnz >7 + | isblack ITYPEd, ITYPEb, TAB:RB, >7 // isblack(table) |3: // Set node value. | mov ITYPE, [BASE+RA*8] | mov [TMPR], ITYPE @@ -3903,8 +3929,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | cmp aword [RC], LJ_TNIL | je >3 // Previous value is nil? |1: - | test byte TAB:RB->marked, LJ_GC_BLACK // isblack(table) - | jnz >7 + | isblack TMPRd, TMPRb, TAB:RB, >7 // isblack(table) |2: // Set array slot. | mov ITYPE, [BASE+RA*8] | mov [RC], ITYPE @@ -3931,8 +3956,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) |.else | cvttsd2si RCd, qword [BASE+RC*8] |.endif - | test byte TAB:RB->marked, LJ_GC_BLACK // isblack(table) - | jnz >7 + | isblack TMPRd, TMPRb, TAB:RB, >7 // isblack(table) |2: | cmp RCd, TAB:RB->asize | jae ->vmeta_tsetr @@ -3956,8 +3980,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | lea RA, [BASE+RA*8] | mov TAB:RB, [RA-8] // Guaranteed to be a table. | cleartp TAB:RB - | test byte TAB:RB->marked, LJ_GC_BLACK // isblack(table) - | jnz >7 + | isblack RDd, RDL, TAB:RB, >7 // isblack(table) |2: | mov RDd, MULTRES | sub RDd, 1 @@ -4670,7 +4693,8 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | ins_AD // BASE = new base, RA = ins RA|RD (unused), RD = nargs+1 | mov CFUNC:RB, [BASE-16] | cleartp CFUNC:RB - | mov KBASE, CFUNC:RB->f + | mov CFUNCD:KBASE, CFUNC:RB->data + | mov KBASE, CFUNCD:KBASE->f | mov L:RB, SAVE_L | lea RD, [BASE+NARGS:RD*8-8] | mov L:RB->base, BASE