Skip to content

Commit

Permalink
Merge branch 'sverker/one_offheap_list/OTP-8737' into dev
Browse files Browse the repository at this point in the history 
* sverker/one_offheap_list/OTP-8737:
  One off-heap list, to eliminate two words per ETS object.
  • Loading branch information
@sverker
sverker committed Jul 20, 2010
2 parents 9f44863 + db20676 commit 9267b2d
Show file tree
Hide file tree
Showing 32 changed files with 465 additions and 821 deletions.
47 changes: 25 additions & 22 deletions erts/emulator/beam/beam_bif_load.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -402,7 +402,7 @@ check_process_code(Process* rp, Module* modp)
BeamInstr* end;
Eterm* sp;
#ifndef HYBRID /* FIND ME! */
ErlFunThing* funp;
struct erl_off_heap_header* oh;
int done_gc = 0;
#endif

Expand Down Expand Up @@ -470,27 +470,30 @@ check_process_code(Process* rp, Module* modp)

#ifndef HYBRID /* FIND ME! */
rescan:
for (funp = MSO(rp).funs; funp; funp = funp->next) {
BeamInstr* fun_code;

fun_code = funp->fe->address;

if (INSIDE((BeamInstr *) funp->fe->address)) {
if (done_gc) {
return am_true;
} else {
/*
* Try to get rid of this fun by garbage collecting.
* Clear both fvalue and ftrace to make sure they
* don't hold any funs.
*/
rp->freason = EXC_NULL;
rp->fvalue = NIL;
rp->ftrace = NIL;
done_gc = 1;
FLAGS(rp) |= F_NEED_FULLSWEEP;
(void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
goto rescan;
for (oh = MSO(rp).first; oh; oh = oh->next) {
if (thing_subtag(oh->thing_word) == FUN_SUBTAG) {
ErlFunThing* funp = (ErlFunThing*) oh;
BeamInstr* fun_code;

fun_code = funp->fe->address;

if (INSIDE((BeamInstr *) funp->fe->address)) {
if (done_gc) {
return am_true;
} else {
/*
* Try to get rid of this fun by garbage collecting.
* Clear both fvalue and ftrace to make sure they
* don't hold any funs.
*/
rp->freason = EXC_NULL;
rp->fvalue = NIL;
rp->ftrace = NIL;
done_gc = 1;
FLAGS(rp) |= F_NEED_FULLSWEEP;
(void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
goto rescan;
}
}
}
}
Expand Down
12 changes: 6 additions & 6 deletions erts/emulator/beam/beam_emu.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3385,8 +3385,8 @@ void process_main(void)
HTOP += PROC_BIN_SIZE;
pb->thing_word = HEADER_PROC_BIN;
pb->size = num_bytes;
pb->next = MSO(c_p).mso;
MSO(c_p).mso = pb;
pb->next = MSO(c_p).first;
MSO(c_p).first = (struct erl_off_heap_header*) pb;
pb->val = bptr;
pb->bytes = (byte*) bptr->orig_bytes;
pb->flags = 0;
Expand Down Expand Up @@ -3486,8 +3486,8 @@ void process_main(void)
HTOP += PROC_BIN_SIZE;
pb->thing_word = HEADER_PROC_BIN;
pb->size = tmp_arg1;
pb->next = MSO(c_p).mso;
MSO(c_p).mso = pb;
pb->next = MSO(c_p).first;
MSO(c_p).first = (struct erl_off_heap_header*) pb;
pb->val = bptr;
pb->bytes = (byte*) bptr->orig_bytes;
pb->flags = 0;
Expand Down Expand Up @@ -6323,8 +6323,8 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free)
erts_refc_inc(&fe->refc, 2);
funp->thing_word = HEADER_FUN;
#ifndef HYBRID /* FIND ME! */
funp->next = MSO(p).funs;
MSO(p).funs = funp;
funp->next = MSO(p).first;
MSO(p).first = (struct erl_off_heap_header*) funp;
#endif
funp->fe = fe;
funp->num_free = num_free;
Expand Down
4 changes: 2 additions & 2 deletions erts/emulator/beam/bif.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3611,11 +3611,11 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1)

etp = (ExternalThing *) HAlloc(BIF_P, EXTERNAL_THING_HEAD_SIZE + 1);
etp->header = make_external_pid_header(1);
etp->next = MSO(BIF_P).externals;
etp->next = MSO(BIF_P).first;
etp->node = enp;
etp->data.ui[0] = make_pid_data(c, b);

MSO(BIF_P).externals = etp;
MSO(BIF_P).first = (struct erl_off_heap_header*) etp;
erts_deref_dist_entry(dep);
BIF_RET(make_external_pid(etp));
}
Expand Down
18 changes: 4 additions & 14 deletions erts/emulator/beam/binary.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,8 @@ new_binary(Process *p, byte *buf, int len)
pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE);
pb->thing_word = HEADER_PROC_BIN;
pb->size = len;
pb->next = MSO(p).mso;
MSO(p).mso = pb;
pb->next = MSO(p).first;
MSO(p).first = (struct erl_off_heap_header*)pb;
pb->val = bptr;
pb->bytes = (byte*) bptr->orig_bytes;
pb->flags = 0;
Expand Down Expand Up @@ -127,8 +127,8 @@ Eterm erts_new_mso_binary(Process *p, byte *buf, int len)
pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE);
pb->thing_word = HEADER_PROC_BIN;
pb->size = len;
pb->next = MSO(p).mso;
MSO(p).mso = pb;
pb->next = MSO(p).first;
MSO(p).first = (struct erl_off_heap_header*)pb;
pb->val = bptr;
pb->bytes = (byte*) bptr->orig_bytes;
pb->flags = 0;
Expand Down Expand Up @@ -487,16 +487,6 @@ BIF_RETTYPE split_binary_2(BIF_ALIST_2)
BIF_ERROR(BIF_P, BADARG);
}

void
erts_cleanup_mso(ProcBin* pb)
{
while (pb != NULL) {
ProcBin* next = pb->next;
if (erts_refc_dectest(&pb->val->refc, 0) == 0)
erts_bin_free(pb->val);
pb = next;
}
}

/*
* Local functions.
Expand Down
32 changes: 16 additions & 16 deletions erts/emulator/beam/break.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -611,29 +611,29 @@ static void
bin_check(void)
{
Process *rp;
ProcBin *bp;
int i, printed;
struct erl_off_heap_header* hdr;
int i, printed = 0;

for (i=0; i < erts_max_processes; i++) {
if ((rp = process_tab[i]) == NULL)
continue;
if (!(bp = rp->off_heap.mso))
continue;
printed = 0;
while (bp) {
if (printed == 0) {
erts_printf("Process %T holding binary data \n", rp->id);
printed = 1;
for (hdr = rp->off_heap.first; hdr; hdr = hdr->next) {
if (hdr->thing_word == HEADER_PROC_BIN) {
ProcBin *bp = (ProcBin*) hdr;
if (!printed) {
erts_printf("Process %T holding binary data \n", rp->id);
printed = 1;
}
erts_printf("0x%08lx orig_size: %ld, norefs = %ld\n",
(unsigned long)bp->val,
(long)bp->val->orig_size,
erts_smp_atomic_read(&bp->val->refc));
}
erts_printf("0x%08lx orig_size: %ld, norefs = %ld\n",
(unsigned long)bp->val,
(long)bp->val->orig_size,
erts_smp_atomic_read(&bp->val->refc));

bp = bp->next;
}
if (printed == 1)
if (printed) {
erts_printf("--------------------------------------\n");
printed = 0;
}
}
/* db_bin_check() has to be rewritten for the AVL trees... */
/*db_bin_check();*/
Expand Down
98 changes: 36 additions & 62 deletions erts/emulator/beam/copy.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -314,9 +314,9 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
*argp = make_binary(hbot);
pb = (ProcBin*) hbot;
erts_refc_inc(&pb->val->refc, 2);
pb->next = off_heap->mso;
pb->next = off_heap->first;
pb->flags = 0;
off_heap->mso = pb;
off_heap->first = (struct erl_off_heap_header*) pb;
off_heap->overhead += pb->size / sizeof(Eterm);
}
break;
Expand Down Expand Up @@ -363,9 +363,9 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
to->val = from->val;
erts_refc_inc(&to->val->refc, 2);
to->bytes = from->bytes + offset;
to->next = off_heap->mso;
to->next = off_heap->first;
to->flags = 0;
off_heap->mso = to;
off_heap->first = (struct erl_off_heap_header*) to;
off_heap->overhead += to->size / sizeof(Eterm);
}
*argp = make_binary(hbot);
Expand Down Expand Up @@ -396,8 +396,8 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
}
#ifndef HYBRID /* FIND ME! */
funp = (ErlFunThing *) tp;
funp->next = off_heap->funs;
off_heap->funs = funp;
funp->next = off_heap->first;
off_heap->first = (struct erl_off_heap_header*) funp;
erts_refc_inc(&funp->fe->refc, 2);
#endif
*argp = make_fun(tp);
Expand All @@ -416,8 +416,8 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
*htop++ = *objp++;
}

etp->next = off_heap->externals;
off_heap->externals = etp;
etp->next = off_heap->first;
off_heap->first = (struct erl_off_heap_header*)etp;
erts_refc_inc(&etp->node->refc, 2);

*argp = make_external(tp);
Expand Down Expand Up @@ -650,8 +650,8 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs)
*hp++ = *objp++;
}
erts_refc_inc(&pb->val->refc, 2);
pb->next = erts_global_offheap.mso;
erts_global_offheap.mso = pb;
pb->next = erts_global_offheap.first;
erts_global_offheap.first = pb;
erts_global_offheap.overhead += pb->size / sizeof(Eterm);
continue;
}
Expand All @@ -672,9 +672,9 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs)
while (i--) {
*hp++ = *objp++;
}
#ifndef HYBRID // FIND ME!
funp->next = erts_global_offheap.funs;
erts_global_offheap.funs = funp;
#ifndef HYBRID /* FIND ME! */
funp->next = erts_global_offheap.first;
erts_global_offheap.first = funp;
erts_refc_inc(&funp->fe->refc, 2);
#endif
for (i = k; i < j; i++) {
Expand Down Expand Up @@ -718,8 +718,8 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs)
*hp++ = *objp++;
}

etp->next = erts_global_offheap.externals;
erts_global_offheap.externals = etp;
etp->next = erts_global_offheap.first;
erts_global_offheap.first = etp;
erts_refc_inc(&etp->node->refc, 2);
continue;
}
Expand Down Expand Up @@ -775,8 +775,8 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs)
to_bin->size = real_size;
to_bin->val = from_bin->val;
to_bin->bytes = from_bin->bytes + sub_offset;
to_bin->next = erts_global_offheap.mso;
erts_global_offheap.mso = to_bin;
to_bin->next = erts_global_offheap.first;
erts_global_offheap.first = to_bin;
erts_global_offheap.overhead += to_bin->size / sizeof(Eterm);
res_binary=make_binary(to_bin);
hp += PROC_BIN_SIZE;
Expand Down Expand Up @@ -910,57 +910,43 @@ copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
break;
case REFC_BINARY_SUBTAG:
{
ProcBin* pb = (ProcBin *) (hp-1);
int tari = thing_arityval(val);

sz -= tari;
while (tari--) {
*hp++ = *tp++;
}
ProcBin* pb = (ProcBin *) (tp-1);
erts_refc_inc(&pb->val->refc, 2);
pb->next = off_heap->mso;
off_heap->mso = pb;
off_heap->overhead += pb->size / sizeof(Eterm);
}
break;
goto off_heap_common;

case FUN_SUBTAG:
{
#ifndef HYBRID /* FIND ME! */
ErlFunThing* funp = (ErlFunThing *) (hp-1);
#endif
int tari = thing_arityval(val);

sz -= tari;
while (tari--) {
*hp++ = *tp++;
}
#ifndef HYBRID /* FIND ME! */
funp->next = off_heap->funs;
off_heap->funs = funp;
ErlFunThing* funp = (ErlFunThing *) (tp-1);
erts_refc_inc(&funp->fe->refc, 2);
#endif
}
break;
goto off_heap_common;

case EXTERNAL_PID_SUBTAG:
case EXTERNAL_PORT_SUBTAG:
case EXTERNAL_REF_SUBTAG:
{
ExternalThing* etp = (ExternalThing *) (hp-1);
ExternalThing* etp = (ExternalThing *) (tp-1);
erts_refc_inc(&etp->node->refc, 2);
}
off_heap_common:
{
struct erl_off_heap_header* ohh = (struct erl_off_heap_header*)(hp-1);
int tari = thing_arityval(val);

sz -= tari;
while (tari--) {
*hp++ = *tp++;
}
etp->next = off_heap->externals;
off_heap->externals = etp;
erts_refc_inc(&etp->node->refc, 2);
ohh->next = off_heap->first;
off_heap->first = ohh;
}
break;
default:
{
int tari = header_arity(val);

sz -= tari;
while (tari--) {
*hp++ = *tp++;
Expand Down Expand Up @@ -1029,12 +1015,6 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first,
static void
move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap* off_heap)
{
union {
Uint *up;
ProcBin *pbp;
ErlFunThing *efp;
ExternalThing *etp;
} ohe;
Eterm* ptr = src;
Eterm* end = ptr + src_sz;
Eterm dummy_ref;
Expand All @@ -1046,23 +1026,17 @@ move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap* off_heap)
val = *ptr;
ASSERT(val != ERTS_HOLE_MARKER);
if (is_header(val)) {
struct erl_off_heap_header* hdr = (struct erl_off_heap_header*)hp;
ASSERT(ptr + header_arity(val) < end);
ohe.up = hp;
MOVE_BOXED(ptr, val, hp, &dummy_ref);
switch (val & _HEADER_SUBTAG_MASK) {
case REFC_BINARY_SUBTAG:
ohe.pbp->next = off_heap->mso;
off_heap->mso = ohe.pbp;
break;
case FUN_SUBTAG:
ohe.efp->next = off_heap->funs;
off_heap->funs = ohe.efp;
break;
case EXTERNAL_PID_SUBTAG:
case EXTERNAL_PORT_SUBTAG:
case EXTERNAL_REF_SUBTAG:
ohe.etp->next = off_heap->externals;
off_heap->externals = ohe.etp;
hdr->next = off_heap->first;
off_heap->first = hdr;
break;
}
}
Expand Down
Loading

0 comments on commit 9267b2d

Please sign in to comment.