Mercurial > hg > jdk9-shenandoah > hotspot
changeset 9753:82e2de7b5401
Merge
line wrap: on
line diff
--- a/.hgtags Fri Sep 25 22:59:24 2015 -0700 +++ b/.hgtags Tue Sep 29 18:47:52 2015 +0200 @@ -374,6 +374,7 @@ acac3bde66b2c22791c257a8d99611d6d08c6713 jdk8-b105 18b4798adbc42c6fa16f5ecb7d5cd3ca130754bf hs25-b48 aed585cafc0d9655726af6d1e1081d1c94cb3b5c jdk8-b106 +54d28edc92cd688d0a4f7a0db720e34b7a7562ea concurrent 50794d8ac11c9579b41dec4de23b808fef9f34a1 hs25-b49 5b7f90aab3ad25a25b75b7b2bb18d5ae23d8231c jdk8-b107 a09fe9d1e016c285307507a5793bc4fa6215e9c9 hs25-b50
--- a/make/Makefile Fri Sep 25 22:59:24 2015 -0700 +++ b/make/Makefile Tue Sep 29 18:47:52 2015 +0200 @@ -204,7 +204,7 @@ generic_build1: $(HOTSPOT_SCRIPT) $(MKDIR) -p $(OUTPUTDIR) ifeq ($(OSNAME),windows) - ifeq ($(ARCH_DATA_MODEL), 32) +# ifeq ($(ARCH_DATA_MODEL), 32) $(CD) $(OUTPUTDIR); \ $(NMAKE) -f $(ABS_OS_MAKEFILE) \ Variant=compiler1 \ @@ -212,17 +212,17 @@ BootStrapDir=$(ABS_BOOTDIR) \ BuildUser=$(USERNAME) \ $(MAKE_ARGS) $(VM_TARGET:%1=%) - else - @$(ECHO) "No compiler1 ($(VM_TARGET)) for ARCH_DATA_MODEL=$(ARCH_DATA_MODEL)" - endif +# else +# @$(ECHO) "No compiler1 ($(VM_TARGET)) for ARCH_DATA_MODEL=$(ARCH_DATA_MODEL)" +# endif else - ifeq ($(ARCH_DATA_MODEL), 32) +# ifeq ($(ARCH_DATA_MODEL), 32) $(CD) $(OUTPUTDIR); \ $(MAKE) -f $(ABS_OS_MAKEFILE) \ $(MAKE_ARGS) $(VM_TARGET) - else - @$(ECHO) "No compiler1 ($(VM_TARGET)) for ARCH_DATA_MODEL=$(ARCH_DATA_MODEL)" - endif +# else +# @$(ECHO) "No compiler1 ($(VM_TARGET)) for ARCH_DATA_MODEL=$(ARCH_DATA_MODEL)" +# endif endif # Build compiler2 (server) rule, different for platforms @@ -656,11 +656,11 @@ # C1 test targets test_product1 test_optimized1 test_fastdebug1 test_debug1: - ifeq ($(ARCH_DATA_MODEL), 32) +# ifeq ($(ARCH_DATA_MODEL), 32) @$(MAKE) generic_test ALTJVM_DIR="$(C1_DIR)/$(@:test_%1=%)" - else - @$(ECHO) "No compiler1 ($(@:test_%=%)) for ARCH_DATA_MODEL=$(ARCH_DATA_MODEL)" - endif +# else +# @$(ECHO) "No compiler1 ($(@:test_%=%)) for ARCH_DATA_MODEL=$(ARCH_DATA_MODEL)" +# endif # Zero test targets test_productzero test_optimizedzero test_fastdebugzero test_debugzero:
--- a/make/linux/makefiles/gcc.make Fri Sep 25 22:59:24 2015 -0700 +++ b/make/linux/makefiles/gcc.make Tue Sep 29 18:47:52 2015 +0200 @@ -231,7 +231,7 @@ OPT_CFLAGS/NOOPT=-O0 OPT_CFLAGS/DEBUG=-O0 OPT_CFLAGS/SIZE=-Os -OPT_CFLAGS/SPEED=-O3 +OPT_CFLAGS/SPEED=-O3 -D_NMT_NOINLINE_ OPT_CFLAGS_DEFAULT ?= SPEED
--- a/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -32,6 +32,7 @@ #include "c1/c1_ValueStack.hpp" #include "ci/ciArrayKlass.hpp" #include "ci/ciInstance.hpp" +#include "gc/shenandoah/shenandoahHeap.hpp" #include "gc/shared/barrierSet.hpp" #include "gc/shared/cardTableModRefBS.hpp" #include "gc/shared/collectedHeap.hpp" @@ -260,7 +261,6 @@ } } - Address LIR_Assembler::as_Address_hi(LIR_Address* addr) { Address base = as_Address(addr); return Address(base._base, base._index, base._scale, base._disp + BytesPerWord); @@ -271,7 +271,6 @@ return as_Address(addr); } - void LIR_Assembler::osr_entry() { offsets()->set_value(CodeOffsets::OSR_Entry, code_offset()); BlockBegin* osr_entry = compilation()->hir()->osr_entry(); @@ -809,7 +808,7 @@ } else { if (is_literal_address(addr)) { ShouldNotReachHere(); - __ movoop(as_Address(addr, noreg), c->as_jobject()); + __ movoop(as_Address(addr), c->as_jobject()); } else { #ifdef _LP64 __ movoop(rscratch1, c->as_jobject()); @@ -1017,34 +1016,36 @@ #endif } + int null_check_here = code_offset(); + Address addr = as_Address(to_addr); + if (patch_code != lir_patch_none) { patch = new PatchingStub(_masm, PatchingStub::access_field_id); - Address toa = as_Address(to_addr); + Address toa = addr; assert(toa.disp() != 0, "must have"); } - int null_check_here = code_offset(); switch (type) { case T_FLOAT: { if (src->is_single_xmm()) { - __ movflt(as_Address(to_addr), src->as_xmm_float_reg()); + __ movflt(addr, src->as_xmm_float_reg()); } else { assert(src->is_single_fpu(), "must be"); assert(src->fpu_regnr() == 0, "argument must be on TOS"); - if (pop_fpu_stack) __ fstp_s(as_Address(to_addr)); - else __ fst_s (as_Address(to_addr)); + if (pop_fpu_stack) __ fstp_s(addr); + else __ fst_s (addr); } break; } case T_DOUBLE: { if (src->is_double_xmm()) { - __ movdbl(as_Address(to_addr), src->as_xmm_double_reg()); + __ movdbl(addr, src->as_xmm_double_reg()); } else { assert(src->is_double_fpu(), "must be"); assert(src->fpu_regnrLo() == 0, "argument must be on TOS"); - if (pop_fpu_stack) __ fstp_d(as_Address(to_addr)); - else __ fst_d (as_Address(to_addr)); + if (pop_fpu_stack) __ fstp_d(addr); + else __ fst_d (addr); } break; } @@ -1052,9 +1053,9 @@ case T_ARRAY: // fall through case T_OBJECT: // fall through if (UseCompressedOops && !wide) { - __ movl(as_Address(to_addr), compressed_src); + __ movl(addr, compressed_src); } else { - __ movptr(as_Address(to_addr), src->as_register()); + __ movptr(addr, src->as_register()); } break; case T_METADATA: @@ -1063,20 +1064,20 @@ // compressed klass ptrs: T_METADATA can be a compressed klass // ptr or a 64 bit method pointer. LP64_ONLY(ShouldNotReachHere()); - __ movptr(as_Address(to_addr), src->as_register()); + __ movptr(addr, src->as_register()); break; case T_ADDRESS: - __ movptr(as_Address(to_addr), src->as_register()); + __ movptr(addr, src->as_register()); break; case T_INT: - __ movl(as_Address(to_addr), src->as_register()); + __ movl(addr, src->as_register()); break; case T_LONG: { Register from_lo = src->as_register_lo(); Register from_hi = src->as_register_hi(); #ifdef _LP64 - __ movptr(as_Address_lo(to_addr), from_lo); + __ movptr(addr, from_lo); #else Register base = to_addr->base()->as_register(); Register index = noreg; @@ -1110,7 +1111,7 @@ case T_BYTE: // fall through case T_BOOLEAN: { Register src_reg = src->as_register(); - Address dst_addr = as_Address(to_addr); + Address dst_addr = addr; assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6"); __ movb(dst_addr, src_reg); break; @@ -1118,7 +1119,7 @@ case T_CHAR: // fall through case T_SHORT: - __ movw(as_Address(to_addr), src->as_register()); + __ movw(addr, src->as_register()); break; default: @@ -1217,6 +1218,10 @@ assert(dest->is_register(), "should not call otherwise"); LIR_Address* addr = src->as_address_ptr(); + + if (info != NULL) { + add_debug_info_for_null_check_here(info); + } Address from_addr = as_Address(addr); if (addr->base()->type() == T_OBJECT) { @@ -1294,7 +1299,7 @@ Register to_lo = dest->as_register_lo(); Register to_hi = dest->as_register_hi(); #ifdef _LP64 - __ movptr(to_lo, as_Address_lo(addr)); + __ movptr(to_lo, from_addr); #else Register base = addr->base()->as_register(); Register index = noreg; @@ -1466,6 +1471,55 @@ } } +void LIR_Assembler::emit_opShenandoahWriteBarrier(LIR_OpShenandoahWriteBarrier* op) { + Label done; + Register obj = op->in_opr()->as_register(); + Register res = op->result_opr()->as_register(); + Register tmp1 = op->tmp1_opr()->as_register(); + Register tmp2 = op->tmp2_opr()->as_register(); + assert_different_registers(res, tmp1, tmp2); + + if (res != obj) { + __ mov(res, obj); + } + + // Check for null. + if (op->need_null_check()) { + __ testptr(res, res); + __ jcc(Assembler::zero, done); + } + + // Check for evacuation-in-progress + Address evacuation_in_progress = Address(r15_thread, in_bytes(JavaThread::evacuation_in_progress_offset())); + __ cmpb(evacuation_in_progress, 0); + + // The read-barrier. + __ movptr(res, Address(res, -8)); + + __ jcc(Assembler::equal, done); + + // Check for object in collection set. + __ movptr(tmp1, res); + __ shrptr(tmp1, ShenandoahHeapRegion::RegionSizeShift); + __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr()); + __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1)); + __ testb(tmp2, 0x1); + __ jcc(Assembler::zero, done); + + if (res != rax) { + __ xchgptr(res, rax); // Move obj into rax and save rax into obj. + } + + __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::shenandoah_write_barrier_slow_id))); + + if (res != rax) { + __ xchgptr(rax, res); // Swap back obj with rax. + } + + __ bind(done); + +} + void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) { LIR_Opr src = op->in_opr(); LIR_Opr dest = op->result_opr(); @@ -1962,10 +2016,43 @@ } else #endif { - if (os::is_MP()) { - __ lock(); + if (UseShenandoahGC) { + Label done; + Label retry; + + __ bind(retry); + + // Save original cmp-value into tmp1, before following cas destroys it. + __ movptr(op->tmp1()->as_register(), op->cmp_value()->as_register()); + + if (os::is_MP()) { + __ lock(); + } + __ cmpxchgptr(newval, Address(addr, 0)); + + // If the cmpxchg succeeded, then we're done. + __ jcc(Assembler::equal, done); + + // Resolve the original cmp value. + oopDesc::bs()->interpreter_read_barrier(masm(), op->tmp1()->as_register()); + // Resolve the old value at address. We get the old value in cmp/rax + // when the comparison in cmpxchg failed. + __ movptr(op->tmp2()->as_register(), cmpval); + oopDesc::bs()->interpreter_read_barrier(masm(), op->tmp2()->as_register()); + + // We're done if the expected/cmp value is not the same as old. It's a valid + // cmpxchg failure then. Otherwise we need special treatment for Shenandoah + // to prevent false positives. + __ cmpptr(op->tmp1()->as_register(), op->tmp2()->as_register()); + __ jcc(Assembler::equal, retry); + + __ bind(done); + } else { + if (os::is_MP()) { + __ lock(); + } + __ cmpxchgptr(newval, Address(addr, 0)); } - __ cmpxchgptr(newval, Address(addr, 0)); } } else { assert(op->code() == lir_cas_int, "lir_cas_int expected"); @@ -3333,6 +3420,7 @@ __ movl(tmp, Address(tmp, Klass::super_check_offset_offset())); __ push(tmp); __ push(length); + __ lea(tmp, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type))); __ push(tmp); __ lea(tmp, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type))); @@ -3985,10 +4073,11 @@ if (data->type() == T_INT) { if (code == lir_xadd) { + Address addr = as_Address(src->as_address_ptr()); if (os::is_MP()) { __ lock(); } - __ xaddl(as_Address(src->as_address_ptr()), data->as_register()); + __ xaddl(addr, data->as_register()); } else { __ xchgl(data->as_register(), as_Address(src->as_address_ptr())); } @@ -4010,10 +4099,11 @@ #ifdef _LP64 assert(data->as_register_lo() == data->as_register_hi(), "should be a single register"); if (code == lir_xadd) { + Address addr = as_Address(src->as_address_ptr()); if (os::is_MP()) { __ lock(); } - __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo()); + __ xaddq(addr, data->as_register_lo()); } else { __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr())); }
--- a/src/cpu/x86/vm/c1_LIRGenerator_x86.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/cpu/x86/vm/c1_LIRGenerator_x86.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -300,15 +300,27 @@ null_check_info = new CodeEmitInfo(range_check_info); } + LIR_Opr ary = array.result(); + ary = shenandoah_write_barrier(ary, null_check_info, x->needs_null_check()); + LIR_Opr val = value.result(); + if (obj_store) { + if (! val->is_register()) { + LIR_Opr tmp = new_register(T_OBJECT); + __ move(val, tmp); + val = tmp; + } + val = shenandoah_read_barrier(val, NULL, true); + } + // emit array address setup early so it schedules better - LIR_Address* array_addr = emit_array_address(array.result(), index.result(), x->elt_type(), obj_store); + LIR_Address* array_addr = emit_array_address(ary, index.result(), x->elt_type(), obj_store); if (GenerateRangeChecks && needs_range_check) { if (use_length) { __ cmp(lir_cond_belowEqual, length.result(), index.result()); __ branch(lir_cond_belowEqual, T_INT, new RangeCheckStub(range_check_info, index.result())); } else { - array_range_check(array.result(), index.result(), null_check_info, range_check_info); + array_range_check(ary, index.result(), null_check_info, range_check_info); // range_check also does the null check null_check_info = NULL; } @@ -320,18 +332,18 @@ LIR_Opr tmp3 = new_register(objectType); CodeEmitInfo* store_check_info = new CodeEmitInfo(range_check_info); - __ store_check(value.result(), array.result(), tmp1, tmp2, tmp3, store_check_info, x->profiled_method(), x->profiled_bci()); + __ store_check(val, ary, tmp1, tmp2, tmp3, store_check_info, x->profiled_method(), x->profiled_bci()); } if (obj_store) { // Needs GC write barriers. pre_barrier(LIR_OprFact::address(array_addr), LIR_OprFact::illegalOpr /* pre_val */, true /* do_load */, false /* patch */, NULL); - __ move(value.result(), array_addr, null_check_info); + __ move(val, array_addr, null_check_info); // Seems to be a precise post_barrier(LIR_OprFact::address(array_addr), value.result()); } else { - __ move(value.result(), array_addr, null_check_info); + __ move(val, array_addr, null_check_info); } } @@ -358,7 +370,9 @@ // this CodeEmitInfo must not have the xhandlers because here the // object is already locked (xhandlers expect object to be unlocked) CodeEmitInfo* info = state_for(x, x->state(), true); - monitor_enter(obj.result(), lock, syncTempOpr(), scratch, + LIR_Opr obj_opr = obj.result(); + obj_opr = shenandoah_write_barrier(obj_opr, state_for(x/*, x->state_before()*/), x->needs_null_check()); + monitor_enter(obj_opr, lock, syncTempOpr(), scratch, x->monitor_no(), info_for_exception, info); } @@ -750,27 +764,31 @@ LIR_Opr addr = new_pointer_register(); LIR_Address* a; + + LIR_Opr obj_op = obj.result(); + obj_op = shenandoah_write_barrier(obj_op, NULL, false); + if(offset.result()->is_constant()) { #ifdef _LP64 jlong c = offset.result()->as_jlong(); if ((jlong)((jint)c) == c) { - a = new LIR_Address(obj.result(), + a = new LIR_Address(obj_op, (jint)c, as_BasicType(type)); } else { LIR_Opr tmp = new_register(T_LONG); __ move(offset.result(), tmp); - a = new LIR_Address(obj.result(), + a = new LIR_Address(obj_op, tmp, as_BasicType(type)); } #else - a = new LIR_Address(obj.result(), + a = new LIR_Address(obj_op, offset.result()->as_jint(), as_BasicType(type)); #endif } else { - a = new LIR_Address(obj.result(), + a = new LIR_Address(obj_op, offset.result(), LIR_Address::times_1, 0, @@ -785,12 +803,17 @@ } LIR_Opr ill = LIR_OprFact::illegalOpr; // for convenience - if (type == objectType) - __ cas_obj(addr, cmp.result(), val.result(), ill, ill); + + LIR_Opr val_op = val.result(); + + if (type == objectType) { + val_op = shenandoah_read_barrier(val_op, NULL, true); + __ cas_obj(addr, cmp.result(), val_op, new_register(T_OBJECT), new_register(T_OBJECT)); + } else if (type == intType) - __ cas_int(addr, cmp.result(), val.result(), ill, ill); + __ cas_int(addr, cmp.result(), val_op, ill, ill); else if (type == longType) - __ cas_long(addr, cmp.result(), val.result(), ill, ill); + __ cas_long(addr, cmp.result(), val_op, ill, ill); else { ShouldNotReachHere(); } @@ -801,7 +824,7 @@ result, as_BasicType(type)); if (type == objectType) { // Write-barrier needed for Object fields. // Seems to be precise - post_barrier(addr, val.result()); + post_barrier(addr, val_op); } } @@ -893,14 +916,19 @@ LIRItem dst_pos(x->argument_at(3), this); LIRItem length(x->argument_at(4), this); + LIR_Opr dst_op = dst.result(); + dst_op = shenandoah_write_barrier(dst_op, info, x->arg_needs_null_check(2) /* (flags & LIR_OpArrayCopy::dst_null_check) != 0 */); + LIR_Opr src_op = src.result(); + src_op = shenandoah_read_barrier(src_op, info, x->arg_needs_null_check(0) /*(flags & LIR_OpArrayCopy::src_null_check) != 0 */); + // operands for arraycopy must use fixed registers, otherwise // LinearScan will fail allocation (because arraycopy always needs a // call) #ifndef _LP64 - src.load_item_force (FrameMap::rcx_oop_opr); + src_op = force_opr_to(src_op, FrameMap::rcx_oop_opr); src_pos.load_item_force (FrameMap::rdx_opr); - dst.load_item_force (FrameMap::rax_oop_opr); + dst_op = force_opr_to(dst_op, FrameMap::rax_oop_opr); dst_pos.load_item_force (FrameMap::rbx_opr); length.load_item_force (FrameMap::rdi_opr); LIR_Opr tmp = (FrameMap::rsi_opr); @@ -914,9 +942,9 @@ // of the C convention we can process the java args trivially into C // args without worry of overwriting during the xfer - src.load_item_force (FrameMap::as_oop_opr(j_rarg0)); + src_op = force_opr_to(src_op, FrameMap::as_oop_opr(j_rarg0)); src_pos.load_item_force (FrameMap::as_opr(j_rarg1)); - dst.load_item_force (FrameMap::as_oop_opr(j_rarg2)); + dst_op = force_opr_to(dst_op, FrameMap::as_oop_opr(j_rarg2)); dst_pos.load_item_force (FrameMap::as_opr(j_rarg3)); length.load_item_force (FrameMap::as_opr(j_rarg4)); @@ -929,7 +957,7 @@ ciArrayKlass* expected_type; arraycopy_helper(x, &flags, &expected_type); - __ arraycopy(src.result(), src_pos.result(), dst.result(), dst_pos.result(), length.result(), tmp, expected_type, flags, info); // does add_safepoint + __ arraycopy(src_op, src_pos.result(), dst_op, dst_pos.result(), length.result(), tmp, expected_type, flags, info); // does add_safepoint } void LIRGenerator::do_update_CRC32(Intrinsic* x) { @@ -980,6 +1008,10 @@ } #endif + if (is_updateBytes) { + base_op = shenandoah_read_barrier(base_op, NULL, false); + } + LIR_Address* a = new LIR_Address(base_op, index, LIR_Address::times_1, @@ -1312,6 +1344,10 @@ LIR_Opr left = xin->result(); LIR_Opr right = yin->result(); + if (tag == objectTag && UseShenandoahGC && x->y()->type() != objectNull) { // Don't need to resolve for ifnull. + left = shenandoah_write_barrier(left, NULL, true); + right = shenandoah_read_barrier(right, NULL, true); + } __ cmp(lir_cond(cond), left, right); // Generate branch profiling. Profiling code doesn't kill flags. profile_branch(x, cond); @@ -1391,6 +1427,7 @@ void LIRGenerator::get_Object_unsafe(LIR_Opr dst, LIR_Opr src, LIR_Opr offset, BasicType type, bool is_volatile) { + src = shenandoah_read_barrier(src, NULL, false); if (is_volatile && type == T_LONG) { LIR_Address* addr = new LIR_Address(src, offset, T_DOUBLE); LIR_Opr tmp = new_register(T_DOUBLE); @@ -1408,6 +1445,7 @@ void LIRGenerator::put_Object_unsafe(LIR_Opr src, LIR_Opr offset, LIR_Opr data, BasicType type, bool is_volatile) { + src = shenandoah_write_barrier(src, NULL, false); if (is_volatile && type == T_LONG) { LIR_Address* addr = new LIR_Address(src, offset, T_DOUBLE); LIR_Opr tmp = new_register(T_DOUBLE); @@ -1423,6 +1461,7 @@ // Do the pre-write barrier, if any. pre_barrier(LIR_OprFact::address(addr), LIR_OprFact::illegalOpr /* pre_val */, true /* do_load */, false /* patch */, NULL); + data = shenandoah_read_barrier(data, NULL, true); __ move(data, addr); assert(src->is_register(), "must be register"); // Seems to be a precise address @@ -1449,22 +1488,29 @@ LIR_Opr offset = off.result(); assert (type == T_INT || (!x->is_add() && is_obj) LP64_ONLY( || type == T_LONG ), "unexpected type"); + + LIR_Opr src_op = src.result(); + src_op = shenandoah_write_barrier(src_op, NULL, false); + if (is_obj) { + data = shenandoah_read_barrier(data, NULL, true); + } + LIR_Address* addr; if (offset->is_constant()) { #ifdef _LP64 jlong c = offset->as_jlong(); if ((jlong)((jint)c) == c) { - addr = new LIR_Address(src.result(), (jint)c, type); + addr = new LIR_Address(src_op, (jint)c, type); } else { LIR_Opr tmp = new_register(T_LONG); __ move(offset, tmp); - addr = new LIR_Address(src.result(), tmp, type); + addr = new LIR_Address(src_op, tmp, type); } #else - addr = new LIR_Address(src.result(), offset->as_jint(), type); + addr = new LIR_Address(src_op, offset->as_jint(), type); #endif } else { - addr = new LIR_Address(src.result(), offset, type); + addr = new LIR_Address(src_op, offset, type); } // Because we want a 2-arg form of xchg and xadd
--- a/src/cpu/x86/vm/c1_MacroAssembler_x86.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/cpu/x86/vm/c1_MacroAssembler_x86.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -109,6 +109,7 @@ if (UseBiasedLocking) { // load object movptr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes())); + biased_locking_exit(obj, hdr, done); }
--- a/src/cpu/x86/vm/c1_Runtime1_x86.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/cpu/x86/vm/c1_Runtime1_x86.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -39,6 +39,9 @@ #include "utilities/macros.hpp" #include "vmreg_x86.inline.hpp" #if INCLUDE_ALL_GCS +#include "gc/shenandoah/shenandoahBarrierSet.hpp" +#include "gc/shenandoah/shenandoahHeap.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.hpp" #include "gc/g1/g1SATBCardTableModRefBS.hpp" #endif @@ -1657,13 +1660,24 @@ break; #if INCLUDE_ALL_GCS + case shenandoah_write_barrier_slow_id: + { + StubFrame f(sasm, "shenandoah_write_barrier", dont_gc_arguments); + + save_live_registers(sasm, 1); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_c1), r15_thread, rax); + restore_live_registers_except_rax(sasm); + __ verify_oop(rax); + + } + break; case g1_pre_barrier_slow_id: { StubFrame f(sasm, "g1_pre_barrier", dont_gc_arguments); // arg0 : previous value of memory BarrierSet* bs = Universe::heap()->barrier_set(); - if (bs->kind() != BarrierSet::G1SATBCTLogging) { + if (bs->kind() != BarrierSet::G1SATBCTLogging && bs->kind() != BarrierSet::ShenandoahBarrierSet) { __ movptr(rax, (int)id); __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), rax); __ should_not_reach_here(); @@ -1741,8 +1755,15 @@ // arg0: store_address Address store_addr(rbp, 2*BytesPerWord); + BarrierSet* bs = Universe::heap()->barrier_set(); + if (bs->kind() == BarrierSet::ShenandoahBarrierSet) { + __ movptr(rax, (int)id); + __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), rax); + __ should_not_reach_here(); + break; + } CardTableModRefBS* ct = - barrier_set_cast<CardTableModRefBS>(Universe::heap()->barrier_set()); + barrier_set_cast<CardTableModRefBS>(bs); assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code"); Label done;
--- a/src/cpu/x86/vm/interp_masm_x86.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/cpu/x86/vm/interp_masm_x86.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -1070,6 +1070,9 @@ // Load object pointer into obj_reg movptr(obj_reg, Address(lock_reg, obj_offset)); + // Need to preemptively evacuate obj because we CAS the mark word later. + oopDesc::bs()->interpreter_write_barrier(this, obj_reg); + if (UseBiasedLocking) { biased_locking_enter(lock_reg, obj_reg, swap_reg, tmp_reg, false, done, &slow_case); } @@ -1165,6 +1168,9 @@ // Load oop into obj_reg(%c_rarg3) movptr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes())); + // Need to preemptively evacuate obj because we CAS the mark word later. + oopDesc::bs()->interpreter_write_barrier(this, obj_reg); + // Free entry movptr(Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()), (int32_t)NULL_WORD);
--- a/src/cpu/x86/vm/jniFastGetField_x86_64.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/cpu/x86/vm/jniFastGetField_x86_64.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -81,6 +81,7 @@ // robj is data dependent on rcounter. } __ movptr(robj, Address(robj, 0)); // *obj + oopDesc::bs()->interpreter_read_barrier(masm, robj); __ mov (roffset, c_rarg2); __ shrptr(roffset, 2); // offset @@ -179,6 +180,7 @@ // robj is data dependent on rcounter. } __ movptr(robj, Address(robj, 0)); // *obj + oopDesc::bs()->interpreter_read_barrier(masm, robj); __ mov (roffset, c_rarg2); __ shrptr(roffset, 2); // offset
--- a/src/cpu/x86/vm/macroAssembler_x86.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/cpu/x86/vm/macroAssembler_x86.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -3729,6 +3729,11 @@ // provoke OS NULL exception if reg = NULL by // accessing M[reg] w/o changing any (non-CC) registers // NOTE: cmpl is plenty here to provoke a segv + + if (ShenandoahVerifyReadsToFromSpace) { + oopDesc::bs()->interpreter_read_barrier(this, reg); + } + cmpptr(rax, Address(reg, 0)); // Note: should probably use testl(rax, Address(reg, 0)); // may be shorter code (however, this version of @@ -4227,6 +4232,13 @@ assert(thread == r15_thread, "must be"); #endif // _LP64 + if (UseShenandoahGC) { + // No need for this in Shenandoah. + return; + } + + assert(UseG1GC, "expect G1 GC"); + Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() + PtrQueue::byte_offset_of_index())); Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() + @@ -4411,10 +4423,15 @@ NOT_LP64(get_thread(thread)); + uint oop_extra_words = Universe::heap()->oop_extra_words(); + movptr(obj, Address(thread, JavaThread::tlab_top_offset())); if (var_size_in_bytes == noreg) { - lea(end, Address(obj, con_size_in_bytes)); + lea(end, Address(obj, con_size_in_bytes + oop_extra_words * HeapWordSize)); } else { + if (oop_extra_words > 0) { + addq(var_size_in_bytes, oop_extra_words * HeapWordSize); + } lea(end, Address(obj, var_size_in_bytes, Address::times_1)); } cmpptr(end, Address(thread, JavaThread::tlab_end_offset())); @@ -4423,6 +4440,8 @@ // update the tlab top pointer movptr(Address(thread, JavaThread::tlab_top_offset()), end); + Universe::heap()->compile_prepare_oop(this, obj); + // recover var_size_in_bytes if necessary if (var_size_in_bytes == end) { subptr(var_size_in_bytes, obj); @@ -5690,6 +5709,9 @@ void MacroAssembler::load_klass(Register dst, Register src) { + if (ShenandoahVerifyReadsToFromSpace) { + oopDesc::bs()->interpreter_read_barrier(this, src); + } #ifdef _LP64 if (UseCompressedClassPointers) { movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
--- a/src/cpu/x86/vm/methodHandles_x86.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/cpu/x86/vm/methodHandles_x86.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -165,8 +165,10 @@ //NOT_PRODUCT({ FlagSetting fs(TraceMethodHandles, true); trace_method_handle(_masm, "LZMH"); }); // Load the invoker, as MH -> MH.form -> LF.vmentry + oopDesc::bs()->interpreter_read_barrier(_masm, recv); __ verify_oop(recv); __ load_heap_oop(method_temp, Address(recv, NONZERO(java_lang_invoke_MethodHandle::form_offset_in_bytes()))); + oopDesc::bs()->interpreter_read_barrier(_masm, method_temp); __ verify_oop(method_temp); __ load_heap_oop(method_temp, Address(method_temp, NONZERO(java_lang_invoke_LambdaForm::vmentry_offset_in_bytes()))); __ verify_oop(method_temp); @@ -180,8 +182,10 @@ Address(temp2, ConstMethod::size_of_parameters_offset()), sizeof(u2), /*is_signed*/ false); // assert(sizeof(u2) == sizeof(Method::_size_of_parameters), ""); + __ movptr(temp2, __ argument_address(temp2, -1)); + oopDesc::bs()->interpreter_read_barrier(_masm, temp2); Label L; - __ cmpptr(recv, __ argument_address(temp2, -1)); + __ cmpptr(recv, temp2); __ jcc(Assembler::equal, L); __ movptr(rax, __ argument_address(temp2, -1)); __ STOP("receiver not on stack"); @@ -376,6 +380,7 @@ // rsi/r13 - interpreter linkage (if interpreted) // rcx, rdx, rsi, rdi, r8 - compiler arguments (if compiled) + oopDesc::bs()->interpreter_read_barrier(_masm, member_reg); Label L_incompatible_class_change_error; switch (iid) { case vmIntrinsics::_linkToSpecial:
--- a/src/cpu/x86/vm/sharedRuntime_x86_64.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/cpu/x86/vm/sharedRuntime_x86_64.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -2446,7 +2446,7 @@ // Load the oop from the handle __ movptr(obj_reg, Address(oop_handle_reg, 0)); - + oopDesc::bs()->interpreter_read_barrier(masm, obj_reg); if (UseBiasedLocking) { __ biased_locking_enter(lock_reg, obj_reg, swap_reg, rscratch1, false, lock_done, &slow_path_lock); } @@ -2615,6 +2615,7 @@ // Get locked oop from the handle we passed to jni __ movptr(obj_reg, Address(oop_handle_reg, 0)); + oopDesc::bs()->interpreter_read_barrier(masm, obj_reg); Label done;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/cpu/x86/vm/shenandoahBarrierSet_x86.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,258 @@ +/* +Copyright 2015 Red Hat, Inc. and/or its affiliates. + */ + +#include "gc/shenandoah/shenandoahBarrierSet.hpp" + +#include "asm/macroAssembler.hpp" +#include "interpreter/interpreter.hpp" + +#define __ masm-> + +#ifndef CC_INTERP +void ShenandoahBarrierSet::compile_resolve_oop_runtime(MacroAssembler* masm, Register dst) { + + __ push(rscratch1); + + if (dst != rax) { + __ push(rax); + } + if (dst != rbx) { + __ push(rbx); + } + if (dst != rcx) { + __ push(rcx); + } + if (dst != rdx) { + __ push(rdx); + } + if (dst != rdi) { + __ push(rdi); + } + if (dst != rsi) { + __ push(rsi); + } + if (dst != rbp) { + __ push(rbp); + } + if (dst != r8) { + __ push(r8); + } + if (dst != r9) { + __ push(r9); + } + if (dst != r11) { + __ push(r11); + } + if (dst != r12) { + __ push(r12); + } + if (dst != r13) { + __ push(r13); + } + if (dst != r14) { + __ push(r14); + } + if (dst != r15) { + __ push(r15); + } + + __ subptr(rsp, 128); + __ movdbl(Address(rsp, 0), xmm0); + __ movdbl(Address(rsp, 8), xmm1); + __ movdbl(Address(rsp, 16), xmm2); + __ movdbl(Address(rsp, 24), xmm3); + __ movdbl(Address(rsp, 32), xmm4); + __ movdbl(Address(rsp, 40), xmm5); + __ movdbl(Address(rsp, 48), xmm6); + __ movdbl(Address(rsp, 56), xmm7); + __ movdbl(Address(rsp, 64), xmm8); + __ movdbl(Address(rsp, 72), xmm9); + __ movdbl(Address(rsp, 80), xmm10); + __ movdbl(Address(rsp, 88), xmm11); + __ movdbl(Address(rsp, 96), xmm12); + __ movdbl(Address(rsp, 104), xmm13); + __ movdbl(Address(rsp, 112), xmm14); + __ movdbl(Address(rsp, 120), xmm15); + + __ mov(c_rarg1, dst); + __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::resolve_oop_static), c_rarg1); + __ mov(rscratch1, rax); + + __ movdbl(xmm0, Address(rsp, 0)); + __ movdbl(xmm1, Address(rsp, 8)); + __ movdbl(xmm2, Address(rsp, 16)); + __ movdbl(xmm3, Address(rsp, 24)); + __ movdbl(xmm4, Address(rsp, 32)); + __ movdbl(xmm5, Address(rsp, 40)); + __ movdbl(xmm6, Address(rsp, 48)); + __ movdbl(xmm7, Address(rsp, 56)); + __ movdbl(xmm8, Address(rsp, 64)); + __ movdbl(xmm9, Address(rsp, 72)); + __ movdbl(xmm10, Address(rsp, 80)); + __ movdbl(xmm11, Address(rsp, 88)); + __ movdbl(xmm12, Address(rsp, 96)); + __ movdbl(xmm13, Address(rsp, 104)); + __ movdbl(xmm14, Address(rsp, 112)); + __ movdbl(xmm15, Address(rsp, 120)); + __ addptr(rsp, 128); + + if (dst != r15) { + __ pop(r15); + } + if (dst != r14) { + __ pop(r14); + } + if (dst != r13) { + __ pop(r13); + } + if (dst != r12) { + __ pop(r12); + } + if (dst != r11) { + __ pop(r11); + } + if (dst != r9) { + __ pop(r9); + } + if (dst != r8) { + __ pop(r8); + } + if (dst != rbp) { + __ pop(rbp); + } + if (dst != rsi) { + __ pop(rsi); + } + if (dst != rdi) { + __ pop(rdi); + } + if (dst != rdx) { + __ pop(rdx); + } + if (dst != rcx) { + __ pop(rcx); + } + if (dst != rbx) { + __ pop(rbx); + } + if (dst != rax) { + __ pop(rax); + } + + __ mov(dst, rscratch1); + + __ pop(rscratch1); +} + +// TODO: The following should really live in an X86 specific subclass. +void ShenandoahBarrierSet::interpreter_read_barrier(MacroAssembler* masm, Register dst) { + if (ShenandoahReadBarrier) { + + Label is_null; + __ testptr(dst, dst); + __ jcc(Assembler::zero, is_null); + interpreter_read_barrier_not_null(masm, dst); + __ bind(is_null); + } +} + +void ShenandoahBarrierSet::interpreter_read_barrier_not_null(MacroAssembler* masm, Register dst) { + if (ShenandoahReadBarrier) { + if (ShenandoahVerifyReadsToFromSpace) { + compile_resolve_oop_runtime(masm, dst); + return; + } + __ movptr(dst, Address(dst, -8)); + } +} + +void ShenandoahBarrierSet::interpreter_write_barrier(MacroAssembler* masm, Register dst) { + + if (! ShenandoahWriteBarrier) { + assert(! ShenandoahConcurrentEvacuation, "Can only do this without concurrent evacuation"); + return interpreter_read_barrier(masm, dst); + } + + assert(dst != rscratch1, "different regs"); + //assert(dst != rscratch2, "Need rscratch2"); + + Label done; + + Address evacuation_in_progress = Address(r15_thread, in_bytes(JavaThread::evacuation_in_progress_offset())); + + __ cmpb(evacuation_in_progress, 0); + + // Now check if evacuation is in progress. + interpreter_read_barrier_not_null(masm, dst); + + __ jcc(Assembler::equal, done); + __ push(rscratch1); + __ push(rscratch2); + + __ movptr(rscratch1, dst); + __ shrptr(rscratch1, ShenandoahHeapRegion::RegionSizeShift); + __ movptr(rscratch2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr()); + __ movbool(rscratch2, Address(rscratch2, rscratch1, Address::times_1)); + __ testb(rscratch2, 0x1); + + __ pop(rscratch2); + __ pop(rscratch1); + + __ jcc(Assembler::zero, done); + + __ push(rscratch1); + + // Save possibly live regs. + if (dst != rax) { + __ push(rax); + } + if (dst != rbx) { + __ push(rbx); + } + if (dst != rcx) { + __ push(rcx); + } + if (dst != rdx) { + __ push(rdx); + } + if (dst != c_rarg1) { + __ push(c_rarg1); + } + + __ subptr(rsp, 2 * wordSize); + __ movdbl(Address(rsp, 0), xmm0); + + // Call into runtime + __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_interp), dst); + __ mov(rscratch1, rax); + + // Restore possibly live regs. + __ movdbl(xmm0, Address(rsp, 0)); + __ addptr(rsp, 2 * Interpreter::stackElementSize); + + if (dst != c_rarg1) { + __ pop(c_rarg1); + } + if (dst != rdx) { + __ pop(rdx); + } + if (dst != rcx) { + __ pop(rcx); + } + if (dst != rbx) { + __ pop(rbx); + } + if (dst != rax) { + __ pop(rax); + } + + // Move result into dst reg. + __ mov(dst, rscratch1); + + __ pop(rscratch1); + + __ bind(done); +} + +#endif
--- a/src/cpu/x86/vm/stubGenerator_x86_64.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/cpu/x86/vm/stubGenerator_x86_64.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -804,6 +804,92 @@ return start; } + address generate_shenandoah_wb() { + StubCodeMark mark(this, "StubRoutines", "shenandoah_wb"); + address start = __ pc(); + + Label done; + + __ push(rbx); + // Check for object beeing in the collection set. + // TODO: Can we use only 1 register here? + __ movptr(rdi, rax); + __ shrptr(rdi, ShenandoahHeapRegion::RegionSizeShift); + __ movptr(rbx, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr()); + __ movbool(rbx, Address(rbx, rdi, Address::times_1)); + __ testbool(rbx); + __ jcc(Assembler::zero, done); + + __ push(rcx); + __ push(rdx); + __ push(rdi); + __ push(rsi); + __ push(r8); + __ push(r9); + __ push(r10); + __ push(r11); + __ push(r12); + __ push(r13); + __ push(r14); + __ push(r15); + __ subptr(rsp, 128); + __ movdbl(Address(rsp, 0), xmm0); + __ movdbl(Address(rsp, 8), xmm1); + __ movdbl(Address(rsp, 16), xmm2); + __ movdbl(Address(rsp, 24), xmm3); + __ movdbl(Address(rsp, 32), xmm4); + __ movdbl(Address(rsp, 40), xmm5); + __ movdbl(Address(rsp, 48), xmm6); + __ movdbl(Address(rsp, 56), xmm7); + __ movdbl(Address(rsp, 64), xmm8); + __ movdbl(Address(rsp, 72), xmm9); + __ movdbl(Address(rsp, 80), xmm10); + __ movdbl(Address(rsp, 88), xmm11); + __ movdbl(Address(rsp, 96), xmm12); + __ movdbl(Address(rsp, 104), xmm13); + __ movdbl(Address(rsp, 112), xmm14); + __ movdbl(Address(rsp, 120), xmm15); + __ movptr(rdi, rax); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_c2), rdi); + __ movdbl(xmm0, Address(rsp, 0)); + __ movdbl(xmm1, Address(rsp, 8)); + __ movdbl(xmm2, Address(rsp, 16)); + __ movdbl(xmm3, Address(rsp, 24)); + __ movdbl(xmm4, Address(rsp, 32)); + __ movdbl(xmm5, Address(rsp, 40)); + __ movdbl(xmm6, Address(rsp, 48)); + __ movdbl(xmm7, Address(rsp, 56)); + __ movdbl(xmm8, Address(rsp, 64)); + __ movdbl(xmm9, Address(rsp, 72)); + __ movdbl(xmm10, Address(rsp, 80)); + __ movdbl(xmm11, Address(rsp, 88)); + __ movdbl(xmm12, Address(rsp, 96)); + __ movdbl(xmm13, Address(rsp, 104)); + __ movdbl(xmm14, Address(rsp, 112)); + __ movdbl(xmm15, Address(rsp, 120)); + __ addptr(rsp, 128); + __ pop(r15); + __ pop(r14); + __ pop(r13); + __ pop(r12); + __ pop(r11); + __ pop(r10); + __ pop(r9); + __ pop(r8); + __ pop(rsi); + __ pop(rdi); + __ pop(rdx); + __ pop(rcx); + + __ bind(done); + + __ pop(rbx); + + __ ret(0); + + return start; + } + address generate_f2i_fixup() { StubCodeMark mark(this, "StubRoutines", "f2i_fixup"); Address inout(rsp, 5 * wordSize); // return address + 4 saves @@ -1234,6 +1320,7 @@ BarrierSet* bs = Universe::heap()->barrier_set(); switch (bs->kind()) { case BarrierSet::G1SATBCTLogging: + case BarrierSet::ShenandoahBarrierSet: // With G1, don't generate the call if we statically know that the target in uninitialized if (!dest_uninitialized) { __ pusha(); // push registers @@ -1278,6 +1365,7 @@ BarrierSet* bs = Universe::heap()->barrier_set(); switch (bs->kind()) { case BarrierSet::G1SATBCTLogging: + case BarrierSet::ShenandoahBarrierSet: { __ pusha(); // push registers (overkill) if (c_rarg0 == count) { // On win64 c_rarg0 == rcx @@ -4275,6 +4363,9 @@ throw_NullPointerException_at_call)); // entry points that are platform specific + if (UseShenandoahGC) { + StubRoutines::x86::_shenandoah_wb = generate_shenandoah_wb(); + } StubRoutines::x86::_f2i_fixup = generate_f2i_fixup(); StubRoutines::x86::_f2l_fixup = generate_f2l_fixup(); StubRoutines::x86::_d2i_fixup = generate_d2i_fixup();
--- a/src/cpu/x86/vm/stubRoutines_x86_64.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/cpu/x86/vm/stubRoutines_x86_64.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -34,6 +34,7 @@ address StubRoutines::x86::_get_previous_fp_entry = NULL; address StubRoutines::x86::_get_previous_sp_entry = NULL; +address StubRoutines::x86::_shenandoah_wb = NULL; address StubRoutines::x86::_f2i_fixup = NULL; address StubRoutines::x86::_f2l_fixup = NULL; address StubRoutines::x86::_d2i_fixup = NULL;
--- a/src/cpu/x86/vm/stubRoutines_x86_64.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/cpu/x86/vm/stubRoutines_x86_64.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -43,6 +43,7 @@ static address _get_previous_fp_entry; static address _get_previous_sp_entry; + static address _shenandoah_wb; static address _f2i_fixup; static address _f2l_fixup; static address _d2i_fixup; @@ -65,6 +66,11 @@ return _get_previous_sp_entry; } + static address shenandoah_wb() + { + return _shenandoah_wb; + } + static address f2i_fixup() { return _f2i_fixup;
--- a/src/cpu/x86/vm/templateInterpreter_x86_64.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/cpu/x86/vm/templateInterpreter_x86_64.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -545,6 +545,7 @@ #endif // ASSERT __ bind(done); + oopDesc::bs()->interpreter_read_barrier(_masm, rax); } // add space for monitor & lock @@ -637,7 +638,7 @@ const int referent_offset = java_lang_ref_Reference::referent_offset; guarantee(referent_offset > 0, "referent offset not initialized"); - if (UseG1GC) { + if (UseG1GC || UseShenandoahGC) { Label slow_path; // rbx: method @@ -648,6 +649,8 @@ __ testptr(rax, rax); __ jcc(Assembler::zero, slow_path); + oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rax); + // rax: local 0 // rbx: method (but can be used as scratch now) // rdx: scratch @@ -778,6 +781,7 @@ __ movl(crc, Address(rsp, 5*wordSize)); // Initial CRC } else { __ movptr(buf, Address(rsp, 3*wordSize)); // byte[] array + oopDesc::bs()->interpreter_read_barrier_not_null(_masm, buf); __ addptr(buf, arrayOopDesc::base_offset_in_bytes(T_BYTE)); // + header size __ movl2ptr(off, Address(rsp, 2*wordSize)); // offset __ addq(buf, off); // + offset
--- a/src/cpu/x86/vm/templateTable_x86.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/cpu/x86/vm/templateTable_x86.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -157,6 +157,7 @@ switch (barrier) { #if INCLUDE_ALL_GCS case BarrierSet::G1SATBCTLogging: + case BarrierSet::ShenandoahBarrierSet: { // flatten object address if needed // We do it regardless of precise because we need the registers @@ -189,6 +190,9 @@ new_val = rbx; __ movptr(new_val, val); } + // For Shenandoah, make sure we only store refs into to-space. + oopDesc::bs()->interpreter_read_barrier(_masm, val); + __ store_heap_oop(Address(rdx, 0), val); __ g1_write_barrier_post(rdx /* store_adr */, new_val /* new_val */, @@ -684,6 +688,11 @@ void TemplateTable::index_check_without_pop(Register array, Register index) { // destroys rbx // check array + + if (ShenandoahVerifyReadsToFromSpace) { + oopDesc::bs()->interpreter_read_barrier(_masm, array); + } + __ null_check(array, arrayOopDesc::length_offset_in_bytes()); // sign extend index for use by indexed load __ movl2ptr(index, index); @@ -704,6 +713,7 @@ // rax: index // rdx: array index_check(rdx, rax); // kills rbx + oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx); __ movl(rax, Address(rdx, rax, Address::times_4, arrayOopDesc::base_offset_in_bytes(T_INT))); @@ -716,6 +726,7 @@ index_check(rdx, rax); // kills rbx NOT_LP64(__ mov(rbx, rax)); // rbx,: index + oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx); __ movptr(rax, Address(rdx, rbx, Address::times_8, arrayOopDesc::base_offset_in_bytes(T_LONG) + 0 * wordSize)); NOT_LP64(__ movl(rdx, Address(rdx, rbx, Address::times_8, arrayOopDesc::base_offset_in_bytes(T_LONG) + 1 * wordSize))); } @@ -727,6 +738,7 @@ // rax: index // rdx: array index_check(rdx, rax); // kills rbx + oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx); __ load_float(Address(rdx, rax, Address::times_4, arrayOopDesc::base_offset_in_bytes(T_FLOAT))); @@ -737,6 +749,7 @@ // rax: index // rdx: array index_check(rdx, rax); // kills rbx + oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx); __ load_double(Address(rdx, rax, Address::times_8, arrayOopDesc::base_offset_in_bytes(T_DOUBLE))); @@ -747,6 +760,7 @@ // rax: index // rdx: array index_check(rdx, rax); // kills rbx + oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx); __ load_heap_oop(rax, Address(rdx, rax, UseCompressedOops ? Address::times_4 : Address::times_ptr, arrayOopDesc::base_offset_in_bytes(T_OBJECT))); @@ -757,6 +771,7 @@ // rax: index // rdx: array index_check(rdx, rax); // kills rbx + oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx); __ load_signed_byte(rax, Address(rdx, rax, Address::times_1, arrayOopDesc::base_offset_in_bytes(T_BYTE))); } @@ -765,6 +780,7 @@ // rax: index // rdx: array index_check(rdx, rax); // kills rbx + oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx); __ load_unsigned_short(rax, Address(rdx, rax, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR))); } @@ -778,6 +794,7 @@ // rax: index // rdx: array index_check(rdx, rax); // kills rbx + oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx); __ load_unsigned_short(rax, Address(rdx, rax, Address::times_2, @@ -790,6 +807,7 @@ // rax: index // rdx: array index_check(rdx, rax); // kills rbx + oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx); __ load_signed_short(rax, Address(rdx, rax, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_SHORT))); } @@ -984,6 +1002,7 @@ // rbx: index // rdx: array index_check(rdx, rbx); // prefer index in rbx + oopDesc::bs()->interpreter_write_barrier(_masm, rdx); __ movl(Address(rdx, rbx, Address::times_4, arrayOopDesc::base_offset_in_bytes(T_INT)), @@ -998,6 +1017,7 @@ // rdx: high(value) index_check(rcx, rbx); // prefer index in rbx, // rbx,: index + oopDesc::bs()->interpreter_write_barrier(_masm, rcx); __ movptr(Address(rcx, rbx, Address::times_8, arrayOopDesc::base_offset_in_bytes(T_LONG) + 0 * wordSize), rax); NOT_LP64(__ movl(Address(rcx, rbx, Address::times_8, arrayOopDesc::base_offset_in_bytes(T_LONG) + 1 * wordSize), rdx)); } @@ -1010,6 +1030,7 @@ // rbx: index // rdx: array index_check(rdx, rbx); // prefer index in rbx + oopDesc::bs()->interpreter_write_barrier(_masm, rdx); __ store_float(Address(rdx, rbx, Address::times_4, arrayOopDesc::base_offset_in_bytes(T_FLOAT))); } @@ -1020,6 +1041,7 @@ // rbx: index // rdx: array index_check(rdx, rbx); // prefer index in rbx + oopDesc::bs()->interpreter_write_barrier(_masm, rdx); __ store_double(Address(rdx, rbx, Address::times_8, arrayOopDesc::base_offset_in_bytes(T_DOUBLE))); } @@ -1036,6 +1058,7 @@ arrayOopDesc::base_offset_in_bytes(T_OBJECT)); index_check_without_pop(rdx, rcx); // kills rbx + oopDesc::bs()->interpreter_write_barrier(_masm, rdx); __ testptr(rax, rax); __ jcc(Assembler::zero, is_null); @@ -1084,6 +1107,7 @@ // rbx: index // rdx: array index_check(rdx, rbx); // prefer index in rbx + oopDesc::bs()->interpreter_write_barrier(_masm, rdx); __ movb(Address(rdx, rbx, Address::times_1, arrayOopDesc::base_offset_in_bytes(T_BYTE)), @@ -1097,6 +1121,7 @@ // rbx: index // rdx: array index_check(rdx, rbx); // prefer index in rbx + oopDesc::bs()->interpreter_write_barrier(_masm, rdx); __ movw(Address(rdx, rbx, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)), @@ -2311,6 +2336,17 @@ // assume branch is more often taken than not (loops use backward branches) Label not_taken; __ pop_ptr(rdx); + if (UseShenandoahGC) { + // For Shenandoah, if the objects are not equal, we try again after + // resolving both objects through a read barrier, to make sure we're + // not comparing from-space and to-space copies of the same object. + Label eq; + __ cmpptr(rdx, rax); + __ jcc(Assembler::equal, eq); + oopDesc::bs()->interpreter_read_barrier(_masm, rax); + oopDesc::bs()->interpreter_read_barrier(_masm, rdx); + __ bind(eq); + } __ cmpptr(rdx, rax); __ jcc(j_not(cc), not_taken); branch(false, false); @@ -2751,6 +2787,7 @@ load_field_cp_cache_entry(obj, cache, index, off, flags, is_static); if (!is_static) pop_and_check_object(obj); + oopDesc::bs()->interpreter_read_barrier_not_null(_masm, obj); const Address field(obj, off, Address::times_1, 0*wordSize); NOT_LP64(const Address hi(obj, off, Address::times_1, 1*wordSize)); @@ -3019,7 +3056,10 @@ // btos { __ pop(btos); - if (!is_static) pop_and_check_object(obj); + if (!is_static) { + pop_and_check_object(obj); + } + oopDesc::bs()->interpreter_write_barrier(_masm, obj); __ movb(field, rax); if (!is_static && rc == may_rewrite) { patch_bytecode(Bytecodes::_fast_bputfield, bc, rbx, true, byte_no); @@ -3034,7 +3074,10 @@ // atos { __ pop(atos); - if (!is_static) pop_and_check_object(obj); + if (!is_static) { + pop_and_check_object(obj); + } + oopDesc::bs()->interpreter_write_barrier(_masm, obj); // Store into the field do_oop_store(_masm, field, rax, _bs->kind(), false); if (!is_static && rc == may_rewrite) { @@ -3050,7 +3093,10 @@ // itos { __ pop(itos); - if (!is_static) pop_and_check_object(obj); + if (!is_static) { + pop_and_check_object(obj); + } + oopDesc::bs()->interpreter_write_barrier(_masm, obj); __ movl(field, rax); if (!is_static && rc == may_rewrite) { patch_bytecode(Bytecodes::_fast_iputfield, bc, rbx, true, byte_no); @@ -3065,7 +3111,10 @@ // ctos { __ pop(ctos); - if (!is_static) pop_and_check_object(obj); + if (!is_static) { + pop_and_check_object(obj); + } + oopDesc::bs()->interpreter_write_barrier(_masm, obj); __ movw(field, rax); if (!is_static && rc == may_rewrite) { patch_bytecode(Bytecodes::_fast_cputfield, bc, rbx, true, byte_no); @@ -3080,7 +3129,10 @@ // stos { __ pop(stos); - if (!is_static) pop_and_check_object(obj); + if (!is_static) { + pop_and_check_object(obj); + } + oopDesc::bs()->interpreter_write_barrier(_masm, obj); __ movw(field, rax); if (!is_static && rc == may_rewrite) { patch_bytecode(Bytecodes::_fast_sputfield, bc, rbx, true, byte_no); @@ -3096,7 +3148,10 @@ #ifdef _LP64 { __ pop(ltos); - if (!is_static) pop_and_check_object(obj); + if (!is_static) { + pop_and_check_object(obj); + } + oopDesc::bs()->interpreter_write_barrier(_masm, obj); __ movq(field, rax); if (!is_static && rc == may_rewrite) { patch_bytecode(Bytecodes::_fast_lputfield, bc, rbx, true, byte_no); @@ -3143,6 +3198,7 @@ { __ pop(ftos); if (!is_static) pop_and_check_object(obj); + oopDesc::bs()->interpreter_write_barrier(_masm, obj); __ store_float(field); if (!is_static && rc == may_rewrite) { patch_bytecode(Bytecodes::_fast_fputfield, bc, rbx, true, byte_no); @@ -3160,6 +3216,7 @@ { __ pop(dtos); if (!is_static) pop_and_check_object(obj); + oopDesc::bs()->interpreter_write_barrier(_masm, obj); __ store_double(field); if (!is_static && rc == may_rewrite) { patch_bytecode(Bytecodes::_fast_dputfield, bc, rbx, true, byte_no); @@ -3286,9 +3343,11 @@ // access field switch (bytecode()) { case Bytecodes::_fast_aputfield: + oopDesc::bs()->interpreter_write_barrier(_masm, rcx); do_oop_store(_masm, field, rax, _bs->kind(), false); break; case Bytecodes::_fast_lputfield: + oopDesc::bs()->interpreter_write_barrier(_masm, rcx); #ifdef _LP64 __ movq(field, rax); #else @@ -3296,20 +3355,25 @@ #endif break; case Bytecodes::_fast_iputfield: + oopDesc::bs()->interpreter_write_barrier(_masm, rcx); __ movl(field, rax); break; case Bytecodes::_fast_bputfield: + oopDesc::bs()->interpreter_write_barrier(_masm, rcx); __ movb(field, rax); break; case Bytecodes::_fast_sputfield: // fall through case Bytecodes::_fast_cputfield: + oopDesc::bs()->interpreter_write_barrier(_masm, rcx); __ movw(field, rax); break; case Bytecodes::_fast_fputfield: + oopDesc::bs()->interpreter_write_barrier(_masm, rcx); __ store_float(field); break; case Bytecodes::_fast_dputfield: + oopDesc::bs()->interpreter_write_barrier(_masm, rcx); __ store_double(field); break; default: @@ -3367,6 +3431,7 @@ // rax: object __ verify_oop(rax); __ null_check(rax); + oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rax); Address field(rax, rbx, Address::times_1); // access field @@ -3428,6 +3493,7 @@ // next instruction) __ increment(rbcp); __ null_check(rax); + oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rax); const Address field = Address(rax, rbx, Address::times_1, 0*wordSize); switch (state) { case itos: @@ -3830,11 +3896,17 @@ #endif // _LP64 if (UseTLAB) { + uint oop_extra_words = Universe::heap()->oop_extra_words(); + if (oop_extra_words > 0) { + __ addq(rdx, oop_extra_words * HeapWordSize); + } + __ movptr(rax, Address(thread, in_bytes(JavaThread::tlab_top_offset()))); __ lea(rbx, Address(rax, rdx, Address::times_1)); __ cmpptr(rbx, Address(thread, in_bytes(JavaThread::tlab_end_offset()))); __ jcc(Assembler::above, allow_shared_alloc ? allocate_shared : slow_case); __ movptr(Address(thread, in_bytes(JavaThread::tlab_top_offset())), rbx); + Universe::heap()->compile_prepare_oop(_masm); if (ZeroTLAB) { // the fields have been already cleared __ jmp(initialize_header); @@ -3975,6 +4047,9 @@ void TemplateTable::arraylength() { transition(atos, itos); + if (ShenandoahVerifyReadsToFromSpace) { + oopDesc::bs()->interpreter_read_barrier(_masm, rax); + } __ null_check(rax, arrayOopDesc::length_offset_in_bytes()); __ movl(rax, Address(rax, arrayOopDesc::length_offset_in_bytes())); } @@ -4075,7 +4150,11 @@ __ pop_ptr(rdx); // restore receiver __ verify_oop(rdx); __ load_klass(rdx, rdx); - __ jmpb(resolved); + if (ShenandoahVerifyReadsToFromSpace) { + __ jmp(resolved); + } else { + __ jmpb(resolved); + } // Get superklass in rax and subklass in rdx __ bind(quicked); @@ -4171,6 +4250,11 @@ // check for NULL object __ null_check(rax); + // We need to preemptively evacuate the object, because we later compare + // it to objects in the BasicObjectLock list, and we might get false negatives + // if another thread evacuates the object in the meantime. See acmp. + oopDesc::bs()->interpreter_write_barrier(_masm, rax); + const Address monitor_block_top( rbp, frame::interpreter_frame_monitor_block_top_offset * wordSize); const Address monitor_block_bot( @@ -4193,7 +4277,11 @@ // starting with top-most entry __ lea(rbot, monitor_block_bot); // points to word before bottom // of monitor block - __ jmpb(entry); + if (UseShenandoahGC && ShenandoahVerifyReadsToFromSpace) { + __ jmp(entry); + } else { + __ jmpb(entry); + } __ bind(loop); // check if current entry is used @@ -4201,7 +4289,9 @@ // if not used then remember entry in rmon __ cmovptr(Assembler::equal, rmon, rtop); // cmov => cmovptr // check if current entry is for same object - __ cmpptr(rax, Address(rtop, BasicObjectLock::obj_offset_in_bytes())); + __ movptr(rscratch1, Address(rtop, BasicObjectLock::obj_offset_in_bytes())); + oopDesc::bs()->interpreter_read_barrier(_masm, rscratch1); + __ cmpptr(rax, rscratch1); // if same object then stop searching __ jccb(Assembler::equal, exit); // otherwise advance to next entry @@ -4268,6 +4358,11 @@ // check for NULL object __ null_check(rax); + // We need to preemptively evacuate the object, because we later compare + // it to objects in the BasicObjectLock list, and we might get false negatives + // if another thread evacuates the object in the meantime. See acmp. + oopDesc::bs()->interpreter_write_barrier(_masm, rax); + const Address monitor_block_top( rbp, frame::interpreter_frame_monitor_block_top_offset * wordSize); const Address monitor_block_bot( @@ -4286,11 +4381,17 @@ // starting with top-most entry __ lea(rbot, monitor_block_bot); // points to word before bottom // of monitor block - __ jmpb(entry); + if (UseShenandoahGC && ShenandoahVerifyReadsToFromSpace) { + __ jmp(entry); + } else { + __ jmpb(entry); + } __ bind(loop); // check if current entry is for same object - __ cmpptr(rax, Address(rtop, BasicObjectLock::obj_offset_in_bytes())); + __ movptr(rscratch1, Address(rtop, BasicObjectLock::obj_offset_in_bytes())); + oopDesc::bs()->interpreter_read_barrier(_masm, rscratch1); + __ cmpptr(rax, rscratch1); // if same object then stop searching __ jcc(Assembler::equal, found); // otherwise advance to next entry
--- a/src/cpu/x86/vm/vtableStubs_x86_64.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/cpu/x86/vm/vtableStubs_x86_64.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -222,11 +222,13 @@ int VtableStub::pd_code_size_limit(bool is_vtable_stub) { if (is_vtable_stub) { // Vtable stub size - return (DebugVtables ? 512 : 24) + (CountCompiledCalls ? 13 : 0) + + return (ShenandoahVerifyReadsToFromSpace ? 512 : 0) + + (DebugVtables ? 512 : 24) + (CountCompiledCalls ? 13 : 0) + (UseCompressedClassPointers ? MacroAssembler::instr_size_for_decode_klass_not_null() : 0); } else { // Itable stub size - return (DebugVtables ? 512 : 74) + (CountCompiledCalls ? 13 : 0) + + return (ShenandoahVerifyReadsToFromSpace ? 512 : 0) + + (DebugVtables ? 512 : 74) + (CountCompiledCalls ? 13 : 0) + (UseCompressedClassPointers ? MacroAssembler::instr_size_for_decode_klass_not_null() : 0); } // In order to tune these parameters, run the JVM with VM options
--- a/src/cpu/x86/vm/x86_64.ad Fri Sep 25 22:59:24 2015 -0700 +++ b/src/cpu/x86/vm/x86_64.ad Tue Sep 29 18:47:52 2015 +0200 @@ -6403,6 +6403,41 @@ ins_pipe(ialu_reg_reg); // XXX %} +instruct shenandoahRB(rRegP dst, rRegP src, rFlagsReg cr) %{ + match(Set dst (ShenandoahReadBarrier src)); + effect(DEF dst, USE src); + ins_cost(125); // XXX + format %{ "shenandoah_rb $dst,$src" %} + ins_encode %{ + Register s = $src$$Register; + Register d = $dst$$Register; + __ movptr(d, Address(s, -8)); + %} + ins_pipe(ialu_reg_mem); +%} + +instruct shenandoahWB(rax_RegP dst, rdi_RegP src, rFlagsReg cr) %{ + match(Set dst (ShenandoahWriteBarrier src)); + effect(DEF dst, USE_KILL src, KILL cr); + ins_cost(300); // XXX + format %{ "shenandoah_wb $dst,$src" %} + ins_encode %{ + Label done; + Register s = $src$$Register; + Register d = $dst$$Register; + assert(s == rdi, "need rdi"); + assert(d == rax, "result in rax"); + Address evacuation_in_progress = Address(r15_thread, in_bytes(JavaThread::evacuation_in_progress_offset())); + __ movptr(d, Address(s, -8)); + __ cmpb(evacuation_in_progress, 0); + __ movptr(d, Address(s, -8)); + __ jcc(Assembler::equal, done); + __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::shenandoah_wb()))); + __ bind(done); + %} + ins_pipe(pipe_slow); +%} + // Convert oop pointer into compressed form instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{ predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
--- a/src/os_cpu/linux_x86/vm/os_linux_x86.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/os_cpu/linux_x86/vm/os_linux_x86.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -435,6 +435,25 @@ } } + if ((sig == SIGSEGV) && UseShenandoahGC + && (ShenandoahVerifyWritesToFromSpace || ShenandoahVerifyReadsToFromSpace)) { + if (Universe::heap()->is_in(info->si_addr)) { + tty->print("OK, we got the SIGSEGV,on address %p now what?\n", (address) info->si_addr); + ucontext_t* uc = (ucontext_t*) ucVoid; + address pc = (address) os::Linux::ucontext_get_pc(uc); + os::print_context(tty, ucVoid); + Universe::heap()->print(); + if (ShenandoahVerifyReadsToFromSpace) { + assert(false, "Illegal read to From Space"); + } else { + assert(false, "Illegal write to From Space"); + } + } + } + + + + #ifndef AMD64 // Execution protection violation //
--- a/src/share/vm/adlc/formssel.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/adlc/formssel.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -781,7 +781,9 @@ !strcmp(_matrule->_rChild->_opType,"CreateEx") || // type of exception !strcmp(_matrule->_rChild->_opType,"CheckCastPP") || !strcmp(_matrule->_rChild->_opType,"GetAndSetP") || - !strcmp(_matrule->_rChild->_opType,"GetAndSetN")) ) return true; + !strcmp(_matrule->_rChild->_opType,"GetAndSetN") || + !strcmp(_matrule->_rChild->_opType,"ShenandoahReadBarrier") || + !strcmp(_matrule->_rChild->_opType,"ShenandoahWriteBarrier")) ) return true; else if ( is_ideal_load() == Form::idealP ) return true; else if ( is_ideal_store() != Form::none ) return true; @@ -3487,6 +3489,7 @@ "ClearArray", "GetAndAddI", "GetAndSetI", "GetAndSetP", "GetAndAddL", "GetAndSetL", "GetAndSetN", + "ShenandoahReadBarrier", "ShenandoahWriteBarrier", }; int cnt = sizeof(needs_ideal_memory_list)/sizeof(char*); if( strcmp(_opType,"PrefetchAllocation")==0 )
--- a/src/share/vm/adlc/main.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/adlc/main.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -237,6 +237,7 @@ AD.addInclude(AD._HPP_file, "opto/regalloc.hpp"); AD.addInclude(AD._HPP_file, "opto/subnode.hpp"); AD.addInclude(AD._HPP_file, "opto/vectornode.hpp"); + AD.addInclude(AD._HPP_file, "gc/shenandoah/shenandoahBarrierSet.hpp"); AD.addInclude(AD._CPP_CLONE_file, "precompiled.hpp"); AD.addInclude(AD._CPP_CLONE_file, "adfiles", get_basename(AD._HPP_file._name)); AD.addInclude(AD._CPP_EXPAND_file, "precompiled.hpp");
--- a/src/share/vm/asm/assembler.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/asm/assembler.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -312,5 +312,5 @@ } } #endif - return offset < 0 || os::vm_page_size() <= offset; + return (offset < 0 && ((!UseShenandoahGC) || offset != -8)) || os::vm_page_size() <= offset; }
--- a/src/share/vm/c1/c1_LIR.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/c1/c1_LIR.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -1010,6 +1010,15 @@ do_temp(opProfileType->_tmp); break; } + case lir_shenandoah_wb: { + assert(op->as_OpShenandoahWriteBarrier() != NULL, "must be"); + LIR_OpShenandoahWriteBarrier* opShenandoahWB = (LIR_OpShenandoahWriteBarrier*) op; + do_input(opShenandoahWB->_opr); + do_output(opShenandoahWB->_result); + do_temp(opShenandoahWB->_tmp1); + do_temp(opShenandoahWB->_tmp2); + break; + } default: ShouldNotReachHere(); } @@ -1108,6 +1117,10 @@ } } +void LIR_OpShenandoahWriteBarrier::emit_code(LIR_Assembler* masm) { + masm->emit_opShenandoahWriteBarrier(this); +} + void LIR_OpConvert::emit_code(LIR_Assembler* masm) { masm->emit_opConvert(this); if (stub() != NULL) { @@ -1816,6 +1829,7 @@ case lir_profile_call: s = "profile_call"; break; // LIR_OpProfileType case lir_profile_type: s = "profile_type"; break; + case lir_shenandoah_wb: s = "shenandoah_wb"; break; // LIR_OpAssert #ifdef ASSERT case lir_assert: s = "assert"; break; @@ -1826,6 +1840,13 @@ return s; } +void LIR_OpShenandoahWriteBarrier::print_instr(outputStream* out) const { + out->print("[obj: "); in_opr()->print(out); out->print("]"); + out->print("[res: "); result_opr()->print(out); out->print("]"); + out->print("[tmp1: "); tmp1_opr()->print(out); out->print("]"); + out->print("[tmp2: "); tmp2_opr()->print(out); out->print("]"); +} + // LIR_OpJavaCall void LIR_OpJavaCall::print_instr(outputStream* out) const { out->print("call: ");
--- a/src/share/vm/c1/c1_LIR.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/c1/c1_LIR.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -876,6 +876,7 @@ class LIR_OpConvert; class LIR_OpAllocObj; class LIR_OpRoundFP; +class LIR_OpShenandoahWriteBarrier; class LIR_Op2; class LIR_OpDelay; class LIR_Op3; @@ -940,6 +941,7 @@ , lir_pack64 , lir_unpack64 , lir_unwind + , lir_shenandoah_wb , end_op1 , begin_op2 , lir_cmp @@ -1154,6 +1156,7 @@ virtual LIR_OpCompareAndSwap* as_OpCompareAndSwap() { return NULL; } virtual LIR_OpProfileCall* as_OpProfileCall() { return NULL; } virtual LIR_OpProfileType* as_OpProfileType() { return NULL; } + virtual LIR_OpShenandoahWriteBarrier* as_OpShenandoahWriteBarrier() { return NULL; } #ifdef ASSERT virtual LIR_OpAssert* as_OpAssert() { return NULL; } #endif @@ -1468,6 +1471,25 @@ virtual void print_instr(outputStream* out) const PRODUCT_RETURN; }; +class LIR_OpShenandoahWriteBarrier : public LIR_Op1 { + friend class LIR_OpVisitState; + +private: + bool _need_null_check; + LIR_Opr _tmp1; + LIR_Opr _tmp2; + +public: + LIR_OpShenandoahWriteBarrier(LIR_Opr obj, LIR_Opr result, LIR_Opr tmp1, LIR_Opr tmp2, CodeEmitInfo* info, bool need_null_check) : LIR_Op1(lir_shenandoah_wb, obj, result, T_OBJECT, lir_patch_none, info), _tmp1(tmp1), _tmp2(tmp2), _need_null_check(need_null_check) { + } + LIR_Opr tmp1_opr() const { return _tmp1; } + LIR_Opr tmp2_opr() const { return _tmp2; } + bool need_null_check() const { return _need_null_check; } + virtual void emit_code(LIR_Assembler* masm); + virtual LIR_OpShenandoahWriteBarrier* as_OpShenandoahWriteBarrier() { return this; } + virtual void print_instr(outputStream* out) const PRODUCT_RETURN; + +}; class ConversionStub; @@ -2149,6 +2171,8 @@ #endif void convert(Bytecodes::Code code, LIR_Opr left, LIR_Opr dst, ConversionStub* stub = NULL/*, bool is_32bit = false*/) { append(new LIR_OpConvert(code, left, dst, stub)); } + void shenandoah_wb(LIR_Opr obj, LIR_Opr result, LIR_Opr tmp1, LIR_Opr tmp2, CodeEmitInfo* info, bool need_null_check) { append(new LIR_OpShenandoahWriteBarrier(obj, result, tmp1, tmp2, info, need_null_check)); } + void logical_and (LIR_Opr left, LIR_Opr right, LIR_Opr dst) { append(new LIR_Op2(lir_logic_and, left, right, dst)); } void logical_or (LIR_Opr left, LIR_Opr right, LIR_Opr dst) { append(new LIR_Op2(lir_logic_or, left, right, dst)); } void logical_xor (LIR_Opr left, LIR_Opr right, LIR_Opr dst) { append(new LIR_Op2(lir_logic_xor, left, right, dst)); }
--- a/src/share/vm/c1/c1_LIRAssembler.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/c1/c1_LIRAssembler.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -196,6 +196,7 @@ void emit_opLabel(LIR_OpLabel* op); void emit_arraycopy(LIR_OpArrayCopy* op); void emit_updatecrc32(LIR_OpUpdateCRC32* op); + void emit_opShenandoahWriteBarrier(LIR_OpShenandoahWriteBarrier* op); void emit_opConvert(LIR_OpConvert* op); void emit_alloc_obj(LIR_OpAllocObj* op); void emit_alloc_array(LIR_OpAllocArray* op);
--- a/src/share/vm/c1/c1_LIRGenerator.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/c1/c1_LIRGenerator.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -41,6 +41,8 @@ #include "utilities/bitMap.inline.hpp" #include "utilities/macros.hpp" #if INCLUDE_ALL_GCS +#include "gc/shenandoah/shenandoahBarrierSet.hpp" +#include "gc/shenandoah/shenandoahHeap.hpp" #include "gc/g1/heapRegion.hpp" #endif // INCLUDE_ALL_GCS @@ -247,14 +249,22 @@ void LIRItem::load_item_force(LIR_Opr reg) { LIR_Opr r = result(); if (r != reg) { + _result = _gen->force_opr_to(r, reg); + } +} + +LIR_Opr LIRGenerator::force_opr_to(LIR_Opr op, LIR_Opr reg) { + if (op != reg) { #if !defined(ARM) && !defined(E500V2) - if (r->type() != reg->type()) { + if (op->type() != reg->type()) { // moves between different types need an intervening spill slot - r = _gen->force_to_spill(r, reg->type()); + op = force_to_spill(op, reg->type()); } #endif - __ move(r, reg); - _result = reg; + __ move(op, reg); + return reg; + } else { + return op; } } @@ -1422,8 +1432,10 @@ switch (_bs->kind()) { #if INCLUDE_ALL_GCS case BarrierSet::G1SATBCTLogging: + case BarrierSet::ShenandoahBarrierSet: G1SATBCardTableModRef_pre_barrier(addr_opr, pre_val, do_load, patch, info); break; + #endif // INCLUDE_ALL_GCS case BarrierSet::CardTableForRS: case BarrierSet::CardTableExtension: @@ -1444,6 +1456,8 @@ case BarrierSet::G1SATBCTLogging: G1SATBCardTableModRef_post_barrier(addr, new_val); break; + case BarrierSet::ShenandoahBarrierSet: + break; #endif // INCLUDE_ALL_GCS case BarrierSet::CardTableForRS: case BarrierSet::CardTableExtension: @@ -1714,11 +1728,24 @@ } #endif + LIR_Opr obj = object.result(); + if (x->needs_null_check() && (needs_patching || MacroAssembler::needs_explicit_null_check(x->offset()))) { // emit an explicit null check because the offset is too large - __ null_check(object.result(), new CodeEmitInfo(info)); + __ null_check(obj, new CodeEmitInfo(info)); + } + + obj = shenandoah_write_barrier(obj, info, x->needs_null_check()); + LIR_Opr val = value.result(); + if (is_oop) { + if (! val->is_register()) { + LIR_Opr tmp = new_register(T_OBJECT); + __ move(val, tmp); + val = tmp; + } + val = shenandoah_read_barrier(val, NULL, true); } LIR_Address* address; @@ -1727,9 +1754,9 @@ // generate_address to try to be smart about emitting the -1. // Otherwise the patching code won't know how to find the // instruction to patch. - address = new LIR_Address(object.result(), PATCHED_ADDR, field_type); + address = new LIR_Address(obj, PATCHED_ADDR, field_type); } else { - address = generate_address(object.result(), x->offset(), field_type); + address = generate_address(obj, x->offset(), field_type); } if (is_volatile && os::is_MP()) { @@ -1747,15 +1774,15 @@ bool needs_atomic_access = is_volatile || AlwaysAtomicAccesses; if (needs_atomic_access && !needs_patching) { - volatile_field_store(value.result(), address, info); + volatile_field_store(val, address, info); } else { LIR_PatchCode patch_code = needs_patching ? lir_patch_normal : lir_patch_none; - __ store(value.result(), address, info, patch_code); + __ store(val, address, info, patch_code); } if (is_oop) { // Store to object so mark the card of the header - post_barrier(object.result(), value.result()); + post_barrier(obj, val); } if (is_volatile && os::is_MP()) { @@ -1792,13 +1819,12 @@ x->is_static() ? "static" : "field", x->printable_bci()); } #endif - + LIR_Opr obj = object.result(); bool stress_deopt = StressLoopInvariantCodeMotion && info && info->deoptimize_on_exception(); if (x->needs_null_check() && (needs_patching || MacroAssembler::needs_explicit_null_check(x->offset()) || stress_deopt)) { - LIR_Opr obj = object.result(); if (stress_deopt) { obj = new_register(T_OBJECT); __ move(LIR_OprFact::oopConst(NULL), obj); @@ -1807,6 +1833,7 @@ __ null_check(obj, new CodeEmitInfo(info)); } + obj = shenandoah_read_barrier(obj, info, x->needs_null_check() && x->explicit_null_check() != NULL); LIR_Opr reg = rlock_result(x, field_type); LIR_Address* address; if (needs_patching) { @@ -1814,9 +1841,9 @@ // generate_address to try to be smart about emitting the -1. // Otherwise the patching code won't know how to find the // instruction to patch. - address = new LIR_Address(object.result(), PATCHED_ADDR, field_type); + address = new LIR_Address(obj, PATCHED_ADDR, field_type); } else { - address = generate_address(object.result(), x->offset(), field_type); + address = generate_address(obj, x->offset(), field_type); } bool needs_atomic_access = is_volatile || AlwaysAtomicAccesses; @@ -1832,6 +1859,39 @@ } } +LIR_Opr LIRGenerator::shenandoah_read_barrier(LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) { + if (UseShenandoahGC) { + + LabelObj* done = new LabelObj(); + LIR_Opr result = new_register(T_OBJECT); + __ move(obj, result); + if (need_null_check) { + __ cmp(lir_cond_equal, result, LIR_OprFact::oopConst(NULL)); + __ branch(lir_cond_equal, T_LONG, done->label()); + } + LIR_Address* brooks_ptr_address = generate_address(result, -8, T_ADDRESS); + __ load(brooks_ptr_address, result, info ? new CodeEmitInfo(info) : NULL, lir_patch_none); + + __ branch_destination(done->label()); + return result; + } else { + return obj; + } +} + +LIR_Opr LIRGenerator::shenandoah_write_barrier(LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) { + if (UseShenandoahGC) { + + LIR_Opr result = new_register(T_OBJECT); + LIR_Opr tmp1 = new_register(T_INT); + LIR_Opr tmp2 = new_register(T_INT); + __ shenandoah_wb(obj, result, tmp1, tmp2, info ? new CodeEmitInfo(info) : NULL, need_null_check); + return result; + + } else { + return obj; + } +} //------------------------java.nio.Buffer.checkIndex------------------------ @@ -1928,8 +1988,11 @@ } } + LIR_Opr ary = array.result(); + ary = shenandoah_read_barrier(ary, null_check_info, null_check_info != NULL); + // emit array address setup early so it schedules better - LIR_Address* array_addr = emit_array_address(array.result(), index.result(), x->elt_type(), false); + LIR_Address* array_addr = emit_array_address(ary, index.result(), x->elt_type(), false); if (GenerateRangeChecks && needs_range_check) { if (StressLoopInvariantCodeMotion && range_check_info->deoptimize_on_exception()) { @@ -1940,7 +2003,7 @@ __ cmp(lir_cond_belowEqual, length.result(), index.result()); __ branch(lir_cond_belowEqual, T_INT, new RangeCheckStub(range_check_info, index.result())); } else { - array_range_check(array.result(), index.result(), null_check_info, range_check_info); + array_range_check(ary, index.result(), null_check_info, range_check_info); // The range check performs the null check, so clear it out for the load null_check_info = NULL; } @@ -2253,7 +2316,7 @@ // } // } - if (UseG1GC && type == T_OBJECT) { + if ((UseShenandoahGC || UseG1GC) && type == T_OBJECT) { bool gen_pre_barrier = true; // Assume we need to generate pre_barrier. bool gen_offset_check = true; // Assume we need to generate the offset guard. bool gen_source_check = true; // Assume we need to check the src object for null. @@ -2795,6 +2858,7 @@ __ load_stack_address_monitor(0, lock); CodeEmitInfo* info = new CodeEmitInfo(scope()->start()->state()->copy(ValueStack::StateBefore, SynchronizationEntryBCI), NULL, x->check_flag(Instruction::DeoptimizeOnException)); + obj = shenandoah_write_barrier(obj, info, false); CodeStub* slow_path = new MonitorEnterStub(obj, lock, info); // receiver is guaranteed non-NULL so don't need CodeEmitInfo @@ -3015,9 +3079,9 @@ // Code for : x->x() {x->cond()} x->y() ? x->tval() : x->fval() void LIRGenerator::do_IfOp(IfOp* x) { + ValueTag xtag = x->x()->type()->tag(); #ifdef ASSERT { - ValueTag xtag = x->x()->type()->tag(); ValueTag ttag = x->tval()->type()->tag(); assert(xtag == intTag || xtag == objectTag, "cannot handle others"); assert(ttag == addressTag || ttag == intTag || ttag == objectTag || ttag == longTag, "cannot handle others"); @@ -3040,7 +3104,14 @@ f_val.dont_load_item(); LIR_Opr reg = rlock_result(x); - __ cmp(lir_cond(x->cond()), left.result(), right.result()); + LIR_Opr left_opr = left.result(); + LIR_Opr right_opr = right.result(); + if (xtag == objectTag && UseShenandoahGC && x->y()->type() != objectNull) { // Don't need to resolve for ifnull. + left_opr = shenandoah_write_barrier(left_opr, NULL, true); + right_opr = shenandoah_read_barrier(right_opr, NULL, true); + } + + __ cmp(lir_cond(x->cond()), left_opr, right_opr); __ cmove(lir_cond(x->cond()), t_val.result(), f_val.result(), reg, as_BasicType(x->x()->type())); }
--- a/src/share/vm/c1/c1_LIRGenerator.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/c1/c1_LIRGenerator.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -226,6 +226,8 @@ LIR_Opr round_item(LIR_Opr opr); LIR_Opr force_to_spill(LIR_Opr value, BasicType t); + LIR_Opr force_opr_to(LIR_Opr op, LIR_Opr reg); + PhiResolverState& resolver_state() { return _resolver_state; } void move_to_phi(PhiResolver* resolver, Value cur_val, Value sux_val); @@ -265,6 +267,9 @@ void pre_barrier(LIR_Opr addr_opr, LIR_Opr pre_val, bool do_load, bool patch, CodeEmitInfo* info); void post_barrier(LIR_OprDesc* addr, LIR_OprDesc* new_val); + LIR_Opr shenandoah_read_barrier(LIR_Opr obj, CodeEmitInfo* info, bool need_null_check); + LIR_Opr shenandoah_write_barrier(LIR_Opr obj, CodeEmitInfo* info, bool need_null_check); + // specific implementations // pre barriers
--- a/src/share/vm/c1/c1_Runtime1.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/c1/c1_Runtime1.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -60,6 +60,7 @@ #include "runtime/vm_version.hpp" #include "utilities/copy.hpp" #include "utilities/events.hpp" +#include "gc/shenandoah/shenandoahBarrierSet.hpp" // Implementation of StubAssembler @@ -204,6 +205,7 @@ switch (id) { // These stubs don't need to have an oopmap case dtrace_object_alloc_id: + case shenandoah_write_barrier_slow_id: case g1_pre_barrier_slow_id: case g1_post_barrier_slow_id: case slow_subtype_check_id: @@ -317,6 +319,7 @@ FUNCTION_CASE(entry, TRACE_TIME_METHOD); #endif FUNCTION_CASE(entry, StubRoutines::updateBytesCRC32()); + FUNCTION_CASE(entry, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_c1); #undef FUNCTION_CASE @@ -1406,6 +1409,10 @@ if ((unsigned int) arrayOop(dst)->length() < (unsigned int)dst_pos + (unsigned int)length) return ac_failed; if (length == 0) return ac_ok; + + oopDesc::bs()->resolve_oop(src); + oopDesc::bs()->resolve_and_maybe_copy_oop(dst); + if (src->is_typeArray()) { Klass* klass_oop = src->klass(); if (klass_oop != dst->klass()) return ac_failed;
--- a/src/share/vm/c1/c1_Runtime1.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/c1/c1_Runtime1.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -68,6 +68,7 @@ stub(load_klass_patching) \ stub(load_mirror_patching) \ stub(load_appendix_patching) \ + stub(shenandoah_write_barrier_slow) \ stub(g1_pre_barrier_slow) \ stub(g1_post_barrier_slow) \ stub(fpu2long_stub) \
--- a/src/share/vm/ci/ciInstanceKlass.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/ci/ciInstanceKlass.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -179,12 +179,12 @@ // ciInstanceKlass* ciInstanceKlass::get_canonical_holder(int offset) { #ifdef ASSERT - if (!(offset >= 0 && offset < layout_helper())) { + if (!(offset >= 0 && offset < layout_helper() || (offset == -8 && UseShenandoahGC))) { tty->print("*** get_canonical_holder(%d) on ", offset); this->print(); tty->print_cr(" ***"); }; - assert(offset >= 0 && offset < layout_helper(), "offset must be tame"); + assert(offset >= 0 && offset < layout_helper() || (offset == -8 && UseShenandoahGC), "offset must be tame"); #endif if (offset < instanceOopDesc::base_offset_in_bytes()) {
--- a/src/share/vm/ci/ciObject.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/ci/ciObject.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -52,9 +52,9 @@ ciObject::ciObject(oop o) { ASSERT_IN_VM; if (ciObjectFactory::is_initialized()) { - _handle = JNIHandles::make_local(o); + _handle = JNIHandles::make_local(oopDesc::bs()->resolve_and_maybe_copy_oop(o)); } else { - _handle = JNIHandles::make_global(o); + _handle = JNIHandles::make_global(oopDesc::bs()->resolve_and_maybe_copy_oop(o)); } _klass = NULL; init_flags_from(o);
--- a/src/share/vm/ci/ciObjectFactory.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/ci/ciObjectFactory.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -242,6 +242,10 @@ assert(Universe::heap()->is_in_reserved(key), "must be"); + // In Shenandoah we need to make sure that nobody forwards the key elsewhere + // under our hood. + key = oopDesc::bs()->resolve_and_maybe_copy_oop(key); + NonPermObject* &bucket = find_non_perm(key); if (bucket != NULL) { return bucket->object();
--- a/src/share/vm/ci/ciObjectFactory.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/ci/ciObjectFactory.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -78,7 +78,8 @@ void ensure_metadata_alive(ciMetadata* m); static bool is_equal(NonPermObject* p, oop key) { - return p->object()->get_oop() == key; + // Shenandoah: We already force-forwarded the key earlier. + return oopDesc::bs()->resolve_oop(p->object()->get_oop()) == key; } NonPermObject* &find_non_perm(oop key);
--- a/src/share/vm/classfile/defaultMethods.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/classfile/defaultMethods.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -390,6 +390,20 @@ Symbol* get_exception_message() { return _exception_message; } Symbol* get_exception_name() { return _exception_name; } + // Return true if the specified klass has a static method that matches + // the name and signature of the target method. + bool has_matching_static(InstanceKlass* root) { + if (_members.length() > 0) { + Pair<Method*,QualifiedState> entry = _members.at(0); + Method* impl = root->find_method(entry.first->name(), + entry.first->signature()); + if ((impl != NULL) && impl->is_static()) { + return true; + } + } + return false; + } + // Either sets the target or the exception error message void determine_target(InstanceKlass* root, TRAPS) { if (has_target() || throws_exception()) {
--- a/src/share/vm/classfile/dictionary.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/classfile/dictionary.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -141,7 +141,7 @@ // to be updated. bool _strongly_reachable; public: - oop protection_domain() { return literal(); } + oop protection_domain() { return oopDesc::bs()->resolve_oop(literal()); } void init() { _strongly_reachable = false; @@ -356,7 +356,7 @@ Method* method() const { return _method; } void set_method(Method* p) { _method = p; } - oop method_type() const { return _method_type; } + oop method_type() const { return oopDesc::bs()->resolve_oop(_method_type); } oop* method_type_addr() { return &_method_type; } void set_method_type(oop p) { _method_type = p; }
--- a/src/share/vm/classfile/javaClasses.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/classfile/javaClasses.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -443,6 +443,9 @@ assert(java_string->klass() == SystemDictionary::String_klass(), "must be java_string"); typeArrayOop value = java_lang_String::value(java_string); + if (ShenandoahVerifyReadsToFromSpace) { + value = (typeArrayOop) oopDesc::bs()->resolve_oop(value); + } int offset = java_lang_String::offset(java_string); int length = java_lang_String::length(java_string); if (length != len) { @@ -681,7 +684,7 @@ oop java_lang_Class::protection_domain(oop java_class) { assert(_protection_domain_offset != 0, "must be set"); - return java_class->obj_field(_protection_domain_offset); + return oopDesc::bs()->resolve_oop(java_class->obj_field(_protection_domain_offset)); } void java_lang_Class::set_protection_domain(oop java_class, oop pd) { assert(_protection_domain_offset != 0, "must be set"); @@ -699,7 +702,7 @@ oop java_lang_Class::init_lock(oop java_class) { assert(_init_lock_offset != 0, "must be set"); - return java_class->obj_field(_init_lock_offset); + return oopDesc::bs()->resolve_oop(java_class->obj_field(_init_lock_offset)); } void java_lang_Class::set_init_lock(oop java_class, oop init_lock) { assert(_init_lock_offset != 0, "must be set"); @@ -708,7 +711,7 @@ objArrayOop java_lang_Class::signers(oop java_class) { assert(_signers_offset != 0, "must be set"); - return (objArrayOop)java_class->obj_field(_signers_offset); + return (objArrayOop) oopDesc::bs()->resolve_oop(java_class->obj_field(_signers_offset)); } void java_lang_Class::set_signers(oop java_class, objArrayOop signers) { assert(_signers_offset != 0, "must be set"); @@ -863,9 +866,11 @@ // Note: create_basic_type_mirror above initializes ak to a non-null value. type = ArrayKlass::cast(ak)->element_type(); } else { - assert(java_class == Universe::void_mirror(), "only valid non-array primitive"); + assert(oopDesc::bs()->resolve_and_maybe_copy_oop(java_class) == oopDesc::bs()->resolve_and_maybe_copy_oop(Universe::void_mirror()), "only valid non-array primitive"); } - assert(Universe::java_mirror(type) == java_class, "must be consistent"); + + assert(oopDesc::bs()->resolve_and_maybe_copy_oop(Universe::java_mirror(type)) == oopDesc::bs()->resolve_and_maybe_copy_oop(java_class), "must be consistent"); + return type; } @@ -1111,7 +1116,7 @@ _park_blocker_offset != 0, "Must support parkBlocker field"); if (_park_blocker_offset > 0) { - return java_thread->obj_field(_park_blocker_offset); + return oopDesc::bs()->resolve_oop(java_thread->obj_field(_park_blocker_offset)); } return NULL; @@ -3107,6 +3112,9 @@ } oop java_lang_ClassLoader::parent(oop loader) { + if (ShenandoahVerifyReadsToFromSpace) { + loader = oopDesc::bs()->resolve_oop(loader); + } assert(is_instance(loader), "loader must be oop"); return loader->obj_field(parent_offset); }
--- a/src/share/vm/classfile/javaClasses.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/classfile/javaClasses.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -157,7 +157,11 @@ if (count_offset > 0) { return java_string->int_field(count_offset); } else { - return ((typeArrayOop)java_string->obj_field(value_offset))->length(); + oop value = java_string->obj_field(value_offset); + if (ShenandoahVerifyReadsToFromSpace) { + value = oopDesc::bs()->resolve_oop(value); + } + return ((typeArrayOop) value)->length(); } } static int utf8_length(oop java_string);
--- a/src/share/vm/classfile/symbolTable.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/classfile/symbolTable.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -51,6 +51,11 @@ CompactHashtable<Symbol*, char> SymbolTable::_shared_table; +Symbol* SymbolTable::resolve_symbol(Symbol* sym) { + // TODO: Casting Symbol* to oopDesc*/oop seems weird. + return (Symbol*) (oopDesc*) oopDesc::bs()->maybe_resolve_oop((oop) sym); +} + Symbol* SymbolTable::allocate_symbol(const u1* name, int len, bool c_heap, TRAPS) { assert (len <= Symbol::max_length(), "should be checked by caller"); @@ -111,7 +116,7 @@ if (entry->is_shared() && !use_alternate_hashcode()) { break; } - Symbol* s = entry->literal(); + Symbol* s = resolve_symbol(entry->literal()); (*memory_total) += s->size(); (*processed)++; assert(s != NULL, "just checking");
--- a/src/share/vm/classfile/symbolTable.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/classfile/symbolTable.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -94,6 +94,8 @@ // shared symbol table. static CompactHashtable<Symbol*, char> _shared_table; + static Symbol* resolve_symbol(Symbol* sym); + Symbol* allocate_symbol(const u1* name, int len, bool c_heap, TRAPS); // Assumes no characters larger than 0x7F // Adding elements
--- a/src/share/vm/classfile/systemDictionary.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/classfile/systemDictionary.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -103,7 +103,7 @@ // Java-level SystemLoader oop SystemDictionary::java_system_loader() { - return _java_system_loader; + return oopDesc::bs()->resolve_oop(_java_system_loader); } void SystemDictionary::compute_java_system_loader(TRAPS) { @@ -498,7 +498,7 @@ bool calledholdinglock = ObjectSynchronizer::current_thread_holds_lock((JavaThread*)THREAD, lockObject); assert(calledholdinglock,"must hold lock for notify"); - assert((!(lockObject() == _system_loader_lock_obj) && !is_parallelCapable(lockObject)), "unexpected double_lock_wait"); + assert((!(lockObject() == oopDesc::bs()->resolve_oop(_system_loader_lock_obj)) && !is_parallelCapable(lockObject)), "unexpected double_lock_wait"); ObjectSynchronizer::notifyall(lockObject, THREAD); intptr_t recursions = ObjectSynchronizer::complete_exit(lockObject, THREAD); SystemDictionary_lock->wait(); @@ -1531,7 +1531,7 @@ Handle SystemDictionary::compute_loader_lock_object(Handle class_loader, TRAPS) { // If class_loader is NULL we synchronize on _system_loader_lock_obj if (class_loader.is_null()) { - return Handle(THREAD, _system_loader_lock_obj); + return Handle(THREAD, oopDesc::bs()->resolve_oop(_system_loader_lock_obj)); } else { return class_loader; } @@ -1552,7 +1552,7 @@ == ObjectSynchronizer::owner_other) { // contention will likely happen, so increment the corresponding // contention counter. - if (loader_lock() == _system_loader_lock_obj) { + if (loader_lock() == oopDesc::bs()->resolve_oop(_system_loader_lock_obj)) { ClassLoader::sync_systemLoaderLockContentionRate()->inc(); } else { ClassLoader::sync_nonSystemLoaderLockContentionRate()->inc();
--- a/src/share/vm/code/codeBlob.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/code/codeBlob.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -28,6 +28,7 @@ #include "code/codeCacheExtensions.hpp" #include "code/relocInfo.hpp" #include "compiler/disassembler.hpp" +#include "gc/shared/barrierSet.hpp" #include "interpreter/bytecode.hpp" #include "memory/allocation.inline.hpp" #include "memory/heap.hpp"
--- a/src/share/vm/code/relocInfo.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/code/relocInfo.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -709,7 +709,7 @@ oop v = *oop_addr(); // clean inline caches store a special pseudo-null if (v == (oop)Universe::non_oop_word()) v = NULL; - return v; + return oopDesc::bs()->resolve_oop(v); }
--- a/src/share/vm/compiler/oopMap.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/compiler/oopMap.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -262,6 +262,9 @@ assert( i < len, "oopmap not found" ); OopMap* m = at(i); + if (m->offset() != pc_offset) { + tty->print_cr("oopmap not found, pc_offset: %d, m->offset(): %d", pc_offset, m->offset()); + } assert( m->offset() == pc_offset, "oopmap not found" ); return m; }
--- a/src/share/vm/gc/g1/concurrentMark.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/gc/g1/concurrentMark.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -58,81 +58,9 @@ #include "runtime/prefetch.inline.hpp" #include "services/memTracker.hpp" -// Concurrent marking bit map wrapper - -CMBitMapRO::CMBitMapRO(int shifter) : - _bm(), - _shifter(shifter) { - _bmStartWord = 0; - _bmWordSize = 0; -} - -HeapWord* CMBitMapRO::getNextMarkedWordAddress(const HeapWord* addr, - const HeapWord* limit) const { - // First we must round addr *up* to a possible object boundary. - addr = (HeapWord*)align_size_up((intptr_t)addr, - HeapWordSize << _shifter); - size_t addrOffset = heapWordToOffset(addr); - if (limit == NULL) { - limit = _bmStartWord + _bmWordSize; - } - size_t limitOffset = heapWordToOffset(limit); - size_t nextOffset = _bm.get_next_one_offset(addrOffset, limitOffset); - HeapWord* nextAddr = offsetToHeapWord(nextOffset); - assert(nextAddr >= addr, "get_next_one postcondition"); - assert(nextAddr == limit || isMarked(nextAddr), - "get_next_one postcondition"); - return nextAddr; -} - -HeapWord* CMBitMapRO::getNextUnmarkedWordAddress(const HeapWord* addr, - const HeapWord* limit) const { - size_t addrOffset = heapWordToOffset(addr); - if (limit == NULL) { - limit = _bmStartWord + _bmWordSize; - } - size_t limitOffset = heapWordToOffset(limit); - size_t nextOffset = _bm.get_next_zero_offset(addrOffset, limitOffset); - HeapWord* nextAddr = offsetToHeapWord(nextOffset); - assert(nextAddr >= addr, "get_next_one postcondition"); - assert(nextAddr == limit || !isMarked(nextAddr), - "get_next_one postcondition"); - return nextAddr; -} - -int CMBitMapRO::heapWordDiffToOffsetDiff(size_t diff) const { - assert((diff & ((1 << _shifter) - 1)) == 0, "argument check"); - return (int) (diff >> _shifter); -} - -#ifndef PRODUCT -bool CMBitMapRO::covers(MemRegion heap_rs) const { - // assert(_bm.map() == _virtual_space.low(), "map inconsistency"); - assert(((size_t)_bm.size() * ((size_t)1 << _shifter)) == _bmWordSize, - "size inconsistency"); - return _bmStartWord == (HeapWord*)(heap_rs.start()) && - _bmWordSize == heap_rs.word_size(); -} -#endif - -void CMBitMapRO::print_on_error(outputStream* st, const char* prefix) const { - _bm.print_on_error(st, prefix); -} - -size_t CMBitMap::compute_size(size_t heap_size) { - return ReservedSpace::allocation_align_size_up(heap_size / mark_distance()); -} - -size_t CMBitMap::mark_distance() { - return MinObjAlignmentInBytes * BitsPerByte; -} - -void CMBitMap::initialize(MemRegion heap, G1RegionToSpaceMapper* storage) { - _bmStartWord = heap.start(); - _bmWordSize = heap.word_size(); - - _bm.set_map((BitMap::bm_word_t*) storage->reserved().start()); - _bm.set_size(_bmWordSize >> _shifter); +void G1CMBitMap::initialize(MemRegion heap, G1RegionToSpaceMapper* storage) { + + CMBitMap::initialize(heap, storage->reserved()); storage->set_mapping_changed_listener(&_listener); } @@ -200,7 +128,7 @@ } }; -void CMBitMap::clearAll() { +void G1CMBitMap::clearAll() { G1CollectedHeap* g1h = G1CollectedHeap::heap(); ClearBitmapHRClosure cl(NULL, this, false /* may_yield */); uint n_workers = g1h->workers()->active_workers(); @@ -210,39 +138,6 @@ return; } -void CMBitMap::markRange(MemRegion mr) { - mr.intersection(MemRegion(_bmStartWord, _bmWordSize)); - assert(!mr.is_empty(), "unexpected empty region"); - assert((offsetToHeapWord(heapWordToOffset(mr.end())) == - ((HeapWord *) mr.end())), - "markRange memory region end is not card aligned"); - // convert address range into offset range - _bm.at_put_range(heapWordToOffset(mr.start()), - heapWordToOffset(mr.end()), true); -} - -void CMBitMap::clearRange(MemRegion mr) { - mr.intersection(MemRegion(_bmStartWord, _bmWordSize)); - assert(!mr.is_empty(), "unexpected empty region"); - // convert address range into offset range - _bm.at_put_range(heapWordToOffset(mr.start()), - heapWordToOffset(mr.end()), false); -} - -MemRegion CMBitMap::getAndClearMarkedRegion(HeapWord* addr, - HeapWord* end_addr) { - HeapWord* start = getNextMarkedWordAddress(addr); - start = MIN2(start, end_addr); - HeapWord* end = getNextUnmarkedWordAddress(start); - end = MIN2(end, end_addr); - assert(start <= end, "Consistency check"); - MemRegion mr(start, end); - if (!mr.is_empty()) { - clearRange(mr); - } - return mr; -} - CMMarkStack::CMMarkStack(ConcurrentMark* cm) : _base(NULL), _cm(cm) #ifdef ASSERT
--- a/src/share/vm/gc/g1/concurrentMark.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/gc/g1/concurrentMark.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -28,6 +28,7 @@ #include "classfile/javaClasses.hpp" #include "gc/g1/g1RegionToSpaceMapper.hpp" #include "gc/g1/heapRegionSet.hpp" +#include "gc/shared/cmBitMap.inline.hpp" #include "gc/shared/gcId.hpp" #include "gc/shared/taskqueue.hpp" @@ -52,74 +53,6 @@ bool do_object_b(oop obj); }; -// A generic CM bit map. This is essentially a wrapper around the BitMap -// class, with one bit per (1<<_shifter) HeapWords. - -class CMBitMapRO VALUE_OBJ_CLASS_SPEC { - protected: - HeapWord* _bmStartWord; // base address of range covered by map - size_t _bmWordSize; // map size (in #HeapWords covered) - const int _shifter; // map to char or bit - BitMap _bm; // the bit map itself - - public: - // constructor - CMBitMapRO(int shifter); - - enum { do_yield = true }; - - // inquiries - HeapWord* startWord() const { return _bmStartWord; } - size_t sizeInWords() const { return _bmWordSize; } - // the following is one past the last word in space - HeapWord* endWord() const { return _bmStartWord + _bmWordSize; } - - // read marks - - bool isMarked(HeapWord* addr) const { - assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize), - "outside underlying space?"); - return _bm.at(heapWordToOffset(addr)); - } - - // iteration - inline bool iterate(BitMapClosure* cl, MemRegion mr); - inline bool iterate(BitMapClosure* cl); - - // Return the address corresponding to the next marked bit at or after - // "addr", and before "limit", if "limit" is non-NULL. If there is no - // such bit, returns "limit" if that is non-NULL, or else "endWord()". - HeapWord* getNextMarkedWordAddress(const HeapWord* addr, - const HeapWord* limit = NULL) const; - // Return the address corresponding to the next unmarked bit at or after - // "addr", and before "limit", if "limit" is non-NULL. If there is no - // such bit, returns "limit" if that is non-NULL, or else "endWord()". - HeapWord* getNextUnmarkedWordAddress(const HeapWord* addr, - const HeapWord* limit = NULL) const; - - // conversion utilities - HeapWord* offsetToHeapWord(size_t offset) const { - return _bmStartWord + (offset << _shifter); - } - size_t heapWordToOffset(const HeapWord* addr) const { - return pointer_delta(addr, _bmStartWord) >> _shifter; - } - int heapWordDiffToOffsetDiff(size_t diff) const; - - // The argument addr should be the start address of a valid object - HeapWord* nextObject(HeapWord* addr) { - oop obj = (oop) addr; - HeapWord* res = addr + obj->size(); - assert(offsetToHeapWord(heapWordToOffset(res)) == res, "sanity"); - return res; - } - - void print_on_error(outputStream* st, const char* prefix) const; - - // debugging - NOT_PRODUCT(bool covers(MemRegion rs) const;) -}; - class CMBitMapMappingChangedListener : public G1MappingChangedListener { private: CMBitMap* _bm; @@ -131,11 +64,12 @@ virtual void on_commit(uint start_idx, size_t num_regions, bool zero_filled); }; -class CMBitMap : public CMBitMapRO { +class G1CMBitMap : public CMBitMap { private: CMBitMapMappingChangedListener _listener; public: + G1CMBitMap() : CMBitMap(), _listener() { _listener.set_bitmap(this); } static size_t compute_size(size_t heap_size); // Returns the amount of bytes on the heap between two marks in the bitmap. static size_t mark_distance(); @@ -145,29 +79,9 @@ return mark_distance(); } - CMBitMap() : CMBitMapRO(LogMinObjAlignment), _listener() { _listener.set_bitmap(this); } - // Initializes the underlying BitMap to cover the given area. void initialize(MemRegion heap, G1RegionToSpaceMapper* storage); - // Write marks. - inline void mark(HeapWord* addr); - inline void clear(HeapWord* addr); - inline bool parMark(HeapWord* addr); - inline bool parClear(HeapWord* addr); - - void markRange(MemRegion mr); - void clearRange(MemRegion mr); - - // Starting at the bit corresponding to "addr" (inclusive), find the next - // "1" bit, if any. This bit starts some run of consecutive "1"'s; find - // the end of this run (stopping at "end_addr"). Return the MemRegion - // covering from the start of the region corresponding to the first bit - // of the run to the end of the region corresponding to the last bit of - // the run. If there is no "1" bit at or after "addr", return an empty - // MemRegion. - MemRegion getAndClearMarkedRegion(HeapWord* addr, HeapWord* end_addr); - // Clear the whole mark bitmap. void clearAll(); }; @@ -378,8 +292,8 @@ FreeRegionList _cleanup_list; // Concurrent marking support structures - CMBitMap _markBitMap1; - CMBitMap _markBitMap2; + G1CMBitMap _markBitMap1; + G1CMBitMap _markBitMap2; CMBitMapRO* _prevMarkBitMap; // Completed mark bitmap CMBitMap* _nextMarkBitMap; // Under-construction mark bitmap
--- a/src/share/vm/gc/g1/concurrentMark.inline.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/gc/g1/concurrentMark.inline.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -166,63 +166,6 @@ return false; } -inline bool CMBitMapRO::iterate(BitMapClosure* cl, MemRegion mr) { - HeapWord* start_addr = MAX2(startWord(), mr.start()); - HeapWord* end_addr = MIN2(endWord(), mr.end()); - - if (end_addr > start_addr) { - // Right-open interval [start-offset, end-offset). - BitMap::idx_t start_offset = heapWordToOffset(start_addr); - BitMap::idx_t end_offset = heapWordToOffset(end_addr); - - start_offset = _bm.get_next_one_offset(start_offset, end_offset); - while (start_offset < end_offset) { - if (!cl->do_bit(start_offset)) { - return false; - } - HeapWord* next_addr = MIN2(nextObject(offsetToHeapWord(start_offset)), end_addr); - BitMap::idx_t next_offset = heapWordToOffset(next_addr); - start_offset = _bm.get_next_one_offset(next_offset, end_offset); - } - } - return true; -} - -inline bool CMBitMapRO::iterate(BitMapClosure* cl) { - MemRegion mr(startWord(), sizeInWords()); - return iterate(cl, mr); -} - -#define check_mark(addr) \ - assert(_bmStartWord <= (addr) && (addr) < (_bmStartWord + _bmWordSize), \ - "outside underlying space?"); \ - assert(G1CollectedHeap::heap()->is_in_exact(addr), \ - err_msg("Trying to access not available bitmap " PTR_FORMAT \ - " corresponding to " PTR_FORMAT " (%u)", \ - p2i(this), p2i(addr), G1CollectedHeap::heap()->addr_to_region(addr))); - -inline void CMBitMap::mark(HeapWord* addr) { - check_mark(addr); - _bm.set_bit(heapWordToOffset(addr)); -} - -inline void CMBitMap::clear(HeapWord* addr) { - check_mark(addr); - _bm.clear_bit(heapWordToOffset(addr)); -} - -inline bool CMBitMap::parMark(HeapWord* addr) { - check_mark(addr); - return _bm.par_set_bit(heapWordToOffset(addr)); -} - -inline bool CMBitMap::parClear(HeapWord* addr) { - check_mark(addr); - return _bm.par_clear_bit(heapWordToOffset(addr)); -} - -#undef check_mark - template<typename Fn> inline void CMMarkStack::iterate(Fn fn) { assert(_saved_index == _index,
--- a/src/share/vm/gc/g1/heapRegionBounds.inline.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/gc/g1/heapRegionBounds.inline.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -22,8 +22,8 @@ * */ -#ifndef SHARE_VM_GC_G1_HEAPREGIONBOUNDS_INLINE_HPP -#define SHARE_VM_GC_G1_HEAPREGIONBOUNDS_INLINE_HPP +#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONBOUNDS_INLINE_HPP +#define SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONBOUNDS_INLINE_HPP #include "gc/g1/heapRegionBounds.hpp" @@ -39,4 +39,4 @@ return TARGET_REGION_NUMBER; } -#endif // SHARE_VM_GC_G1_HEAPREGIONBOUNDS_INLINE_HPP +#endif
--- a/src/share/vm/gc/g1/satbQueue.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/gc/g1/satbQueue.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -74,30 +74,13 @@ // processing must be somewhat circumspect and not assume entries // in an unfiltered buffer refer to valid objects. -inline bool requires_marking(const void* entry, G1CollectedHeap* heap) { - // Includes rejection of NULL pointers. - assert(heap->is_in_reserved(entry), - err_msg("Non-heap pointer in SATB buffer: " PTR_FORMAT, p2i(entry))); - - HeapRegion* region = heap->heap_region_containing_raw(entry); - assert(region != NULL, err_msg("No region for " PTR_FORMAT, p2i(entry))); - if (entry >= region->next_top_at_mark_start()) { - return false; - } - - assert(((oop)entry)->is_oop(true /* ignore mark word */), - err_msg("Invalid oop in SATB buffer: " PTR_FORMAT, p2i(entry))); - - return true; -} - // This method removes entries from a SATB buffer that will not be // useful to the concurrent marking threads. Entries are retained if // they require marking and are not already marked. Retained entries // are compacted toward the top of the buffer. void ObjPtrQueue::filter() { - G1CollectedHeap* g1h = G1CollectedHeap::heap(); + CollectedHeap* heap = Universe::heap(); void** buf = _buf; size_t sz = _sz; @@ -124,7 +107,8 @@ // far, we'll just end up copying it to the same place. *p = NULL; - if (requires_marking(entry, g1h) && !g1h->isMarkedNext((oop)entry)) { + bool retain = heap->is_obj_ill(oop(entry)); + if (retain) { assert(new_index > 0, "we should not have already filled up the buffer"); new_index -= oopSize; assert(new_index >= i,
--- a/src/share/vm/gc/serial/genMarkSweep.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/gc/serial/genMarkSweep.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -173,7 +173,7 @@ void GenMarkSweep::deallocate_stacks() { - if (!UseG1GC) { + if (!UseG1GC && !UseShenandoahGC) { GenCollectedHeap* gch = GenCollectedHeap::heap(); gch->release_scratch(); }
--- a/src/share/vm/gc/serial/genMarkSweep.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/gc/serial/genMarkSweep.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -29,7 +29,9 @@ class GenMarkSweep : public MarkSweep { friend class VM_MarkSweep; + friend class ShenandoahMarkCompact; friend class G1MarkSweep; + public: static void invoke_at_safepoint(ReferenceProcessor* rp, bool clear_all_softrefs);
--- a/src/share/vm/gc/shared/barrierSet.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/gc/shared/barrierSet.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -27,11 +27,14 @@ #include "memory/memRegion.hpp" #include "oops/oopsHierarchy.hpp" +#include "asm/register.hpp" #include "utilities/fakeRttiSupport.hpp" // This class provides the interface between a barrier implementation and // the rest of the system. +class MacroAssembler; + class BarrierSet: public CHeapObj<mtGC> { friend class VMStructs; public: @@ -52,7 +55,8 @@ CardTableForRS, // CardTableModRefBSForCTRS CardTableExtension, // CardTableExtension G1SATBCT, // G1SATBCardTableModRefBS - G1SATBCTLogging // G1SATBCardTableLoggingModRefBS + G1SATBCTLogging, // G1SATBCardTableLoggingModRefBS + ShenandoahBarrierSet }; protected: @@ -213,6 +217,30 @@ // Print a description of the memory for the barrier set virtual void print_on(outputStream* st) const = 0; + + virtual oop resolve_oop(oop src) { + return src; + } + virtual oop maybe_resolve_oop(oop src) { + return src; + } + virtual oop resolve_and_maybe_copy_oop(oop src) { + return src; + } + +#ifndef CC_INTERP + virtual void interpreter_read_barrier(MacroAssembler* masm, Register dst) { + // Default implementation does nothing. + } + + virtual void interpreter_read_barrier_not_null(MacroAssembler* masm, Register dst) { + // Default implementation does nothing. + } + + virtual void interpreter_write_barrier(MacroAssembler* masm, Register dst) { + // Default implementation does nothing. + } +#endif }; template<typename T>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shared/cmBitMap.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,128 @@ + +// Concurrent marking bit map wrapper + +#include "gc/shared/cmBitMap.inline.hpp" +#include "utilities/bitMap.inline.hpp" + +CMBitMapRO::CMBitMapRO(int shifter) : + _bm(), + _shifter(shifter) { + _bmStartWord = 0; + _bmWordSize = 0; +} + +HeapWord* CMBitMapRO::getNextMarkedWordAddress(const HeapWord* addr, + const HeapWord* limit) const { + // First we must round addr *up* to a possible object boundary. + addr = (HeapWord*)align_size_up((intptr_t)addr, + HeapWordSize << _shifter); + size_t addrOffset = heapWordToOffset(addr); + if (limit == NULL) { + limit = _bmStartWord + _bmWordSize; + } + size_t limitOffset = heapWordToOffset(limit); + size_t nextOffset = _bm.get_next_one_offset(addrOffset, limitOffset); + HeapWord* nextAddr = offsetToHeapWord(nextOffset); + assert(nextAddr >= addr, "get_next_one postcondition"); + assert(nextAddr == limit || isMarked(nextAddr), + "get_next_one postcondition"); + return nextAddr; +} + +HeapWord* CMBitMapRO::getNextUnmarkedWordAddress(const HeapWord* addr, + const HeapWord* limit) const { + size_t addrOffset = heapWordToOffset(addr); + if (limit == NULL) { + limit = _bmStartWord + _bmWordSize; + } + size_t limitOffset = heapWordToOffset(limit); + size_t nextOffset = _bm.get_next_zero_offset(addrOffset, limitOffset); + HeapWord* nextAddr = offsetToHeapWord(nextOffset); + assert(nextAddr >= addr, "get_next_one postcondition"); + assert(nextAddr == limit || !isMarked(nextAddr), + "get_next_one postcondition"); + return nextAddr; +} + +int CMBitMapRO::heapWordDiffToOffsetDiff(size_t diff) const { + assert((diff & ((1 << _shifter) - 1)) == 0, "argument check"); + return (int) (diff >> _shifter); +} + +#ifndef PRODUCT +bool CMBitMapRO::covers(MemRegion heap_rs) const { + // assert(_bm.map() == _virtual_space.low(), "map inconsistency"); + assert(((size_t)_bm.size() * ((size_t)1 << _shifter)) == _bmWordSize, + "size inconsistency"); + return _bmStartWord == (HeapWord*)(heap_rs.start()) && + _bmWordSize == heap_rs.word_size(); +} +#endif + +void CMBitMapRO::print_on_error(outputStream* st, const char* prefix) const { + _bm.print_on_error(st, prefix); +} + +size_t CMBitMap::compute_size(size_t heap_size) { + return heap_size / mark_distance(); +} + +size_t CMBitMap::mark_distance() { + return MinObjAlignmentInBytes * BitsPerByte; +} + +void CMBitMap::initialize(MemRegion heap, MemRegion bitmap) { + _bmStartWord = heap.start(); + _bmWordSize = heap.word_size(); + + _bm.set_map((BitMap::bm_word_t*) bitmap.start()); + _bm.set_size(_bmWordSize >> _shifter); +} + +void CMBitMap::clearAll() { + _bm.clear(); +} + +void CMBitMap::markRange(MemRegion mr) { + mr.intersection(MemRegion(_bmStartWord, _bmWordSize)); + assert(!mr.is_empty(), "unexpected empty region"); + assert((offsetToHeapWord(heapWordToOffset(mr.end())) == + ((HeapWord *) mr.end())), + "markRange memory region end is not card aligned"); + // convert address range into offset range + _bm.at_put_range(heapWordToOffset(mr.start()), + heapWordToOffset(mr.end()), true); +} + +void CMBitMap::parMarkRange(MemRegion mr) { + mr.intersection(MemRegion(_bmStartWord, _bmWordSize)); + assert(!mr.is_empty(), "unexpected empty region"); + assert((offsetToHeapWord(heapWordToOffset(mr.end())) == + ((HeapWord *) mr.end())), + "markRange memory region end is not card aligned"); + // convert address range into offset range + _bm.par_at_put_range(heapWordToOffset(mr.start()), + heapWordToOffset(mr.end()), true); +} + +void CMBitMap::clearRange(MemRegion mr) { + mr.intersection(MemRegion(_bmStartWord, _bmWordSize)); + assert(!mr.is_empty(), "unexpected empty region"); + // convert address range into offset range + _bm.at_put_range(heapWordToOffset(mr.start()), + heapWordToOffset(mr.end()), false); +} + +MemRegion CMBitMap::getAndClearMarkedRegion(HeapWord* addr, + HeapWord* end_addr) { + HeapWord* start = getNextMarkedWordAddress(addr); + start = MIN2(start, end_addr); + HeapWord* end = getNextUnmarkedWordAddress(start); + end = MIN2(end, end_addr); + assert(start <= end, "Consistency check"); + MemRegion mr(start, end); + if (!mr.is_empty()) { + clearRange(mr); + } + return mr; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shared/cmBitMap.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,118 @@ + +#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_CMBITMAP_HPP +#define SHARE_VM_GC_IMPLEMENTATION_SHARED_CMBITMAP_HPP + +#include "memory/memRegion.hpp" +#include "oops/oop.inline.hpp" +#include "utilities/bitMap.hpp" +#include "utilities/globalDefinitions.hpp" + +// A generic CM bit map. This is essentially a wrapper around the BitMap +// class, with one bit per (1<<_shifter) HeapWords. + +class CMBitMapRO VALUE_OBJ_CLASS_SPEC { + protected: + HeapWord* _bmStartWord; // base address of range covered by map + size_t _bmWordSize; // map size (in #HeapWords covered) + const int _shifter; // map to char or bit + BitMap _bm; // the bit map itself + + public: + // constructor + CMBitMapRO(int shifter); + + enum { do_yield = true }; + + // inquiries + HeapWord* startWord() const { return _bmStartWord; } + size_t sizeInWords() const { return _bmWordSize; } + // the following is one past the last word in space + HeapWord* endWord() const { return _bmStartWord + _bmWordSize; } + + // read marks + + bool isMarked(HeapWord* addr) const { + assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize), + "outside underlying space?"); + return _bm.at(heapWordToOffset(addr)); + } + + // iteration + inline bool iterate(BitMapClosure* cl, MemRegion mr); + inline bool iterate(BitMapClosure* cl); + + // Return the address corresponding to the next marked bit at or after + // "addr", and before "limit", if "limit" is non-NULL. If there is no + // such bit, returns "limit" if that is non-NULL, or else "endWord()". + HeapWord* getNextMarkedWordAddress(const HeapWord* addr, + const HeapWord* limit = NULL) const; + // Return the address corresponding to the next unmarked bit at or after + // "addr", and before "limit", if "limit" is non-NULL. If there is no + // such bit, returns "limit" if that is non-NULL, or else "endWord()". + HeapWord* getNextUnmarkedWordAddress(const HeapWord* addr, + const HeapWord* limit = NULL) const; + + // conversion utilities + HeapWord* offsetToHeapWord(size_t offset) const { + return _bmStartWord + (offset << _shifter); + } + size_t heapWordToOffset(const HeapWord* addr) const { + return pointer_delta(addr, _bmStartWord) >> _shifter; + } + int heapWordDiffToOffsetDiff(size_t diff) const; + + // The argument addr should be the start address of a valid object + HeapWord* nextObject(HeapWord* addr) { + oop obj = (oop) addr; + HeapWord* res = addr + obj->size(); + assert(offsetToHeapWord(heapWordToOffset(res)) == res, "sanity"); + return res; + } + + void print_on_error(outputStream* st, const char* prefix) const; + + // debugging + NOT_PRODUCT(bool covers(MemRegion rs) const;) +}; + +class CMBitMap : public CMBitMapRO { + + public: + static size_t compute_size(size_t heap_size); + // Returns the amount of bytes on the heap between two marks in the bitmap. + static size_t mark_distance(); + // Returns how many bytes (or bits) of the heap a single byte (or bit) of the + // mark bitmap corresponds to. This is the same as the mark distance above. static size_t heap_map_factor() { + static size_t heap_map_factor() { + return mark_distance(); + } + + CMBitMap() : CMBitMapRO(LogMinObjAlignment) {} + + // Initializes the underlying BitMap to cover the given area. + void initialize(MemRegion heap, MemRegion bitmap); + + // Write marks. + inline void mark(HeapWord* addr); + inline void clear(HeapWord* addr); + inline bool parMark(HeapWord* addr); + inline bool parClear(HeapWord* addr); + + void markRange(MemRegion mr); + void parMarkRange(MemRegion mr); + void clearRange(MemRegion mr); + + // Starting at the bit corresponding to "addr" (inclusive), find the next + // "1" bit, if any. This bit starts some run of consecutive "1"'s; find + // the end of this run (stopping at "end_addr"). Return the MemRegion + // covering from the start of the region corresponding to the first bit + // of the run to the end of the region corresponding to the last bit of + // the run. If there is no "1" bit at or after "addr", return an empty + // MemRegion. + MemRegion getAndClearMarkedRegion(HeapWord* addr, HeapWord* end_addr); + + // Clear the whole mark bitmap. + virtual void clearAll(); +}; + +#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_CMBITMAP_HPP
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shared/cmBitMap.inline.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,65 @@ + +#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_CMBITMAP_INLINE_HPP +#define SHARE_VM_GC_IMPLEMENTATION_SHARED_CMBITMAP_INLINE_HPP + +#include "gc/shared/cmBitMap.hpp" +#include "utilities/bitMap.inline.hpp" + +inline bool CMBitMapRO::iterate(BitMapClosure* cl, MemRegion mr) { + HeapWord* start_addr = MAX2(startWord(), mr.start()); + HeapWord* end_addr = MIN2(endWord(), mr.end()); + + if (end_addr > start_addr) { + // Right-open interval [start-offset, end-offset). + BitMap::idx_t start_offset = heapWordToOffset(start_addr); + BitMap::idx_t end_offset = heapWordToOffset(end_addr); + + start_offset = _bm.get_next_one_offset(start_offset, end_offset); + while (start_offset < end_offset) { + if (!cl->do_bit(start_offset)) { + return false; + } + HeapWord* next_addr = MIN2(nextObject(offsetToHeapWord(start_offset)), end_addr); + BitMap::idx_t next_offset = heapWordToOffset(next_addr); + start_offset = _bm.get_next_one_offset(next_offset, end_offset); + } + } + return true; +} + +inline bool CMBitMapRO::iterate(BitMapClosure* cl) { + MemRegion mr(startWord(), sizeInWords()); + return iterate(cl, mr); +} + +#define check_mark(addr) \ + assert(_bmStartWord <= (addr) && (addr) < (_bmStartWord + _bmWordSize), \ + "outside underlying space?"); \ + /* assert(G1CollectedHeap::heap()->is_in_exact(addr), \ + err_msg("Trying to access not available bitmap "PTR_FORMAT \ + " corresponding to "PTR_FORMAT" (%u)", \ + p2i(this), p2i(addr), G1CollectedHeap::heap()->addr_to_region(addr))); */ + +inline void CMBitMap::mark(HeapWord* addr) { + check_mark(addr); + _bm.set_bit(heapWordToOffset(addr)); +} + +inline void CMBitMap::clear(HeapWord* addr) { + check_mark(addr); + _bm.clear_bit(heapWordToOffset(addr)); +} + +inline bool CMBitMap::parMark(HeapWord* addr) { + check_mark(addr); + return _bm.par_set_bit(heapWordToOffset(addr)); +} + +inline bool CMBitMap::parClear(HeapWord* addr) { + check_mark(addr); + return _bm.par_clear_bit(heapWordToOffset(addr)); +} + +#undef check_mark + +#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_CMBITMAP_INLINE_HPP
--- a/src/share/vm/gc/shared/collectedHeap.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/gc/shared/collectedHeap.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -452,7 +452,7 @@ #endif // ASSERT void -CollectedHeap::fill_with_array(HeapWord* start, size_t words, bool zap) +CollectedHeap::fill_with_array(HeapWord* start, size_t words, bool zap, bool gc_init) { assert(words >= filler_array_min_size(), "too small for an array"); assert(words <= filler_array_max_size(), "too big for a single object"); @@ -463,28 +463,28 @@ // Set the length first for concurrent GC. ((arrayOop)start)->set_length((int)len); - post_allocation_setup_common(Universe::intArrayKlassObj(), start); + post_allocation_setup_common(Universe::intArrayKlassObj(), start, gc_init); DEBUG_ONLY(zap_filler_array(start, words, zap);) } void -CollectedHeap::fill_with_object_impl(HeapWord* start, size_t words, bool zap) +CollectedHeap::fill_with_object_impl(HeapWord* start, size_t words, bool zap, bool gc_init) { assert(words <= filler_array_max_size(), "too big for a single object"); if (words >= filler_array_min_size()) { - fill_with_array(start, words, zap); + fill_with_array(start, words, zap, gc_init); } else if (words > 0) { assert(words == min_fill_size(), "unaligned size"); - post_allocation_setup_common(SystemDictionary::Object_klass(), start); + post_allocation_setup_common(SystemDictionary::Object_klass(), start, gc_init); } } -void CollectedHeap::fill_with_object(HeapWord* start, size_t words, bool zap) +void CollectedHeap::fill_with_object(HeapWord* start, size_t words, bool zap, bool gc_init) { DEBUG_ONLY(fill_args_check(start, words);) HandleMark hm; // Free handles before leaving. - fill_with_object_impl(start, words, zap); + fill_with_object_impl(start, words, zap, gc_init); } void CollectedHeap::fill_with_objects(HeapWord* start, size_t words, bool zap) @@ -631,3 +631,26 @@ err_msg("after_heap: " PTR_FORMAT " is unexpectedly in the heap", p2i(after_heap))); } #endif + +HeapWord* CollectedHeap::tlab_post_allocation_setup(HeapWord* obj, bool new_obj) { + return obj; +} + +uint CollectedHeap::oop_extra_words() { + // Default implementation doesn't need extra space for oops. + return 0; +} + +void CollectedHeap::shutdown() { + // Default implementation does nothing. +} + +void CollectedHeap::accumulate_statistics_all_gclabs() { + // Default implementation does nothing. +} + +#ifndef CC_INTERP +void CollectedHeap::compile_prepare_oop(MacroAssembler* masm, Register obj) { + // Default implementation does nothing. +} +#endif
--- a/src/share/vm/gc/shared/collectedHeap.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/gc/shared/collectedHeap.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -77,6 +77,7 @@ // CollectedHeap // GenCollectedHeap // G1CollectedHeap +// ShenandoahHeap // ParallelScavengeHeap // class CollectedHeap : public CHeapObj<mtInternal> { @@ -135,6 +136,7 @@ // Allocate from the current thread's TLAB, with broken-out slow path. inline static HeapWord* allocate_from_tlab(KlassHandle klass, Thread* thread, size_t size); + inline static HeapWord* allocate_from_tlab_work(KlassHandle klass, Thread* thread, size_t size); static HeapWord* allocate_from_tlab_slow(KlassHandle klass, Thread* thread, size_t size); // Allocate an uninitialized block of the given size, or returns NULL if @@ -146,7 +148,7 @@ inline static HeapWord* common_mem_allocate_init(KlassHandle klass, size_t size, TRAPS); // Helper functions for (VM) allocation. - inline static void post_allocation_setup_common(KlassHandle klass, HeapWord* obj); + inline static void post_allocation_setup_common(KlassHandle klass, HeapWord* obj, bool gc_init = true); inline static void post_allocation_setup_no_klass_install(KlassHandle klass, HeapWord* objPtr); @@ -167,10 +169,10 @@ // Fill with a single array; caller must ensure filler_array_min_size() <= // words <= filler_array_max_size(). - static inline void fill_with_array(HeapWord* start, size_t words, bool zap = true); + static void fill_with_array(HeapWord* start, size_t words, bool zap = true, bool gc_init = true); // Fill with a single object (either an int array or a java.lang.Object). - static inline void fill_with_object_impl(HeapWord* start, size_t words, bool zap = true); + static inline void fill_with_object_impl(HeapWord* start, size_t words, bool zap = true, bool gc_init = true); virtual void trace_heap(GCWhen::Type when, const GCTracer* tracer); @@ -185,7 +187,8 @@ enum Name { GenCollectedHeap, ParallelScavengeHeap, - G1CollectedHeap + G1CollectedHeap, + ShenandoahHeap }; static inline size_t filler_array_max_size() { @@ -194,6 +197,12 @@ virtual Name kind() const = 0; + // TLAB Post-allocation setup, specific to GC. + virtual HeapWord* tlab_post_allocation_setup(HeapWord* obj, bool new_obj = true); + + // Collector specific initialization. + virtual void post_allocation_collector_specific_setup(HeapWord* obj) { } + /** * Returns JNI error code JNI_ENOMEM if memory could not be allocated, * and JNI_OK on success. @@ -298,6 +307,12 @@ inline static void post_allocation_install_obj_klass(KlassHandle klass, oop obj); + virtual uint oop_extra_words(); + +#ifndef CC_INTERP + virtual void compile_prepare_oop(MacroAssembler* masm, Register obj = rax); +#endif + // Raw memory allocation facilities // The obj and array allocate methods are covers for these methods. // mem_allocate() should never be @@ -318,12 +333,12 @@ static void fill_with_objects(HeapWord* start, size_t words, bool zap = true); - static void fill_with_object(HeapWord* start, size_t words, bool zap = true); - static void fill_with_object(MemRegion region, bool zap = true) { - fill_with_object(region.start(), region.word_size(), zap); + static void fill_with_object(HeapWord* start, size_t words, bool zap = true, bool gc_init = true); + static void fill_with_object(MemRegion region, bool zap = true, bool gc_init = true) { + fill_with_object(region.start(), region.word_size(), zap, gc_init); } - static void fill_with_object(HeapWord* start, HeapWord* end, bool zap = true) { - fill_with_object(start, pointer_delta(end, start), zap); + static void fill_with_object(HeapWord* start, HeapWord* end, bool zap = true, bool gc_init = true) { + fill_with_object(start, pointer_delta(end, start), zap, gc_init); } // Return the address "addr" aligned by "alignment_in_bytes" if such @@ -570,6 +585,12 @@ // Heap verification virtual void verify(bool silent, VerifyOption option) = 0; + // Shut down all GC workers and other GC related threads. + virtual void shutdown(); + + // Accumulate additional statistics from GCLABs. + virtual void accumulate_statistics_all_gclabs(); + // Non product verification and debugging. #ifndef PRODUCT // Support for PromotionFailureALot. Return true if it's time to cause a @@ -603,6 +624,10 @@ return false; } + virtual bool is_obj_ill(const oop obj) const { + return true; + } + /////////////// Unit tests /////////////// NOT_PRODUCT(static void test_is_in();)
--- a/src/share/vm/gc/shared/collectedHeap.inline.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/gc/shared/collectedHeap.inline.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -39,9 +39,12 @@ // Inline allocation implementations. void CollectedHeap::post_allocation_setup_common(KlassHandle klass, - HeapWord* obj) { + HeapWord* obj, bool gc_init) { post_allocation_setup_no_klass_install(klass, obj); post_allocation_install_obj_klass(klass, oop(obj)); + if (gc_init) { + Universe::heap()->post_allocation_collector_specific_setup(obj); + } } void CollectedHeap::post_allocation_setup_no_klass_install(KlassHandle klass, @@ -179,7 +182,15 @@ HeapWord* CollectedHeap::allocate_from_tlab(KlassHandle klass, Thread* thread, size_t size) { assert(UseTLAB, "should use UseTLAB"); + size += Universe::heap()->oop_extra_words(); + HeapWord* obj = allocate_from_tlab_work(klass, thread, size); + if (obj != NULL) { + obj = Universe::heap()->tlab_post_allocation_setup(obj); + } + return obj; +} +HeapWord* CollectedHeap::allocate_from_tlab_work(KlassHandle klass, Thread* thread, size_t size) { HeapWord* obj = thread->tlab().allocate(size); if (obj != NULL) { return obj;
--- a/src/share/vm/gc/shared/collectorPolicy.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/gc/shared/collectorPolicy.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -51,6 +51,7 @@ #if INCLUDE_ALL_GCS class ConcurrentMarkSweepPolicy; class G1CollectorPolicy; +class ShenandoahCollectorPolicy; #endif // INCLUDE_ALL_GCS class GCPolicyCounters; @@ -130,6 +131,7 @@ #if INCLUDE_ALL_GCS virtual ConcurrentMarkSweepPolicy* as_concurrent_mark_sweep_policy() { return NULL; } virtual G1CollectorPolicy* as_g1_policy() { return NULL; } + virtual ShenandoahCollectorPolicy* as_pgc_policy() { return NULL; } #endif // INCLUDE_ALL_GCS // Note that these are not virtual. bool is_generation_policy() { return as_generation_policy() != NULL; } @@ -137,9 +139,11 @@ #if INCLUDE_ALL_GCS bool is_concurrent_mark_sweep_policy() { return as_concurrent_mark_sweep_policy() != NULL; } bool is_g1_policy() { return as_g1_policy() != NULL; } + bool is_pgc_policy() { return as_pgc_policy() != NULL; } #else // INCLUDE_ALL_GCS bool is_concurrent_mark_sweep_policy() { return false; } bool is_g1_policy() { return false; } + bool is_pgc_policy() { return false; } #endif // INCLUDE_ALL_GCS
--- a/src/share/vm/gc/shared/gcCause.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/gc/shared/gcCause.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -73,6 +73,8 @@ _g1_inc_collection_pause, _g1_humongous_allocation, + _shenandoah_init_mark, + _last_ditch_collection, _dcmd_gc_run,
--- a/src/share/vm/gc/shared/gcName.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/gc/shared/gcName.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -37,6 +37,7 @@ G1New, ConcurrentMarkSweep, G1Old, + Shenandoah, GCNameEndSentinel };
--- a/src/share/vm/gc/shared/gcTrace.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/gc/shared/gcTrace.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -206,6 +206,11 @@ void send_concurrent_mode_failure_event(); }; +class ShenandoahTracer : public GCTracer { +public: + ShenandoahTracer() : GCTracer(Shenandoah) {} +}; + class ParallelOldTracer : public OldGCTracer { ParallelOldGCInfo _parallel_old_gc_info;
--- a/src/share/vm/gc/shared/referenceProcessor.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/gc/shared/referenceProcessor.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -446,6 +446,8 @@ _next = discovered; _referent_addr = java_lang_ref_Reference::referent_addr(_ref); _referent = java_lang_ref_Reference::referent(_ref); + _referent = oopDesc::bs()->resolve_oop(_referent); + assert(Universe::heap()->is_in_reserved_or_null(_referent), "Wrong oop found in java.lang.Reference object"); assert(allow_null_referent ? @@ -643,11 +645,12 @@ void ReferenceProcessor::clear_discovered_references(DiscoveredList& refs_list) { + BarrierSet* bs = oopDesc::bs(); oop obj = NULL; - oop next = refs_list.head(); + oop next = bs->resolve_and_maybe_copy_oop(refs_list.head()); while (next != obj) { obj = next; - next = java_lang_ref_Reference::discovered(obj); + next = bs->resolve_and_maybe_copy_oop(java_lang_ref_Reference::discovered(obj)); java_lang_ref_Reference::set_discovered_raw(obj, NULL); } refs_list.set_head(NULL); @@ -962,8 +965,9 @@ oop obj, HeapWord* discovered_addr) { assert(_discovery_is_mt, "!_discovery_is_mt should have been handled by caller"); + // First we must make sure this object is only enqueued once. CAS in a non null - // discovered_addr. + // discovered_addr oop current_head = refs_list.head(); // The last ref must have its discovered field pointing to itself. oop next_discovered = (current_head != NULL) ? current_head : obj; @@ -974,9 +978,9 @@ // This thread just won the right to enqueue the object. // We have separate lists for enqueueing, so no synchronization // is necessary. + refs_list.set_head(obj); refs_list.inc_length(1); - if (TraceReferenceGC) { gclog_or_tty->print_cr("Discovered reference (mt) (" INTPTR_FORMAT ": %s)", p2i(obj), obj->klass()->internal_name()); @@ -1032,6 +1036,7 @@ // and complexity in processing these references. // We call this choice the "RefeferentBasedDiscovery" policy. bool ReferenceProcessor::discover_reference(oop obj, ReferenceType rt) { + // Make sure we are discovering refs (rather than processing discovered refs). if (!_discovering_refs || !RegisterReferences) { return false; @@ -1054,8 +1059,11 @@ // We only discover references whose referents are not (yet) // known to be strongly reachable. if (is_alive_non_header() != NULL) { + verify_referent(obj); + if (is_alive_non_header()->do_object_b(java_lang_ref_Reference::referent(obj))) { + return false; // referent is reachable } } @@ -1094,7 +1102,7 @@ // Check assumption that an object is not potentially // discovered twice except by concurrent collectors that potentially // trace the same Reference object twice. - assert(UseConcMarkSweepGC || UseG1GC, + assert(UseConcMarkSweepGC || UseG1GC || UseShenandoahGC, "Only possible with a concurrent marking collector"); return true; } @@ -1129,6 +1137,7 @@ // We do a raw store here: the field will be visited later when processing // the discovered references. oop current_head = list->head(); + // The last ref must have its discovered field pointing to itself. oop next_discovered = (current_head != NULL) ? current_head : obj;
--- a/src/share/vm/gc/shared/space.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/gc/shared/space.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -392,7 +392,7 @@ // store the forwarding pointer into the mark word if ((HeapWord*)q != compact_top) { - q->forward_to(oop(compact_top)); + q->forward_to(compact_oop(compact_top)); assert(q->is_gc_marked(), "encoding the pointer should preserve the mark"); } else { // if the object isn't moving we can just set the mark to the default @@ -631,8 +631,11 @@ // This version requires locking. inline HeapWord* ContiguousSpace::allocate_impl(size_t size) { + // Shenandoah is currently partitioning by region so this assertion + // is too strong. If we move to a smaller granularity we will + // need to revisit this. assert(Heap_lock->owned_by_self() || - (SafepointSynchronize::is_at_safepoint() && Thread::current()->is_VM_thread()), + (SafepointSynchronize::is_at_safepoint() && (Thread::current()->is_VM_thread() || UseShenandoahGC)), "not locked"); HeapWord* obj = top(); if (pointer_delta(end(), obj) >= size) {
--- a/src/share/vm/gc/shared/space.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/gc/shared/space.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -367,6 +367,10 @@ return oop(addr)->size(); } + inline oop make_oop(HeapWord* addr) const { + return oop(addr); + } + public: CompactibleSpace() : _compaction_top(NULL), _next_compaction_space(NULL) {} @@ -442,6 +446,10 @@ virtual HeapWord* forward(oop q, size_t size, CompactPoint* cp, HeapWord* compact_top); + virtual oop compact_oop(HeapWord* addr) const { + return oop(addr); + } + // Return a size with adjustments as required of the space. virtual size_t adjust_object_size_v(size_t size) const { return size; } @@ -584,7 +592,7 @@ // Iteration void oop_iterate(ExtendedOopClosure* cl); - void object_iterate(ObjectClosure* blk); + virtual void object_iterate(ObjectClosure* blk); // For contiguous spaces this method will iterate safely over objects // in the space (i.e., between bottom and top) when at a safepoint. void safe_object_iterate(ObjectClosure* blk);
--- a/src/share/vm/gc/shared/space.inline.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/gc/shared/space.inline.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -81,7 +81,7 @@ // We're sure to be here before any objects are compacted into this // space, so this is a good time to initialize this: space->set_compaction_top(space->bottom()); - + if (cp->space == NULL) { assert(cp->gen != NULL, "need a generation"); assert(cp->threshold == NULL, "just checking"); @@ -120,15 +120,18 @@ const intx interval = PrefetchScanIntervalInBytes; while (q < t) { + assert(!space->scanned_block_is_obj(q) || - oop(q)->mark()->is_marked() || oop(q)->mark()->is_unlocked() || - oop(q)->mark()->has_bias_pattern(), + space->make_oop(q)->mark()->is_marked() || + oopDesc::bs()->resolve_oop(space->make_oop(q))->mark()->is_marked() || + space->make_oop(q)->mark()->is_unlocked() || + space->make_oop(q)->mark()->has_bias_pattern(), "these are the only valid states during a mark sweep"); - if (space->scanned_block_is_obj(q) && oop(q)->is_gc_marked()) { + if (space->scanned_block_is_obj(q) && space->make_oop(q)->is_gc_marked()) { // prefetch beyond q Prefetch::write(q, interval); size_t size = space->scanned_block_size(q); - compact_top = cp->space->forward(oop(q), size, cp, compact_top); + compact_top = cp->space->forward(space->make_oop(q), size, cp, compact_top); q += size; end_of_live = q; } else { @@ -138,14 +141,14 @@ // prefetch beyond end Prefetch::write(end, interval); end += space->scanned_block_size(end); - } while (end < t && (!space->scanned_block_is_obj(end) || !oop(end)->is_gc_marked())); + } while (end < t && (!space->scanned_block_is_obj(end) || !space->make_oop(end)->is_gc_marked())); // see if we might want to pretend this object is alive so that // we don't have to compact quite as often. if (allowed_deadspace > 0 && q == compact_top) { size_t sz = pointer_delta(end, q); if (space->insert_deadspace(allowed_deadspace, q, sz)) { - compact_top = cp->space->forward(oop(q), sz, cp, compact_top); + compact_top = cp->space->forward(space->make_oop(q), sz, cp, compact_top); q = end; end_of_live = end; continue; @@ -161,7 +164,7 @@ // record the current LiveRange object. // liveRange->start() is overlaid on the mark word. - liveRange = (LiveRange*)q; + liveRange = (LiveRange*) (HeapWord*) space->make_oop(q); liveRange->set_start(end); liveRange->set_end(end); @@ -176,6 +179,7 @@ } assert(q == t, "just checking"); + if (liveRange != NULL) { liveRange->set_end(q); } @@ -191,7 +195,7 @@ template <class SpaceType> inline void CompactibleSpace::scan_and_adjust_pointers(SpaceType* space) { - // adjust all the interior pointers to point at the new locations of objects + // adjust all the interior pointers to point at the new locations of objectsH // Used by MarkSweep::mark_sweep_phase3() HeapWord* q = space->bottom(); @@ -199,7 +203,7 @@ assert(space->_first_dead <= space->_end_of_live, "Stands to reason, no?"); - if (q < t && space->_first_dead > q && !oop(q)->is_gc_marked()) { + if (q < t && space->_first_dead > q && !space->make_oop(q)->is_gc_marked()) { // we have a chunk of the space which hasn't moved and we've // reinitialized the mark word during the previous pass, so we can't // use is_gc_marked for the traversal. @@ -214,7 +218,7 @@ assert(space->block_is_obj(q), "should be at block boundaries, and should be looking at objs"); // point all the oops to the new location - size_t size = MarkSweep::adjust_pointers(oop(q)); + size_t size = MarkSweep::adjust_pointers(space->make_oop(q)); size = space->adjust_obj_size(size); q += size; @@ -225,7 +229,7 @@ } else { // $$$ This is funky. Using this to read the previously written // LiveRange. See also use below. - q = (HeapWord*)oop(space->_first_dead)->mark()->decode_pointer(); + q = (HeapWord*) oop(space->_first_dead)->mark()->decode_pointer(); } } @@ -235,10 +239,10 @@ while (q < t) { // prefetch beyond q Prefetch::write(q, interval); - if (oop(q)->is_gc_marked()) { + if (space->make_oop(q)->is_gc_marked()) { // q is alive // point all the oops to the new location - size_t size = MarkSweep::adjust_pointers(oop(q)); + size_t size = MarkSweep::adjust_pointers(space->make_oop(q)); size = space->adjust_obj_size(size); debug_only(prev_q = q); q += size; @@ -246,7 +250,7 @@ // q is not a live object, so its mark should point at the next // live object debug_only(prev_q = q); - q = (HeapWord*) oop(q)->mark()->decode_pointer(); + q = (HeapWord*) space->make_oop(q)->mark()->decode_pointer(); assert(q > prev_q, "we should be moving forward through memory"); } } @@ -262,8 +266,8 @@ HeapWord* q = space->bottom(); HeapWord* const t = space->_end_of_live; debug_only(HeapWord* prev_q = NULL); - - if (q < t && space->_first_dead > q && !oop(q)->is_gc_marked()) { + + if (q < t && space->_first_dead > q && !space->make_oop(q)->is_gc_marked()) { #ifdef ASSERT // Debug only // we have a chunk of the space which hasn't moved and we've reinitialized // the mark word during the previous pass, so we can't use is_gc_marked for @@ -272,7 +276,7 @@ while (q < end) { size_t size = space->obj_size(q); - assert(!oop(q)->is_gc_marked(), "should be unmarked (special dense prefix handling)"); + assert(!space->make_oop(q)->is_gc_marked(), "should be unmarked (special dense prefix handling)"); prev_q = q; q += size; } @@ -289,10 +293,10 @@ const intx scan_interval = PrefetchScanIntervalInBytes; const intx copy_interval = PrefetchCopyIntervalInBytes; while (q < t) { - if (!oop(q)->is_gc_marked()) { + if (!space->make_oop(q)->is_gc_marked()) { // mark is pointer to next marked oop debug_only(prev_q = q); - q = (HeapWord*) oop(q)->mark()->decode_pointer(); + q = (HeapWord*) space->make_oop(q)->mark()->decode_pointer(); assert(q > prev_q, "we should be moving forward through memory"); } else { // prefetch beyond q @@ -300,14 +304,14 @@ // size and destination size_t size = space->obj_size(q); - HeapWord* compaction_top = (HeapWord*)oop(q)->forwardee(); + HeapWord* compaction_top = (HeapWord*)space->make_oop(q)->forwardee(); // prefetch beyond compaction_top Prefetch::write(compaction_top, copy_interval); // copy object and reinit its mark assert(q != compaction_top, "everything in this pass should be moving"); - Copy::aligned_conjoint_words(q, compaction_top, size); + Copy::aligned_conjoint_words((HeapWord*) space->make_oop(q), compaction_top, size); oop(compaction_top)->init_mark(); assert(oop(compaction_top)->klass() != NULL, "should have a class");
--- a/src/share/vm/gc/shared/taskqueue.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/gc/shared/taskqueue.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -107,6 +107,17 @@ err_msg("overflow_max_len=" SIZE_FORMAT " overflow=" SIZE_FORMAT, get(overflow_max_len), get(overflow))); } + +void TaskQueueStats::verify_only_pushes() const +{ + assert((get(pop) == 0), + err_msg("pops=" SIZE_FORMAT , + get(pop))); + assert((get(steal) == 0), + err_msg("steals=" SIZE_FORMAT , + get(steal))); +} + #endif // ASSERT #endif // TASKQUEUE_STATS
--- a/src/share/vm/gc/shared/taskqueue.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/gc/shared/taskqueue.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -79,6 +79,7 @@ void print(outputStream* const stream = tty, unsigned int width = 10) const; DEBUG_ONLY(void verify() const;) + DEBUG_ONLY(void verify_only_pushes() const;) private: size_t _stats[last_stat_id];
--- a/src/share/vm/gc/shared/threadLocalAllocBuffer.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/gc/shared/threadLocalAllocBuffer.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -51,6 +51,8 @@ thread->tlab().initialize_statistics(); } + Universe::heap()->accumulate_statistics_all_gclabs(); + // Publish new stats if some allocation occurred. if (global_stats()->allocation() != 0) { global_stats()->publish(); @@ -66,7 +68,7 @@ size_t used = Universe::heap()->tlab_used(thread); _gc_waste += (unsigned)remaining(); - size_t total_allocated = thread->allocated_bytes(); + size_t total_allocated = _gclab ? thread->allocated_bytes_gclab() : thread->allocated_bytes(); size_t allocated_since_last_gc = total_allocated - _allocated_before_last_gc; _allocated_before_last_gc = total_allocated; @@ -98,7 +100,7 @@ } else { assert(_number_of_refills == 0 && _fast_refill_waste == 0 && - _slow_refill_waste == 0 && _gc_waste == 0, + _slow_refill_waste == 0, // && _gc_waste == 0, "tlab stats == 0"); } global_stats()->update_slow_allocations(_slow_allocations); @@ -113,10 +115,15 @@ invariants(); if (retire) { - myThread()->incr_allocated_bytes(used_bytes()); + if (_gclab) { + myThread()->incr_allocated_bytes_gclab(used_bytes()); + } else { + myThread()->incr_allocated_bytes(used_bytes()); + } } - CollectedHeap::fill_with_object(top(), hard_end(), retire); + HeapWord* obj = Universe::heap()->tlab_post_allocation_setup(top(), false); + CollectedHeap::fill_with_object(obj, hard_end(), retire, false); if (retire || ZeroTLAB) { // "Reset" the TLAB set_start(NULL); @@ -191,17 +198,19 @@ invariants(); } -void ThreadLocalAllocBuffer::initialize() { +void ThreadLocalAllocBuffer::initialize(bool gclab) { + _gclab = gclab; initialize(NULL, // start NULL, // top NULL); // end - set_desired_size(initial_desired_size()); - // Following check is needed because at startup the main (primordial) // thread is initialized before the heap is. The initialization for // this thread is redone in startup_initialization below. if (Universe::heap() != NULL) { + + set_desired_size(initial_desired_size()); + size_t capacity = Universe::heap()->tlab_capacity(myThread()) / HeapWordSize; double alloc_frac = desired_size() * target_refills() / (double) capacity; _allocation_fraction.sample(alloc_frac); @@ -224,7 +233,8 @@ // During jvm startup, the main (primordial) thread is initialized // before the heap is initialized. So reinitialize it now. guarantee(Thread::current()->is_Java_thread(), "tlab initialization thread not Java thread"); - Thread::current()->tlab().initialize(); + Thread::current()->tlab().initialize(false); + Thread::current()->gclab().initialize(true); if (PrintTLAB && Verbose) { gclog_or_tty->print("TLAB min: " SIZE_FORMAT " initial: " SIZE_FORMAT " max: " SIZE_FORMAT "\n", @@ -256,12 +266,12 @@ double waste_percent = alloc == 0 ? 0.0 : 100.0 * waste / alloc; size_t tlab_used = Universe::heap()->tlab_used(thrd); - gclog_or_tty->print("TLAB: %s thread: " INTPTR_FORMAT " [id: %2d]" + gclog_or_tty->print("TLAB: %s %s thread: " INTPTR_FORMAT " [id: %2d]" " desired_size: " SIZE_FORMAT "KB" " slow allocs: %d refill waste: " SIZE_FORMAT "B" " alloc:%8.5f %8.0fKB refills: %d waste %4.1f%% gc: %dB" " slow: %dB fast: %dB\n", - tag, p2i(thrd), thrd->osthread()->thread_id(), + tag, _gclab ? "gclab" : "tlab ", p2i(thrd), thrd->osthread()->thread_id(), _desired_size / (K / HeapWordSize), _slow_allocations, _refill_waste_limit * HeapWordSize, _allocation_fraction.average(), @@ -285,11 +295,28 @@ } Thread* ThreadLocalAllocBuffer::myThread() { - return (Thread*)(((char *)this) + - in_bytes(start_offset()) - - in_bytes(Thread::tlab_start_offset())); + ByteSize gclab_offset = Thread::gclab_start_offset(); + ByteSize tlab_offset = Thread::tlab_start_offset(); + ByteSize offs = _gclab ? gclab_offset : tlab_offset; + Thread* thread = (Thread*)(((char *)this) + + in_bytes(start_offset()) - in_bytes(offs)); +#ifdef ASSERT + assert(this == (_gclab ? &thread->gclab() : &thread->tlab()), "must be"); +#endif + return thread; } +size_t ThreadLocalAllocBuffer::end_reserve() { + int reserve_size = typeArrayOopDesc::header_size(T_INT) + Universe::heap()->oop_extra_words(); + return MAX2(reserve_size, VM_Version::reserve_for_allocation_prefetch()); +} + +void ThreadLocalAllocBuffer::rollback(size_t size) { + HeapWord* old_top = top(); + if (old_top != NULL) { // Pathological case: we accept that we can't rollback. + set_top(old_top - size); + } +} GlobalTLABStats::GlobalTLABStats() : _allocating_threads_avg(TLABAllocationWeight) {
--- a/src/share/vm/gc/shared/threadLocalAllocBuffer.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/gc/shared/threadLocalAllocBuffer.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -57,10 +57,9 @@ unsigned _gc_waste; unsigned _slow_allocations; - AdaptiveWeightedAverage _allocation_fraction; // fraction of eden allocated in tlabs + bool _gclab; - void accumulate_statistics(); - void initialize_statistics(); + AdaptiveWeightedAverage _allocation_fraction; // fraction of eden allocated in tlabs void set_start(HeapWord* start) { _start = start; } void set_end(HeapWord* end) { _end = end; } @@ -79,9 +78,6 @@ // Make parsable and release it. void reset(); - // Resize based on amount of allocation, etc. - void resize(); - void invariants() const { assert(top() >= start() && top() <= end(), "invalid tlab"); } void initialize(HeapWord* start, HeapWord* top, HeapWord* end); @@ -106,6 +102,12 @@ // do nothing. tlabs must be inited by initialize() calls } + // Resize based on amount of allocation, etc. + void resize(); + + void accumulate_statistics(); + void initialize_statistics(); + static const size_t min_size() { return align_object_size(MinTLABSize / HeapWordSize) + alignment_reserve(); } static const size_t max_size() { assert(_max_size != 0, "max_size not set up"); return _max_size; } static void set_max_size(size_t max_size) { _max_size = max_size; } @@ -125,11 +127,11 @@ // Allocate size HeapWords. The memory is NOT initialized to zero. inline HeapWord* allocate(size_t size); + // Rolls back a single allocation of the given size. + void rollback(size_t size); + // Reserve space at the end of TLAB - static size_t end_reserve() { - int reserve_size = typeArrayOopDesc::header_size(T_INT); - return MAX2(reserve_size, VM_Version::reserve_for_allocation_prefetch()); - } + static size_t end_reserve(); static size_t alignment_reserve() { return align_object_size(end_reserve()); } static size_t alignment_reserve_in_bytes() { return alignment_reserve() * HeapWordSize; } @@ -157,7 +159,7 @@ static void resize_all_tlabs(); void fill(HeapWord* start, HeapWord* top, size_t new_size); - void initialize(); + void initialize(bool gclab); static size_t refill_waste_limit_increment() { return TLABWasteIncrement; }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/brooksPointer.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,73 @@ +/* +Copyright 2014 Red Hat, Inc. and/or its affiliates. + */ + +#include "memory/universe.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shenandoah/brooksPointer.hpp" +#include "gc/shenandoah/shenandoahBarrierSet.hpp" + +BrooksPointer::BrooksPointer(HeapWord** hw) : _heap_word(hw) {} + +BrooksPointer BrooksPointer::get(oop obj) { + HeapWord* hw_obj = (HeapWord*) obj; + HeapWord* brooks_ptr = hw_obj - 1; + // We know that the value in that memory location is a pointer to another + // heapword/oop. + return BrooksPointer((HeapWord**) brooks_ptr); +} + +void BrooksPointer::set_forwardee(oop forwardee) { + assert(ShenandoahHeap::heap()->is_in(forwardee), "forwardee must be valid oop in the heap"); + *_heap_word = (HeapWord*) forwardee; +#ifdef ASSERT + if (ShenandoahTraceBrooksPointers) { + tty->print_cr("setting_forwardee to "PTR_FORMAT" = "PTR_FORMAT, p2i((HeapWord*) forwardee), p2i(*_heap_word)); + } +#endif +} + +HeapWord* BrooksPointer::cas_forwardee(HeapWord* old, HeapWord* forwardee) { + assert(ShenandoahHeap::heap()->is_in(forwardee), "forwardee must point to a heap address"); + + + + HeapWord* o = old; + HeapWord* n = forwardee; + HeapWord* result; + +#ifdef ASSERT + if (ShenandoahTraceBrooksPointers) { + tty->print_cr("Attempting to CAS "PTR_FORMAT" value "PTR_FORMAT" from "PTR_FORMAT" to "PTR_FORMAT, p2i(_heap_word), p2i(*_heap_word), p2i(o), p2i(n)); + } +#endif + +#ifdef ASSERT + if (ShenandoahVerifyWritesToFromSpace || ShenandoahVerifyReadsToFromSpace) { + ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap(); + ShenandoahHeapRegion* hr = sh->heap_region_containing(old); + + { + hr->memProtectionOff(); + result = (HeapWord*) (HeapWord*) Atomic::cmpxchg_ptr(n, _heap_word, o); + hr->memProtectionOn(); + } + } else { + result = (HeapWord*) (HeapWord*) Atomic::cmpxchg_ptr(n, _heap_word, o); + } +#else + result = (HeapWord*) (HeapWord*) Atomic::cmpxchg_ptr(n, _heap_word, o); +#endif + +#ifdef ASSERT + if (ShenandoahTraceBrooksPointers) { + tty->print_cr("Result of CAS from "PTR_FORMAT" to "PTR_FORMAT" was "PTR_FORMAT" read value was "PTR_FORMAT, p2i(o), p2i(n), p2i(result), p2i(*_heap_word)); + } +#endif + + return result; +} + +bool BrooksPointer::check_forwardee_is_in_heap(oop forwardee) { + return Universe::heap()->is_in(forwardee); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/brooksPointer.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,62 @@ +/* +Copyright 2014 Red Hat, Inc. and/or its affiliates. + */ + +#ifndef SHARE_VM_GC_SHENANDOAH_BROOKSPOINTER_HPP +#define SHARE_VM_GC_SHENANDOAH_BROOKSPOINTER_HPP + +#include "oops/oop.hpp" +#include "utilities/globalDefinitions.hpp" +#include "gc/shenandoah/shenandoahHeap.hpp" + +class BrooksPointer { + +public: + static const uint BROOKS_POINTER_OBJ_SIZE = 1; + +private: + + HeapWord** _heap_word; + + BrooksPointer(HeapWord** heap_word); + +public: + + bool check_forwardee_is_in_heap(oop forwardee); + + inline oop get_forwardee_raw() { + return oop(*_heap_word); + } + + inline oop get_forwardee() { + oop forwardee; + +#ifdef ASSERT + if (ShenandoahVerifyReadsToFromSpace) { + ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap(); + ShenandoahHeapRegion* hr = sh->heap_region_containing(_heap_word); + + { + hr->memProtectionOff(); + forwardee = (oop) (*_heap_word); + hr->memProtectionOn(); + } + } else { + forwardee = get_forwardee_raw(); + } +#else + forwardee = get_forwardee_raw(); +#endif + + assert(check_forwardee_is_in_heap(forwardee), "forwardee must be in heap"); + assert(forwardee->is_oop(), "forwardee must be valid oop"); + return forwardee; + } + + void set_forwardee(oop forwardee); + HeapWord* cas_forwardee(HeapWord* old, HeapWord* forwardee); + + static BrooksPointer get(oop obj); +}; + +#endif // SHARE_VM_GC_SHENANDOAH_BROOKSPOINTER_HPP
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/shenandoahBarrierSet.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,342 @@ +/* +Copyright 2014 Red Hat, Inc. and/or its affiliates. + */ + +#include "precompiled.hpp" +#include "gc/g1/g1SATBCardTableModRefBS.hpp" +#include "gc/shenandoah/brooksPointer.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahBarrierSet.hpp" +#include "memory/universe.hpp" +#include "utilities/array.hpp" + +class UpdateRefsForOopClosure: public ExtendedOopClosure { + +private: + ShenandoahHeap* _heap; +public: + UpdateRefsForOopClosure() { + _heap = ShenandoahHeap::heap(); + } + + void do_oop(oop* p) { + _heap->maybe_update_oop_ref(p); + } + + void do_oop(narrowOop* p) { + Unimplemented(); + } + +}; + +ShenandoahBarrierSet::ShenandoahBarrierSet() : + BarrierSet(BarrierSet::FakeRtti(BarrierSet::ShenandoahBarrierSet)) +{ +} + +void ShenandoahBarrierSet::print_on(outputStream* st) const { + st->print("ShenandoahBarrierSet"); +} + +bool ShenandoahBarrierSet::is_a(BarrierSet::Name bsn) { + return bsn == BarrierSet::ShenandoahBarrierSet; +} + +bool ShenandoahBarrierSet::has_read_prim_array_opt() { + return true; +} + +bool ShenandoahBarrierSet::has_read_prim_barrier() { + return false; +} + +bool ShenandoahBarrierSet::has_read_ref_array_opt() { + return true; +} + +bool ShenandoahBarrierSet::has_read_ref_barrier() { + return false; +} + +bool ShenandoahBarrierSet::has_read_region_opt() { + return true; +} + +bool ShenandoahBarrierSet::has_write_prim_array_opt() { + return true; +} + +bool ShenandoahBarrierSet::has_write_prim_barrier() { + return false; +} + +bool ShenandoahBarrierSet::has_write_ref_array_opt() { + return true; +} + +bool ShenandoahBarrierSet::has_write_ref_barrier() { + return true; +} + +bool ShenandoahBarrierSet::has_write_ref_pre_barrier() { + return true; +} + +bool ShenandoahBarrierSet::has_write_region_opt() { + return true; +} + +bool ShenandoahBarrierSet::is_aligned(HeapWord* hw) { + return true; +} + +void ShenandoahBarrierSet::read_prim_array(MemRegion mr) { + Unimplemented(); +} + +void ShenandoahBarrierSet::read_prim_field(HeapWord* hw, size_t s){ + Unimplemented(); +} + +bool ShenandoahBarrierSet::read_prim_needs_barrier(HeapWord* hw, size_t s) { + return false; +} + +void ShenandoahBarrierSet::read_ref_array(MemRegion mr) { + Unimplemented(); +} + +void ShenandoahBarrierSet::read_ref_field(void* v) { + // tty->print_cr("read_ref_field: v = "PTR_FORMAT, v); + // return *v; +} + +bool ShenandoahBarrierSet::read_ref_needs_barrier(void* v) { + Unimplemented(); + return false; +} + +void ShenandoahBarrierSet::read_region(MemRegion mr) { + Unimplemented(); +} + +void ShenandoahBarrierSet::resize_covered_region(MemRegion mr) { + Unimplemented(); +} + +void ShenandoahBarrierSet::write_prim_array(MemRegion mr) { + Unimplemented(); +} + +void ShenandoahBarrierSet::write_prim_field(HeapWord* hw, size_t s , juint x, juint y) { + Unimplemented(); +} + +bool ShenandoahBarrierSet::write_prim_needs_barrier(HeapWord* hw, size_t s, juint x, juint y) { + Unimplemented(); + return false; +} + +bool ShenandoahBarrierSet::need_update_refs_barrier() { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + return heap->is_update_references_in_progress() || (heap->concurrent_mark_in_progress() && heap->need_update_refs()); +} + +void ShenandoahBarrierSet::write_ref_array_work(MemRegion mr) { + if (! need_update_refs_barrier()) return; + ShenandoahHeap* heap = ShenandoahHeap::heap(); + for (HeapWord* word = mr.start(); word < mr.end(); word++) { + oop* oop_ptr = (oop*) word; + heap->maybe_update_oop_ref(oop_ptr); + } +} + +template <class T> +void ShenandoahBarrierSet::write_ref_array_pre_work(T* dst, int count) { + +#ifdef ASSERT + ShenandoahHeap *sh = (ShenandoahHeap*) Universe::heap(); + if (sh->is_in(dst) && + sh->heap_region_containing((HeapWord*) dst)->is_in_collection_set() && + ! sh->cancelled_concgc()) { + tty->print_cr("dst = "PTR_FORMAT, p2i(dst)); + sh->heap_region_containing((HeapWord*) dst)->print(); + assert(false, "We should have fixed this earlier"); + } +#endif + + if (! JavaThread::satb_mark_queue_set().is_active()) return; + // tty->print_cr("write_ref_array_pre_work: "PTR_FORMAT", "INT32_FORMAT, dst, count); + T* elem_ptr = dst; + for (int i = 0; i < count; i++, elem_ptr++) { + T heap_oop = oopDesc::load_heap_oop(elem_ptr); + if (!oopDesc::is_null(heap_oop)) { + G1SATBCardTableModRefBS::enqueue(oopDesc::decode_heap_oop_not_null(heap_oop)); + } + // tty->print_cr("write_ref_array_pre_work: oop: "PTR_FORMAT, heap_oop); + } +} + +void ShenandoahBarrierSet::write_ref_array_pre(oop* dst, int count, bool dest_uninitialized) { + if (! dest_uninitialized) { + write_ref_array_pre_work(dst, count); + } +} + +void ShenandoahBarrierSet::write_ref_array_pre(narrowOop* dst, int count, bool dest_uninitialized) { + if (! dest_uninitialized) { + write_ref_array_pre_work(dst, count); + } +} + +template <class T> +void ShenandoahBarrierSet::write_ref_field_pre_static(T* field, oop newVal) { + T heap_oop = oopDesc::load_heap_oop(field); + +#ifdef ASSERT + ShenandoahHeap *sh = (ShenandoahHeap*) Universe::heap(); + if (sh->is_in(field) && + sh->heap_region_containing((HeapWord*)field)->is_in_collection_set() && + ! sh->cancelled_concgc()) { + tty->print_cr("field = "PTR_FORMAT, p2i(field)); + sh->heap_region_containing((HeapWord*)field)->print(); + assert(false, "We should have fixed this earlier"); + } +#endif + + if (!oopDesc::is_null(heap_oop)) { + G1SATBCardTableModRefBS::enqueue(oopDesc::decode_heap_oop(heap_oop)); + // tty->print_cr("write_ref_field_pre_static: v = "PTR_FORMAT" o = "PTR_FORMAT" old: "PTR_FORMAT, field, newVal, heap_oop); + } +} + +template <class T> +inline void ShenandoahBarrierSet::inline_write_ref_field_pre(T* field, oop newVal) { + write_ref_field_pre_static(field, newVal); +} + +// These are the more general virtual versions. +void ShenandoahBarrierSet::write_ref_field_pre_work(oop* field, oop new_val) { + write_ref_field_pre_static(field, new_val); +} + +void ShenandoahBarrierSet::write_ref_field_pre_work(narrowOop* field, oop new_val) { + write_ref_field_pre_static(field, new_val); +} + +void ShenandoahBarrierSet::write_ref_field_pre_work(void* field, oop new_val) { + guarantee(false, "Not needed"); +} + +void ShenandoahBarrierSet::write_ref_field_work(void* v, oop o, bool release) { + if (! need_update_refs_barrier()) return; + assert (! UseCompressedOops, "compressed oops not supported yet"); + ShenandoahHeap::heap()->maybe_update_oop_ref((oop*) v); + // tty->print_cr("write_ref_field_work: v = "PTR_FORMAT" o = "PTR_FORMAT, v, o); +} + +void ShenandoahBarrierSet::write_region_work(MemRegion mr) { + + if (! need_update_refs_barrier()) return; + + // This is called for cloning an object (see jvm.cpp) after the clone + // has been made. We are not interested in any 'previous value' because + // it would be NULL in any case. But we *are* interested in any oop* + // that potentially need to be updated. + + // tty->print_cr("write_region_work: "PTR_FORMAT", "PTR_FORMAT, mr.start(), mr.end()); + oop obj = oop(mr.start()); + assert(obj->is_oop(), "must be an oop"); + UpdateRefsForOopClosure cl; + obj->oop_iterate(&cl); +} + +oop ShenandoahBarrierSet::resolve_oop(oop src) { + return ShenandoahBarrierSet::resolve_oop_static(src); +} + +oop ShenandoahBarrierSet::maybe_resolve_oop(oop src) { + if (Universe::heap()->is_in(src)) { + return resolve_oop_static(src); + } else { + return src; + } +} + +oop ShenandoahBarrierSet::resolve_and_maybe_copy_oop_work(oop src) { + ShenandoahHeap *sh = (ShenandoahHeap*) Universe::heap(); + assert(src != NULL, "only evacuated non NULL oops"); + + if (sh->in_cset_fast_test((HeapWord*) src)) { + return resolve_and_maybe_copy_oop_work2(src); + } else { + return src; + } +} + +oop ShenandoahBarrierSet::resolve_and_maybe_copy_oop_work2(oop src) { + ShenandoahHeap *sh = (ShenandoahHeap*) Universe::heap(); + if (! sh->is_evacuation_in_progress()) { + // We may get here through a barrier that just took a safepoint that + // turned off evacuation. In this case, return right away. + return ShenandoahBarrierSet::resolve_oop_static(src); + } + assert(src != NULL, "only evacuated non NULL oops"); + assert(sh->heap_region_containing(src)->is_in_collection_set(), "only evacuate objects in collection set"); + assert(! sh->heap_region_containing(src)->is_humongous(), "never evacuate humongous objects"); + // TODO: Consider passing thread from caller. + oop dst = sh->evacuate_object(src, Thread::current()); +#ifdef ASSERT + if (ShenandoahTraceEvacuations) { + tty->print_cr("src = "PTR_FORMAT" dst = "PTR_FORMAT" src = "PTR_FORMAT" src-2 = "PTR_FORMAT, + p2i((HeapWord*) src), p2i((HeapWord*) dst), p2i((HeapWord*) src), p2i(((HeapWord*) src) - 2)); + } +#endif + assert(sh->is_in(dst), "result should be in the heap"); + return dst; +} + +oop ShenandoahBarrierSet::resolve_and_maybe_copy_oopHelper(oop src) { + if (src != NULL) { + ShenandoahHeap *sh = (ShenandoahHeap*) Universe::heap(); + oop tmp = resolve_oop_static(src); + if (! sh->is_evacuation_in_progress()) { + return tmp; + } + return resolve_and_maybe_copy_oop_work(src); + } else { + return NULL; + } +} + +JRT_LEAF(oopDesc*, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_c2(oopDesc* src)) + oop result = ((ShenandoahBarrierSet*) oopDesc::bs())->resolve_and_maybe_copy_oop_work2(oop(src)); + // tty->print_cr("called C2 write barrier with: %p result: %p copy: %d", (oopDesc*) src, (oopDesc*) result, src != result); + return (oopDesc*) result; +JRT_END + +IRT_LEAF(oopDesc*, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_interp(oopDesc* src)) + oop result = ((ShenandoahBarrierSet*)oopDesc::bs())->resolve_and_maybe_copy_oop_work2(oop(src)); + // tty->print_cr("called interpreter write barrier with: %p result: %p", src, result); + return (oopDesc*) result; +IRT_END + +JRT_LEAF(oopDesc*, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_c1(JavaThread* thread, oopDesc* src)) + oop result = ((ShenandoahBarrierSet*)oopDesc::bs())->resolve_and_maybe_copy_oop_work2(oop(src)); + // tty->print_cr("called static write barrier (2) with: "PTR_FORMAT" result: "PTR_FORMAT, p2i(src), p2i((oopDesc*)(result))); + return (oopDesc*) result; +JRT_END + +oop ShenandoahBarrierSet::resolve_and_maybe_copy_oop(oop src) { + ShenandoahHeap *sh = (ShenandoahHeap*) Universe::heap(); + oop result; + if (src != NULL && sh->is_in(src)) { + result = resolve_and_maybe_copy_oopHelper(src); + assert(sh->is_in(result), "result should be in the heap"); + } else { + result = src; + } + assert(result == NULL || (sh->is_in(result) && result->is_oop()), "resolved oop must be NULL, or a valid oop in the heap"); + return result; + } +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/shenandoahBarrierSet.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,187 @@ +/* +Copyright 2014 Red Hat, Inc. and/or its affiliates. + */ +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_HPP + +#include "gc/shenandoah/shenandoahHeap.hpp" +#include "gc/shared/barrierSet.hpp" + +class ShenandoahBarrierSet: public BarrierSet { +private: + + static inline oop get_shenandoah_forwardee_helper(oop p) { + assert(UseShenandoahGC, "must only be called when Shenandoah is used."); + assert(Universe::heap()->is_in(p), "We shouldn't be calling this on objects not in the heap"); + oop forwardee; +#ifdef ASSERT + if (ShenandoahVerifyReadsToFromSpace) { + ShenandoahHeap* heap = (ShenandoahHeap *) Universe::heap(); + ShenandoahHeapRegion* region = heap->heap_region_containing(p); + { + region->memProtectionOff(); + forwardee = oop( *((HeapWord**) ((HeapWord*) p) - 1)); + region->memProtectionOn(); + } + } else { + forwardee = oop( *((HeapWord**) ((HeapWord*) p) - 1)); + } +#else + forwardee = oop( *((HeapWord**) ((HeapWord*) p) - 1)); +#endif + return forwardee; + } + +public: + + ShenandoahBarrierSet(); + + void print_on(outputStream* st) const; + + bool is_a(BarrierSet::Name bsn); + + bool has_read_prim_array_opt(); + bool has_read_prim_barrier(); + bool has_read_ref_array_opt(); + bool has_read_ref_barrier(); + bool has_read_region_opt(); + bool has_write_prim_array_opt(); + bool has_write_prim_barrier(); + bool has_write_ref_array_opt(); + bool has_write_ref_barrier(); + bool has_write_ref_pre_barrier(); + bool has_write_region_opt(); + bool is_aligned(HeapWord* hw); + void read_prim_array(MemRegion mr); + void read_prim_field(HeapWord* hw, size_t s); + bool read_prim_needs_barrier(HeapWord* hw, size_t s); + void read_ref_array(MemRegion mr); + + void read_ref_field(void* v); + + bool read_ref_needs_barrier(void* v); + void read_region(MemRegion mr); + void resize_covered_region(MemRegion mr); + void write_prim_array(MemRegion mr); + void write_prim_field(HeapWord* hw, size_t s , juint x, juint y); + bool write_prim_needs_barrier(HeapWord* hw, size_t s, juint x, juint y); + void write_ref_array_work(MemRegion mr); + + template <class T> void + write_ref_array_pre_work(T* dst, int count); + + void write_ref_array_pre(oop* dst, int count, bool dest_uninitialized); + + void write_ref_array_pre(narrowOop* dst, int count, bool dest_uninitialized); + + + template <class T> static void write_ref_field_pre_static(T* field, oop newVal); + + // We export this to make it available in cases where the static + // type of the barrier set is known. Note that it is non-virtual. + template <class T> inline void inline_write_ref_field_pre(T* field, oop newVal); + + // These are the more general virtual versions. + void write_ref_field_pre_work(oop* field, oop new_val); + void write_ref_field_pre_work(narrowOop* field, oop new_val); + void write_ref_field_pre_work(void* field, oop new_val); + + void write_ref_field_work(void* v, oop o, bool release = false); + void write_region_work(MemRegion mr); + + virtual oop resolve_oop(oop src); + + template <class T> + static inline oop resolve_and_update_oop_static(T p, oop obj) { + oop forw = ShenandoahBarrierSet::resolve_oop_static_not_null(obj); + if (forw != obj) { + obj = forw; + oopDesc::encode_store_heap_oop_not_null(p, obj); + } + return obj; + } + + static inline oop resolve_oop_static_not_null(oop p) { + assert(p != NULL, "Must be NULL checked"); + + oop result = get_shenandoah_forwardee_helper(p); + + if (result != NULL) { +#ifdef ASSERT + if (result != p) { + oop second_forwarding = get_shenandoah_forwardee_helper(result); + + // We should never be forwarded more than once. + if (result != second_forwarding) { + ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap(); + tty->print("first reference "PTR_FORMAT" is in heap region:\n", p2i((HeapWord*) p)); + sh->heap_region_containing(p)->print(); + tty->print("first_forwarding "PTR_FORMAT" is in heap region:\n", p2i((HeapWord*) result)); + sh->heap_region_containing(result)->print(); + tty->print("final reference "PTR_FORMAT" is in heap region:\n", p2i((HeapWord*) second_forwarding)); + sh->heap_region_containing(second_forwarding)->print(); + assert(get_shenandoah_forwardee_helper(result) == result, "Only one fowarding per customer"); + } + } +#endif + if (! ShenandoahVerifyReadsToFromSpace) { + // is_oop() would trigger a SEGFAULT when we're checking from-space-access. + assert(ShenandoahHeap::heap()->is_in(result) && result->is_oop(), "resolved oop must be a valid oop in the heap"); + } + } + return result; + } + + static inline oop resolve_oop_static(oop p) { + if (((HeapWord*) p) != NULL) { + return resolve_oop_static_not_null(p); + } else { + return p; + } + } + + static inline oop resolve_oop_static_no_check(oop p) { + if (((HeapWord*) p) != NULL) { + return get_shenandoah_forwardee_helper(p); + } else { + return p; + } + } + + + virtual oop maybe_resolve_oop(oop src); + oop resolve_and_maybe_copy_oopHelper(oop src); + oop resolve_and_maybe_copy_oop_work(oop src); + oop resolve_and_maybe_copy_oop_work2(oop src); + virtual oop resolve_and_maybe_copy_oop(oop src); + + static oopDesc* resolve_and_maybe_copy_oop_c2(oopDesc* src); + static oopDesc* resolve_and_maybe_copy_oop_interp(oopDesc* src); + static oopDesc* resolve_and_maybe_copy_oop_c1(JavaThread* thread, oopDesc* src); + +private: + bool need_update_refs_barrier(); + +#ifndef CC_INTERP +public: + virtual void interpreter_read_barrier(MacroAssembler* masm, Register dst); + virtual void interpreter_read_barrier_not_null(MacroAssembler* masm, Register dst); + void interpreter_write_barrier(MacroAssembler* masm, Register dst); + +private: + void compile_resolve_oop_runtime(MacroAssembler* masm, Register dst); + +#endif +}; + +class ShenandoahMarkCompactBarrierSet : public ShenandoahBarrierSet { + + oop resolve_oop(oop src) { + return src; + } + oop maybe_resolve_oop(oop src) { + return src; + } +}; + +#endif //SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_HPP
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/shenandoahCollectorPolicy.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,757 @@ +#include "gc/shenandoah/shenandoahCollectorPolicy.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" + +class ShenandoahHeuristics : public CHeapObj<mtGC> { + + NumberSeq _allocation_rate_bytes; + NumberSeq _reclamation_rate_bytes; + + size_t _bytes_allocated_since_CM; + size_t _bytes_reclaimed_this_cycle; + +protected: + size_t _bytes_allocated_start_CM; + size_t _bytes_allocated_during_CM; + +public: + + ShenandoahHeuristics(); + + void record_bytes_allocated(size_t bytes); + void record_bytes_reclaimed(size_t bytes); + void record_bytes_start_CM(size_t bytes); + void record_bytes_end_CM(size_t bytes); + + virtual bool should_start_concurrent_mark(size_t used, size_t capacity) const=0; + virtual bool update_refs_early(); + virtual void choose_collection_and_free_sets(ShenandoahHeapRegionSet* region_set, + ShenandoahHeapRegionSet* collection_set, + ShenandoahHeapRegionSet* free_set) =0; + void print_tracing_info(); +}; + +ShenandoahHeuristics::ShenandoahHeuristics() : + _bytes_allocated_since_CM(0), + _bytes_reclaimed_this_cycle(0), + _bytes_allocated_start_CM(0), + _bytes_allocated_during_CM(0) +{ + if (PrintGCDetails) + tty->print_cr("initializing heuristics"); +} + +void ShenandoahCollectorPolicy::record_phase_start(TimingPhase phase) { + _timing_data[phase]._start = os::elapsedTime(); + + if (PrintGCTimeStamps) { + if (phase == init_mark) + _tracer->report_gc_start(GCCause::_shenandoah_init_mark, _conc_timer->gc_start()); + else if (phase == full_gc) + _tracer->report_gc_start(GCCause::_last_ditch_collection, _stw_timer->gc_start()); + + gclog_or_tty->gclog_stamp(_tracer->gc_id()); + gclog_or_tty->print("[GC %s start", _phase_names[phase]); + ShenandoahHeap* heap = (ShenandoahHeap*) Universe::heap(); + + gclog_or_tty->print(" total = " SIZE_FORMAT " K, used = " SIZE_FORMAT " K free = " SIZE_FORMAT " K", heap->capacity()/ K, heap->used() /K, + ((heap->capacity() - heap->used())/K) ); + + if (heap->calculateUsed() != heap->used()) { + gclog_or_tty->print("calc used = " SIZE_FORMAT " K heap used = " SIZE_FORMAT " K", + heap->calculateUsed() / K, heap->used() / K); + } + // assert(heap->calculateUsed() == heap->used(), "Just checking"); + gclog_or_tty->print_cr("]"); + } +} + +void ShenandoahCollectorPolicy::record_phase_end(TimingPhase phase) { + double end = os::elapsedTime(); + double elapsed = end - _timing_data[phase]._start; + _timing_data[phase]._ms.add(elapsed * 1000); + + if (ShenandoahGCVerbose && PrintGCDetails) { + tty->print_cr("PolicyPrint: %s "SIZE_FORMAT" took %lf ms", _phase_names[phase], + _timing_data[phase]._count++, elapsed * 1000); + } + if (PrintGCTimeStamps) { + ShenandoahHeap* heap = (ShenandoahHeap*) Universe::heap(); + gclog_or_tty->gclog_stamp(_tracer->gc_id()); + + gclog_or_tty->print("[GC %s end, %lf secs", _phase_names[phase], elapsed ); + gclog_or_tty->print(" total = " SIZE_FORMAT " K, used = " SIZE_FORMAT " K free = " SIZE_FORMAT " K", heap->capacity()/ K, heap->used() /K, + ((heap->capacity() - heap->used())/K) ); + + if (heap->calculateUsed() != heap->used()) { + gclog_or_tty->print("calc used = " SIZE_FORMAT " K heap used = " SIZE_FORMAT " K", + heap->calculateUsed() / K, heap->used() / K); + } + // assert(heap->calculateUsed() == heap->used(), "Stashed heap used must be equal to calculated heap used"); + gclog_or_tty->print_cr("]"); + + if (phase == recycle_regions) { + _tracer->report_gc_end(_conc_timer->gc_end(), _conc_timer->time_partitions()); + } else if (phase == full_gc) { + _tracer->report_gc_end(_stw_timer->gc_end(), _stw_timer->time_partitions()); + } else if (phase == conc_mark || phase == conc_evac || phase == conc_uprefs || phase == prepare_evac) { + if (_conc_gc_aborted) { + _tracer->report_gc_end(_conc_timer->gc_end(), _conc_timer->time_partitions()); + clear_conc_gc_aborted(); + } + } else if (phase == final_evac) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + this->record_bytes_end_CM(heap->_bytesAllocSinceCM); + } + } +} + +void ShenandoahCollectorPolicy::report_concgc_cancelled() { + if (PrintGCTimeStamps) { + gclog_or_tty->print("Concurrent GC Cancelled\n"); + set_conc_gc_aborted(); + // _tracer->report_gc_end(_conc_timer->gc_end(), _conc_timer->time_partitions()); + } +} + +bool ShenandoahHeuristics::update_refs_early() { + return ShenandoahUpdateRefsEarly; +} + +void ShenandoahHeuristics::record_bytes_allocated(size_t bytes) { + _bytes_allocated_since_CM = bytes; + _bytes_allocated_start_CM = bytes; + _allocation_rate_bytes.add(bytes); +} + +void ShenandoahHeuristics::record_bytes_reclaimed(size_t bytes) { + _bytes_reclaimed_this_cycle = bytes; + _reclamation_rate_bytes.add(bytes); +} + +void ShenandoahHeuristics::record_bytes_start_CM(size_t bytes) { + _bytes_allocated_start_CM = bytes; +} + +void ShenandoahHeuristics::record_bytes_end_CM(size_t bytes) { + _bytes_allocated_during_CM = (bytes > _bytes_allocated_start_CM) ? (bytes - _bytes_allocated_start_CM) + : bytes; +} + +class AggressiveHeuristics : public ShenandoahHeuristics { +public: + AggressiveHeuristics() : ShenandoahHeuristics(){ + if (PrintGCDetails) + tty->print_cr("Initializing aggressive heuristics"); + } + + virtual bool should_start_concurrent_mark(size_t used, size_t capacity) const { + return true; + } + virtual void choose_collection_and_free_sets(ShenandoahHeapRegionSet* region_set, + ShenandoahHeapRegionSet* collection_set, + ShenandoahHeapRegionSet* free_set) { + region_set->set_garbage_threshold(8); + region_set->choose_collection_and_free_sets(collection_set, free_set); + } +}; + +class HalfwayHeuristics : public ShenandoahHeuristics { +public: + HalfwayHeuristics() : ShenandoahHeuristics() { + if (PrintGCDetails) + tty->print_cr("Initializing halfway heuristics"); + } + + bool should_start_concurrent_mark(size_t used, size_t capacity) const { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + size_t threshold_bytes_allocated = heap->capacity() / 4; + if (used * 2 > capacity && heap->_bytesAllocSinceCM > threshold_bytes_allocated) + return true; + else + return false; + } + void choose_collection_and_free_sets(ShenandoahHeapRegionSet* region_set, + ShenandoahHeapRegionSet* collection_set, + ShenandoahHeapRegionSet* free_set) { + region_set->set_garbage_threshold(ShenandoahHeapRegion::RegionSizeBytes / 2); + region_set->choose_collection_and_free_sets(collection_set, free_set); + } +}; + +// GC as little as possible +class LazyHeuristics : public ShenandoahHeuristics { +public: + LazyHeuristics() : ShenandoahHeuristics() { + if (PrintGCDetails) { + tty->print_cr("Initializing lazy heuristics"); + } + } + + virtual bool should_start_concurrent_mark(size_t used, size_t capacity) const { + size_t targetStartMarking = (capacity / 5) * 4; + if (used > targetStartMarking) { + return true; + } else { + return false; + } + } + + virtual void choose_collection_and_free_sets(ShenandoahHeapRegionSet* region_set, + ShenandoahHeapRegionSet* collection_set, + ShenandoahHeapRegionSet* free_set) { + region_set->choose_collection_and_free_sets(collection_set, free_set); + } +}; + +// These are the heuristics in place when we made this class +class StatusQuoHeuristics : public ShenandoahHeuristics { +public: + StatusQuoHeuristics() : ShenandoahHeuristics() { + if (PrintGCDetails) { + tty->print_cr("Initializing status quo heuristics"); + } + } + + virtual bool should_start_concurrent_mark(size_t used, size_t capacity) const { + size_t targetStartMarking = capacity / 16; + ShenandoahHeap* heap = ShenandoahHeap::heap(); + size_t threshold_bytes_allocated = heap->capacity() / 4; + + if (used > targetStartMarking + && heap->_bytesAllocSinceCM > threshold_bytes_allocated) { + // Need to check that an appropriate number of regions have + // been allocated since last concurrent mark too. + return true; + } else { + return false; + } + } + + virtual void choose_collection_and_free_sets(ShenandoahHeapRegionSet* region_set, + ShenandoahHeapRegionSet* collection_set, + ShenandoahHeapRegionSet* free_set) { + region_set->choose_collection_and_free_sets(collection_set, free_set); + } +}; + +static uintx clamp(uintx value, uintx min, uintx max) { + value = MAX2(value, min); + value = MIN2(value, max); + return value; +} + +static double get_percent(uintx value) { + double _percent = static_cast<double>(clamp(value, 0, 100)); + return _percent / 100.; +} + +class DynamicHeuristics : public ShenandoahHeuristics { +private: + double _free_threshold_factor; + double _garbage_threshold_factor; + double _allocation_threshold_factor; + + uintx _free_threshold; + uintx _garbage_threshold; + uintx _allocation_threshold; + +public: + DynamicHeuristics() : ShenandoahHeuristics() { + if (PrintGCDetails) { + tty->print_cr("Initializing dynamic heuristics"); + } + + _free_threshold = 0; + _garbage_threshold = 0; + _allocation_threshold = 0; + + _free_threshold_factor = 0.; + _garbage_threshold_factor = 0.; + _allocation_threshold_factor = 0.; + } + + virtual ~DynamicHeuristics() {} + + virtual bool should_start_concurrent_mark(size_t used, size_t capacity) const { + + bool shouldStartConcurrentMark = false; + + ShenandoahHeap* heap = ShenandoahHeap::heap(); + size_t available = heap->free_regions()->available(); + uintx factor = heap->need_update_refs() ? ShenandoahFreeThreshold : ShenandoahInitialFreeThreshold; + size_t targetStartMarking = (capacity * factor) / 100; + + size_t threshold_bytes_allocated = heap->capacity() * _allocation_threshold_factor; + if (available < targetStartMarking && + heap->_bytesAllocSinceCM > threshold_bytes_allocated) + { + // Need to check that an appropriate number of regions have + // been allocated since last concurrent mark too. + shouldStartConcurrentMark = true; + } + + if (shouldStartConcurrentMark && ShenandoahTracePhases) { + tty->print_cr("Start GC at available: "SIZE_FORMAT", factor: "UINTX_FORMAT", update-refs: %s", available, factor, BOOL_TO_STR(heap->need_update_refs())); + } + return shouldStartConcurrentMark; + } + + virtual void choose_collection_and_free_sets(ShenandoahHeapRegionSet* region_set, + ShenandoahHeapRegionSet* collection_set, + ShenandoahHeapRegionSet* free_set) + { + region_set->set_garbage_threshold(ShenandoahHeapRegion::RegionSizeBytes * _garbage_threshold_factor); + region_set->choose_collection_and_free_sets(collection_set, free_set); + } + + void set_free_threshold(uintx free_threshold) { + this->_free_threshold_factor = get_percent(free_threshold); + this->_free_threshold = free_threshold; + } + + void set_garbage_threshold(uintx garbage_threshold) { + this->_garbage_threshold_factor = get_percent(garbage_threshold); + this->_garbage_threshold = garbage_threshold; + } + + void set_allocation_threshold(uintx allocationThreshold) { + this->_allocation_threshold_factor = get_percent(allocationThreshold); + this->_allocation_threshold = allocationThreshold; + } + + uintx get_allocation_threshold() { + return this->_allocation_threshold; + } + + uintx get_garbage_threshold() { + return this->_garbage_threshold; + } + + uintx get_free_threshold() { + return this->_free_threshold; + } +}; + + +class AdaptiveHeuristics : public ShenandoahHeuristics { +private: + size_t _max_live_data; + double _used_threshold_factor; + double _garbage_threshold_factor; + double _allocation_threshold_factor; + + uintx _used_threshold; + uintx _garbage_threshold; + uintx _allocation_threshold; + +public: + AdaptiveHeuristics() : ShenandoahHeuristics() { + if (PrintGCDetails) { + tty->print_cr("Initializing dynamic heuristics"); + } + + _max_live_data = 0; + + _used_threshold = 0; + _garbage_threshold = 0; + _allocation_threshold = 0; + + _used_threshold_factor = 0.; + _garbage_threshold_factor = 0.1; + _allocation_threshold_factor = 0.; + } + + virtual ~AdaptiveHeuristics() {} + + virtual bool should_start_concurrent_mark(size_t used, size_t capacity) const { + + ShenandoahHeap* _heap = ShenandoahHeap::heap(); + bool shouldStartConcurrentMark = false; + + size_t max_live_data = _max_live_data; + if (max_live_data == 0) { + max_live_data = capacity * 0.2; // Very generous initial value. + } else { + max_live_data *= 1.3; // Add some wiggle room. + } + size_t max_cycle_allocated = _heap->_max_allocated_gc; + if (max_cycle_allocated == 0) { + max_cycle_allocated = capacity * 0.3; // Very generous. + } else { + max_cycle_allocated *= 1.3; // Add 20% wiggle room. Should be enough. + } + size_t threshold = _heap->capacity() - max_cycle_allocated - max_live_data; + if (used > threshold) + { + shouldStartConcurrentMark = true; + } + + return shouldStartConcurrentMark; + } + + virtual void choose_collection_and_free_sets(ShenandoahHeapRegionSet* region_set, + ShenandoahHeapRegionSet* collection_set, + ShenandoahHeapRegionSet* free_set) + { + size_t bytes_alloc = ShenandoahHeap::heap()->_bytesAllocSinceCM; + size_t min_garbage = bytes_alloc/* * 1.1*/; + region_set->set_garbage_threshold(ShenandoahHeapRegion::RegionSizeBytes * _garbage_threshold_factor); + region_set->choose_collection_and_free_sets_min_garbage(collection_set, free_set, min_garbage); + /* + tty->print_cr("garbage to be collected: "SIZE_FORMAT, collection_set->garbage()); + tty->print_cr("objects to be evacuated: "SIZE_FORMAT, collection_set->live_data()); + */ + _max_live_data = MAX2(_max_live_data, collection_set->live_data()); + } + + void set_used_threshold(uintx used_threshold) { + this->_used_threshold_factor = get_percent(used_threshold); + this->_used_threshold = used_threshold; + } + + void set_garbage_threshold(uintx garbage_threshold) { + this->_garbage_threshold_factor = get_percent(garbage_threshold); + this->_garbage_threshold = garbage_threshold; + } + + void set_allocation_threshold(uintx allocationThreshold) { + this->_allocation_threshold_factor = get_percent(allocationThreshold); + this->_allocation_threshold = allocationThreshold; + } + + uintx get_allocation_threshold() { + return this->_allocation_threshold; + } + + uintx get_garbage_threshold() { + return this->_garbage_threshold; + } + + uintx get_used_threshold() { + return this->_used_threshold; + } +}; + +class NewAdaptiveHeuristics : public ShenandoahHeuristics { +private: + size_t _max_live_data; + double _target_heap_occupancy_factor; + double _allocation_threshold_factor; + size_t _last_bytesAllocSinceCM; + + uintx _target_heap_occupancy; + uintx _allocation_threshold; + +public: + NewAdaptiveHeuristics() : ShenandoahHeuristics() + { + if (PrintGCDetails) { + tty->print_cr("Initializing newadaptive heuristics"); + } + _max_live_data = 0; + _allocation_threshold = 0; + _target_heap_occupancy_factor = 0.; + _allocation_threshold_factor = 0.; + _last_bytesAllocSinceCM = 0; + } + + virtual ~NewAdaptiveHeuristics() {} + + virtual bool should_start_concurrent_mark(size_t used, size_t capacity) const + { + if (this->_bytes_allocated_during_CM > 0) { + // Not the first concurrent mark. + // _bytes_allocated_during_CM + ShenandoahHeap *heap = ShenandoahHeap::heap(); + size_t threshold_bytes_allocated = heap->capacity() / 4; + size_t targetStartMarking = (size_t) capacity * this->_target_heap_occupancy_factor; + return (used > targetStartMarking) && (this->_bytes_allocated_during_CM > threshold_bytes_allocated); + } else { + // First concurrent mark. + size_t targetStartMarking = capacity / 2; + ShenandoahHeap *heap = ShenandoahHeap::heap(); + size_t threshold_bytes_allocated = heap->capacity() / 4; + + // Need to check that an appropriate number of regions have + // been allocated since last concurrent mark too. + return (used > targetStartMarking) && (heap->_bytesAllocSinceCM > threshold_bytes_allocated); + } + } + + virtual void choose_collection_and_free_sets(ShenandoahHeapRegionSet* region_set, + ShenandoahHeapRegionSet* collection_set, + ShenandoahHeapRegionSet* free_set) + { + ShenandoahHeap *_heap = ShenandoahHeap::heap(); + this->_last_bytesAllocSinceCM = ShenandoahHeap::heap()->_bytesAllocSinceCM; + if (this->_last_bytesAllocSinceCM > 0) { + size_t min_garbage = this->_last_bytesAllocSinceCM; + region_set->choose_collection_and_free_sets_min_garbage(collection_set, free_set, min_garbage); + } else { + region_set->set_garbage_threshold(ShenandoahHeapRegion::RegionSizeBytes / 2); + region_set->choose_collection_and_free_sets(collection_set, free_set); + } + this->_max_live_data = MAX2(this->_max_live_data, collection_set->live_data()); + } + + void set_target_heap_occupancy(uintx target_heap_occupancy) { + this->_target_heap_occupancy_factor = get_percent(target_heap_occupancy); + this->_target_heap_occupancy = target_heap_occupancy; + } + + void set_allocation_threshold(uintx allocationThreshold) { + this->_allocation_threshold_factor = get_percent(allocationThreshold); + this->_allocation_threshold = allocationThreshold; + } + + uintx get_allocation_threshold() { + return this->_allocation_threshold; + } + + uintx get_target_heap_occupancy() { + return this->_target_heap_occupancy; + } +}; + + +static DynamicHeuristics *configureDynamicHeuristics() { + DynamicHeuristics *heuristics = new DynamicHeuristics(); + + heuristics->set_garbage_threshold(ShenandoahGarbageThreshold); + heuristics->set_allocation_threshold(ShenandoahAllocationThreshold); + heuristics->set_free_threshold(ShenandoahFreeThreshold); + if (ShenandoahLogConfig) { + tty->print_cr("Shenandoah dynamic heuristics thresholds: allocation "SIZE_FORMAT", used "SIZE_FORMAT", garbage "SIZE_FORMAT, + heuristics->get_allocation_threshold(), + heuristics->get_free_threshold(), + heuristics->get_garbage_threshold()); + } + return heuristics; +} + + +static NewAdaptiveHeuristics* configureNewAdaptiveHeuristics() { + NewAdaptiveHeuristics* heuristics = new NewAdaptiveHeuristics(); + + heuristics->set_target_heap_occupancy(ShenandoahTargetHeapOccupancy); + if (ShenandoahLogConfig) { + tty->print_cr( "Shenandoah newadaptive heuristics target heap occupancy: "SIZE_FORMAT, + heuristics->get_target_heap_occupancy() ); + } + return heuristics; +} + + +ShenandoahCollectorPolicy::ShenandoahCollectorPolicy() { + + ShenandoahHeapRegion::setup_heap_region_size(initial_heap_byte_size(), initial_heap_byte_size()); + + initialize_all(); + + _tracer = new (ResourceObj::C_HEAP, mtGC) ShenandoahTracer(); + _stw_timer = new (ResourceObj::C_HEAP, mtGC) STWGCTimer(); + _conc_timer = new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer(); + _user_requested_gcs = 0; + _allocation_failure_gcs = 0; + _conc_gc_aborted = false; + + _phase_names[init_mark] = "InitMark"; + _phase_names[final_mark] = "FinalMark"; + _phase_names[rescan_roots] = "RescanRoots"; + _phase_names[drain_satb] = "DrainSATB"; + _phase_names[drain_queues] = "DrainQueues"; + _phase_names[weakrefs] = "WeakRefs"; + _phase_names[prepare_evac] = "PrepareEvac"; + _phase_names[init_evac] = "InitEvac"; + _phase_names[final_evac] = "FinalEvacuation"; + _phase_names[final_uprefs] = "FinalUpdateRefs"; + + _phase_names[update_roots] = "UpdateRoots"; + _phase_names[recycle_regions] = "RecycleRegions"; + _phase_names[reset_bitmaps] = "ResetBitmaps"; + _phase_names[resize_tlabs] = "ResizeTLABs"; + + _phase_names[full_gc] = "FullGC"; + _phase_names[conc_mark] = "ConcurrentMark"; + _phase_names[conc_evac] = "ConcurrentEvacuation"; + _phase_names[conc_uprefs] = "ConcurrentUpdateReferences"; + + if (ShenandoahGCHeuristics != NULL) { + if (strcmp(ShenandoahGCHeuristics, "aggressive") == 0) { + if (ShenandoahLogConfig) { + tty->print_cr("Shenandoah heuristics: aggressive"); + } + _heuristics = new AggressiveHeuristics(); + } else if (strcmp(ShenandoahGCHeuristics, "statusquo") == 0) { + if (ShenandoahLogConfig) { + tty->print_cr("Shenandoah heuristics: statusquo"); + } + _heuristics = new StatusQuoHeuristics(); + } else if (strcmp(ShenandoahGCHeuristics, "halfway") == 0) { + if (ShenandoahLogConfig) { + tty->print_cr("Shenandoah heuristics: halfway"); + } + _heuristics = new HalfwayHeuristics(); + } else if (strcmp(ShenandoahGCHeuristics, "lazy") == 0) { + if (ShenandoahLogConfig) { + tty->print_cr("Shenandoah heuristics: lazy"); + } + _heuristics = new LazyHeuristics(); + } else if (strcmp(ShenandoahGCHeuristics, "dynamic") == 0) { + if (ShenandoahLogConfig) { + tty->print_cr("Shenandoah heuristics: dynamic"); + } + _heuristics = configureDynamicHeuristics(); + } else if (strcmp(ShenandoahGCHeuristics, "adaptive") == 0) { + if (ShenandoahLogConfig) { + tty->print_cr("Shenandoah heuristics: adaptive"); + } + _heuristics = new AdaptiveHeuristics(); + } else if (strcmp(ShenandoahGCHeuristics, "newadaptive") == 0) { + if (ShenandoahLogConfig) { + tty->print_cr("Shenandoah heuristics: newadaptive"); + } + _heuristics = configureNewAdaptiveHeuristics(); + } else { + fatal("Unknown -XX:ShenandoahGCHeuristics option"); + } + } else { + if (ShenandoahLogConfig) { + tty->print_cr("Shenandoah heuristics: statusquo (default)"); + } + _heuristics = new StatusQuoHeuristics(); + } + +} + +ShenandoahCollectorPolicy* ShenandoahCollectorPolicy::as_pgc_policy() { + return this; +} + +BarrierSet::Name ShenandoahCollectorPolicy::barrier_set_name() { + return BarrierSet::ShenandoahBarrierSet; +} + +HeapWord* ShenandoahCollectorPolicy::mem_allocate_work(size_t size, + bool is_tlab, + bool* gc_overhead_limit_was_exceeded) { + guarantee(false, "Not using this policy feature yet."); + return NULL; +} + +HeapWord* ShenandoahCollectorPolicy::satisfy_failed_allocation(size_t size, bool is_tlab) { + guarantee(false, "Not using this policy feature yet."); + return NULL; +} + +void ShenandoahCollectorPolicy::initialize_alignments() { + + // This is expected by our algorithm for ShenandoahHeap::heap_region_containing(). + _space_alignment = ShenandoahHeapRegion::RegionSizeBytes; + _heap_alignment = ShenandoahHeapRegion::RegionSizeBytes; +} + +void ShenandoahCollectorPolicy::post_heap_initialize() { + // Nothing to do here (yet). +} + +void ShenandoahCollectorPolicy::record_bytes_allocated(size_t bytes) { + _heuristics->record_bytes_allocated(bytes); +} + +void ShenandoahCollectorPolicy::record_bytes_start_CM(size_t bytes) { + _heuristics->record_bytes_start_CM(bytes); +} + +void ShenandoahCollectorPolicy::record_bytes_end_CM(size_t bytes) { + _heuristics->record_bytes_end_CM(bytes); +} + +void ShenandoahCollectorPolicy::record_bytes_reclaimed(size_t bytes) { + _heuristics->record_bytes_reclaimed(bytes); +} + +void ShenandoahCollectorPolicy::record_user_requested_gc() { + _user_requested_gcs++; +} + +void ShenandoahCollectorPolicy::record_allocation_failure_gc() { + _allocation_failure_gcs++; +} + +bool ShenandoahCollectorPolicy::should_start_concurrent_mark(size_t used, + size_t capacity) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + return _heuristics->should_start_concurrent_mark(used, capacity); +} + +bool ShenandoahCollectorPolicy::update_refs_early() { + return _heuristics->update_refs_early(); +} + +void ShenandoahCollectorPolicy::choose_collection_and_free_sets( + ShenandoahHeapRegionSet* region_set, + ShenandoahHeapRegionSet* collection_set, + ShenandoahHeapRegionSet* free_set) { + _heuristics->choose_collection_and_free_sets(region_set, collection_set, free_set); +} + +void ShenandoahCollectorPolicy::print_tracing_info() { + print_summary_sd("Initial Mark Pauses", 0, &(_timing_data[init_mark]._ms)); + print_summary_sd("Final Mark Pauses", 0, &(_timing_data[final_mark]._ms)); + + print_summary_sd("Rescan Roots", 2, &(_timing_data[rescan_roots]._ms)); + print_summary_sd("Drain SATB", 2, &(_timing_data[drain_satb]._ms)); + print_summary_sd("Drain Queues", 2, &(_timing_data[drain_queues]._ms)); + if (ShenandoahProcessReferences) { + print_summary_sd("Weak References", 2, &(_timing_data[weakrefs]._ms)); + } + print_summary_sd("Prepare Evacuation", 2, &(_timing_data[prepare_evac]._ms)); + print_summary_sd("Initial Evacuation", 2, &(_timing_data[init_evac]._ms)); + + print_summary_sd("Final Evacuation Pauses", 0, &(_timing_data[final_evac]._ms)); + print_summary_sd("Final Update Refs Pauses", 0, &(_timing_data[final_uprefs]._ms)); + print_summary_sd("Update roots", 2, &(_timing_data[update_roots]._ms)); + print_summary_sd("Recycle regions", 2, &(_timing_data[recycle_regions]._ms)); + print_summary_sd("Reset bitmaps", 2, &(_timing_data[reset_bitmaps]._ms)); + print_summary_sd("Resize TLABs", 2, &(_timing_data[resize_tlabs]._ms)); + gclog_or_tty->print_cr(" "); + print_summary_sd("Concurrent Marking Times", 0, &(_timing_data[conc_mark]._ms)); + print_summary_sd("Concurrent Evacuation Times", 0, &(_timing_data[conc_evac]._ms)); + print_summary_sd("Concurrent Update References Times", 0, &(_timing_data[conc_uprefs]._ms)); + print_summary_sd("Full GC Times", 0, &(_timing_data[full_gc]._ms)); + + gclog_or_tty->print_cr("User requested GCs: "SIZE_FORMAT, _user_requested_gcs); + gclog_or_tty->print_cr("Allocation failure GCs: "SIZE_FORMAT, _allocation_failure_gcs); + + gclog_or_tty->print_cr(" "); + double total_sum = _timing_data[init_mark]._ms.sum() + + _timing_data[final_mark]._ms.sum() + + _timing_data[final_evac]._ms.sum() + + _timing_data[final_uprefs]._ms.sum(); + double total_avg = (_timing_data[init_mark]._ms.avg() + + _timing_data[final_mark]._ms.avg() + + _timing_data[final_evac]._ms.avg() + + _timing_data[final_uprefs]._ms.avg()) / 4.0; + double total_max = MAX2( + MAX2( + MAX2(_timing_data[init_mark]._ms.maximum(), + _timing_data[final_mark]._ms.maximum()), + _timing_data[final_evac]._ms.maximum()), + _timing_data[final_uprefs]._ms.maximum()); + + gclog_or_tty->print_cr("%-27s = %8.2lf s, avg = %8.2lf ms, max = %8.2lf ms", + "Total", total_sum / 1000.0, total_avg, total_max); + +} + +void ShenandoahCollectorPolicy::print_summary_sd(const char* str, uint indent, const NumberSeq* seq) { + double sum = seq->sum(); + for (uint i = 0; i < indent; i++) gclog_or_tty->print(" "); + gclog_or_tty->print_cr("%-27s = %8.2lf s (avg = %8.2lf ms)", + str, sum / 1000.0, seq->avg()); + for (uint i = 0; i < indent; i++) gclog_or_tty->print(" "); + gclog_or_tty->print_cr("%s = "INT32_FORMAT_W(5)", std dev = %8.2lf ms, max = %8.2lf ms)", + "(num", seq->num(), seq->sd(), seq->maximum()); +} +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/shenandoahCollectorPolicy.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,117 @@ +/* + Copyright 2014 Red Hat, Inc. and/or its affiliates. +*/ +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAH_COLLECTOR_POLICY_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAH_COLLECTOR_POLICY_HPP + +#include "gc/shenandoah/shenandoahHeapRegion.hpp" +#include "gc/shenandoah/shenandoahHeapRegionSet.hpp" +#include "gc/shared/gcTrace.hpp" +#include "gc/shared/gcTimer.hpp" +#include "gc/shared/collectorPolicy.hpp" +#include "runtime/arguments.hpp" +#include "utilities/numberSeq.hpp" + + +class ShenandoahHeap; +class ShenandoahHeuristics; + +class ShenandoahCollectorPolicy: public CollectorPolicy { + +public: + enum TimingPhase { + init_mark, + final_mark, + rescan_roots, + drain_satb, + drain_queues, + weakrefs, + prepare_evac, + init_evac, + + final_evac, + final_uprefs, + update_roots, + recycle_regions, + reset_bitmaps, + resize_tlabs, + full_gc, + conc_mark, + conc_evac, + conc_uprefs, + + _num_phases + }; + +private: + struct TimingData { + NumberSeq _ms; + double _start; + size_t _count; + }; + +private: + TimingData _timing_data[_num_phases]; + const char* _phase_names[_num_phases]; + + size_t _user_requested_gcs; + size_t _allocation_failure_gcs; + + ShenandoahHeap* _pgc; + ShenandoahHeuristics* _heuristics; + ShenandoahTracer* _tracer; + STWGCTimer* _stw_timer; + ConcurrentGCTimer* _conc_timer; + + bool _conc_gc_aborted; + +public: + ShenandoahCollectorPolicy(); + + virtual ShenandoahCollectorPolicy* as_pgc_policy(); + + BarrierSet::Name barrier_set_name(); + + HeapWord* mem_allocate_work(size_t size, + bool is_tlab, + bool* gc_overhead_limit_was_exceeded); + + HeapWord* satisfy_failed_allocation(size_t size, bool is_tlab); + + void initialize_alignments(); + + void post_heap_initialize(); + + void record_phase_start(TimingPhase phase); + void record_phase_end(TimingPhase phase); + void report_concgc_cancelled(); + + void record_user_requested_gc(); + void record_allocation_failure_gc(); + + void record_bytes_allocated(size_t bytes); + void record_bytes_reclaimed(size_t bytes); + void record_bytes_start_CM(size_t bytes); + void record_bytes_end_CM(size_t bytes); + bool should_start_concurrent_mark(size_t used, size_t capacity); + void choose_collection_and_free_sets(ShenandoahHeapRegionSet* region_set, + ShenandoahHeapRegionSet* collection_set, + ShenandoahHeapRegionSet* free_set); + + bool update_refs_early(); + + void print_tracing_info(); + + GCTimer* conc_timer(){return _conc_timer;} + GCTimer* stw_timer() {return _stw_timer;} + ShenandoahTracer* tracer() {return _tracer;} + + void set_conc_gc_aborted() { _conc_gc_aborted = true;} + void clear_conc_gc_aborted() {_conc_gc_aborted = false;} + +private: + void print_summary_sd(const char* str, uint indent, const NumberSeq* seq); +}; + + +#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAH_COLLECTOR_POLICY_HPP
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/shenandoahConcurrentMark.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,742 @@ +/* + Copyright 2014 Red Hat, Inc. and/or its affiliates. +*/ +/* + * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "classfile/stringTable.hpp" +#include "gc/shared/gcTimer.hpp" +#include "gc/shared/isGCActiveMark.hpp" +#include "gc/shared/strongRootsScope.hpp" +#include "gc/shenandoah/shenandoahBarrierSet.hpp" +#include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahRootProcessor.hpp" +#include "gc/shenandoah/brooksPointer.hpp" +#include "gc/shared/referenceProcessor.hpp" +#include "code/codeCache.hpp" +#include "classfile/symbolTable.hpp" +#include "classfile/systemDictionary.hpp" +#include "memory/iterator.inline.hpp" +#include "oops/oop.inline.hpp" +#include "gc/shared/taskqueue.inline.hpp" + +// Mark the object and add it to the queue to be scanned +ShenandoahMarkObjsClosure::ShenandoahMarkObjsClosure(SCMObjToScanQueue* q, bool update_refs) : + _heap((ShenandoahHeap*)(Universe::heap())), + _mark_refs(ShenandoahMarkRefsClosure(q, update_refs)), + _live_data(NEW_C_HEAP_ARRAY(size_t, _heap->max_regions(), mtGC)) +{ + Copy::zero_to_bytes(_live_data, _heap->max_regions() * sizeof(size_t)); +} + +ShenandoahMarkObjsClosure::~ShenandoahMarkObjsClosure() { + // Merge liveness data back into actual regions. + + // We need to lock the heap here, to avoid race with growing of heap. + MutexLockerEx ml(ShenandoahHeap_lock, true); + ShenandoahHeapRegion** regions = _heap->heap_regions(); + for (uint i = 0; i < _heap->num_regions(); i++) { + regions[i]->increase_live_data(_live_data[i]); + } + FREE_C_HEAP_ARRAY(size_t, _live_data); +} + +ShenandoahMarkRefsClosure::ShenandoahMarkRefsClosure(SCMObjToScanQueue* q, bool update_refs) : + MetadataAwareOopClosure(((ShenandoahHeap *) Universe::heap())->ref_processor()), + _queue(q), + _heap((ShenandoahHeap*) Universe::heap()), + _scm(_heap->concurrentMark()), + _update_refs(update_refs) +{ +} + +void ShenandoahMarkRefsClosure::do_oop(narrowOop* p) { + Unimplemented(); +} + + +// Walks over all the objects in the generation updating any +// references to from space. + +class CLDMarkAliveClosure : public CLDClosure { +private: + CLDClosure* _cl; +public: + CLDMarkAliveClosure(CLDClosure* cl) : _cl(cl) { + } + void do_cld(ClassLoaderData* cld) { + ShenandoahIsAliveClosure is_alive; + if (cld->is_alive(&is_alive)) { + _cl->do_cld(cld); + } + } +}; + +class ShenandoahMarkRootsTask : public AbstractGangTask { +private: + ShenandoahRootProcessor* _rp; + bool _update_refs; +public: + ShenandoahMarkRootsTask(ShenandoahRootProcessor* rp, bool update_refs) : + AbstractGangTask("Shenandoah update roots task"), _update_refs(update_refs), + _rp(rp) { + } + + void work(uint worker_id) { + // tty->print_cr("start mark roots worker: "INT32_FORMAT, worker_id); + ShenandoahHeap* heap = ShenandoahHeap::heap(); + SCMObjToScanQueue* q = heap->concurrentMark()->get_queue(worker_id); + ShenandoahMarkRefsClosure cl(q, _update_refs); + + CodeBlobToOopClosure blobsCl(&cl, true); + CLDToOopClosure cldCl(&cl); + + ResourceMark m; + if (ShenandoahProcessReferences && ClassUnloadingWithConcurrentMark) { + _rp->process_strong_roots(&cl, &cldCl, &blobsCl); + } else { + _rp->process_all_roots(&cl, &cldCl, &blobsCl); + } + // tty->print_cr("finish mark roots worker: "INT32_FORMAT, worker_id); + } +}; + +class SCMConcurrentMarkingTask : public AbstractGangTask { +private: + ShenandoahConcurrentMark* _cm; + ParallelTaskTerminator* _terminator; + int _seed; + bool _update_refs; + +public: + SCMConcurrentMarkingTask(ShenandoahConcurrentMark* cm, ParallelTaskTerminator* terminator, bool update_refs) : + AbstractGangTask("Root Region Scan"), _cm(cm), _terminator(terminator), _update_refs(update_refs), _seed(17) { + } + + + void work(uint worker_id) { + + SCMObjToScanQueue* q = _cm->get_queue(worker_id); + ShenandoahMarkObjsClosure cl(q, _update_refs); + ShenandoahHeap* heap = ShenandoahHeap::heap(); + while (true) { + if (heap->cancelled_concgc() || + (!_cm->try_queue(q, &cl) && + !_cm->try_draining_an_satb_buffer(worker_id) && + !_cm->try_to_steal(worker_id, &cl, &_seed)) + ) { + if (_terminator->offer_termination()) break; + } + } + if (ShenandoahTracePhases && heap->cancelled_concgc()) { + tty->print_cr("Cancelled concurrent marking"); + } + } +}; + +void ShenandoahConcurrentMark::prepare_unmarked_root_objs() { + + ShenandoahHeap* heap = ShenandoahHeap::heap(); + bool update_refs = heap->need_update_refs(); + + if (update_refs) { + COMPILER2_PRESENT(DerivedPointerTable::clear()); + } + + prepare_unmarked_root_objs_no_derived_ptrs(update_refs); + + if (update_refs) { + COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); + } + +} + +void ShenandoahConcurrentMark::prepare_unmarked_root_objs_no_derived_ptrs(bool update_refs) { + assert(Thread::current()->is_VM_thread(), "can only do this in VMThread"); + + ShenandoahHeap* heap = ShenandoahHeap::heap(); + if (ShenandoahParallelRootScan) { + + ClassLoaderDataGraph::clear_claimed_marks(); + heap->conc_workers()->set_active_workers(_max_conc_worker_id); + ShenandoahRootProcessor root_proc(heap, _max_conc_worker_id); + TASKQUEUE_STATS_ONLY(reset_taskqueue_stats()); + ShenandoahMarkRootsTask mark_roots(&root_proc, update_refs); + heap->conc_workers()->run_task(&mark_roots); + + // Mark through any class loaders that have been found alive. + ShenandoahMarkRefsClosure cl(get_queue(0), update_refs); + CLDToOopClosure cldCl(&cl); + CLDMarkAliveClosure cld_keep_alive(&cldCl); + ClassLoaderDataGraph::roots_cld_do(NULL, &cld_keep_alive); + + } else { + ShenandoahMarkRefsClosure cl(get_queue(0), update_refs); + heap->roots_iterate(&cl); + } + + if (!(ShenandoahProcessReferences && ClassUnloadingWithConcurrentMark)) { + ShenandoahMarkRefsClosure cl(get_queue(0), update_refs); + heap->weak_roots_iterate(&cl); + } + + // tty->print_cr("all root marker threads done"); +} + + +void ShenandoahConcurrentMark::initialize() { + _max_conc_worker_id = MAX2((uint) ConcGCThreads, 1U); + _task_queues = new SCMObjToScanQueueSet((int) _max_conc_worker_id); + + for (uint i = 0; i < _max_conc_worker_id; ++i) { + SCMObjToScanQueue* task_queue = new SCMObjToScanQueue(); + task_queue->initialize(); + _task_queues->register_queue(i, task_queue); + } + JavaThread::satb_mark_queue_set().set_buffer_size(1014 /* G1SATBBufferSize */); +} + +void ShenandoahConcurrentMark::mark_from_roots() { + if (ShenandoahGCVerbose) { + tty->print_cr("STOPPING THE WORLD: before marking"); + tty->print_cr("Starting markFromRoots"); + } + + ShenandoahHeap* sh = (ShenandoahHeap *) Universe::heap(); + + bool update_refs = sh->need_update_refs(); + + sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::conc_mark); + ParallelTaskTerminator terminator(_max_conc_worker_id, _task_queues); + + if (ShenandoahProcessReferences) { + ReferenceProcessor* rp = sh->ref_processor(); + // enable ("weak") refs discovery + rp->enable_discovery(true /*verify_no_refs*/); + rp->setup_policy(false); // snapshot the soft ref policy to be used in this cycle + } + + SCMConcurrentMarkingTask markingTask = SCMConcurrentMarkingTask(this, &terminator, update_refs); + sh->conc_workers()->set_active_workers(_max_conc_worker_id); + sh->conc_workers()->run_task(&markingTask); + + if (ShenandoahGCVerbose) { + tty->print("total workers = %u active workers = %u\n", + sh->conc_workers()->total_workers(), + sh->conc_workers()->active_workers()); + TASKQUEUE_STATS_ONLY(print_taskqueue_stats()); + TASKQUEUE_STATS_ONLY(reset_taskqueue_stats()); + } + + if (ShenandoahGCVerbose) { + tty->print_cr("Finishing markFromRoots"); + tty->print_cr("RESUMING THE WORLD: after marking"); + } + + sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::conc_mark); +} + +class FinishDrainSATBBuffersTask : public AbstractGangTask { +private: + ShenandoahConcurrentMark* _cm; + ParallelTaskTerminator* _terminator; +public: + FinishDrainSATBBuffersTask(ShenandoahConcurrentMark* cm, ParallelTaskTerminator* terminator) : + AbstractGangTask("Finish draining SATB buffers"), _cm(cm), _terminator(terminator) { + } + + void work(uint worker_id) { + _cm->drain_satb_buffers(worker_id, true); + } +}; + +class ShenandoahUpdateAliveRefs : public OopClosure { +private: + ShenandoahHeap* _heap; +public: + ShenandoahUpdateAliveRefs() : _heap(ShenandoahHeap::heap()) { + } + virtual void do_oop(oop* p) { + _heap->maybe_update_oop_ref(p); + } + + virtual void do_oop(narrowOop* p) { + Unimplemented(); + } +}; + +void ShenandoahConcurrentMark::finish_mark_from_roots() { + if (ShenandoahGCVerbose) { + tty->print_cr("Starting finishMarkFromRoots"); + } + + IsGCActiveMark is_active; + + ShenandoahHeap* sh = (ShenandoahHeap *) Universe::heap(); + + // Trace any (new) unmarked root references. + sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::rescan_roots); + prepare_unmarked_root_objs(); + sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::rescan_roots); + sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::drain_satb); + { + StrongRootsScope scope(_max_conc_worker_id); + ParallelTaskTerminator terminator(_max_conc_worker_id, _task_queues); + // drain_satb_buffers(0, true); + FinishDrainSATBBuffersTask drain_satb_buffers(this, &terminator); + sh->conc_workers()->set_active_workers(_max_conc_worker_id); + sh->conc_workers()->run_task(&drain_satb_buffers); + sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::drain_satb); + } + + // Finally mark everything else we've got in our queues during the previous steps. + { + sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::drain_queues); + ParallelTaskTerminator terminator(_max_conc_worker_id, _task_queues); + SCMConcurrentMarkingTask markingTask = SCMConcurrentMarkingTask(this, &terminator, sh->need_update_refs()); + sh->conc_workers()->set_active_workers(_max_conc_worker_id); + sh->conc_workers()->run_task(&markingTask); + sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::drain_queues); + } + +#ifdef ASSERT + for (int i = 0; i < (int) _max_conc_worker_id; i++) { + assert(_task_queues->queue(i)->is_empty(), "Should be empty"); + } +#endif + + // When we're done marking everything, we process weak references. + if (ShenandoahProcessReferences) { + sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::weakrefs); + weak_refs_work(); + sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::weakrefs); + } + +#ifdef ASSERT + for (int i = 0; i < (int) _max_conc_worker_id; i++) { + assert(_task_queues->queue(i)->is_empty(), "Should be empty"); + } +#endif + + if (ShenandoahGCVerbose) { + tty->print_cr("Finishing finishMarkFromRoots"); +#ifdef SLOWDEBUG + for (int i = 0; i <(int)_max_conc_worker_id; i++) { + tty->print("Queue: "INT32_FORMAT":", i); + _task_queues->queue(i)->stats.print(tty, 10); + tty->cr(); + _task_queues->queue(i)->stats.verify(); + } +#endif + } + + // We still need to update (without marking) alive refs in JNI handles. + if (ShenandoahProcessReferences && ClassUnloadingWithConcurrentMark) { + ShenandoahUpdateAliveRefs cl; + ShenandoahIsAliveClosure is_alive; + JNIHandles::weak_oops_do(&is_alive, &cl); + } + +#ifdef ASSERT + verify_roots(); + + if (ShenandoahDumpHeapAfterConcurrentMark) { + sh->ensure_parsability(false); + sh->print_all_refs("post-mark"); + } +#endif +} + +#ifdef ASSERT +void ShenandoahVerifyRootsClosure1::do_oop(oop* p) { + oop obj = oopDesc::load_heap_oop(p); + if (! oopDesc::is_null(obj)) { + guarantee(ShenandoahHeap::heap()->is_marked_current(obj), "oop must be marked"); + guarantee(obj == ShenandoahBarrierSet::resolve_oop_static_not_null(obj), "oop must not be forwarded"); + } +} + +void ShenandoahConcurrentMark::verify_roots() { + ShenandoahVerifyRootsClosure1 cl; + CodeBlobToOopClosure blobsCl(&cl, true); + CLDToOopClosure cldCl(&cl); + ClassLoaderDataGraph::clear_claimed_marks(); + ShenandoahRootProcessor rp(ShenandoahHeap::heap(), 1); + rp.process_roots(&cl, &cl, &cldCl, &cldCl, &cldCl, &blobsCl); +} +#endif + +class ShenandoahSATBBufferClosure : public SATBBufferClosure { +private: + SCMObjToScanQueue* _queue; + +public: + ShenandoahSATBBufferClosure(SCMObjToScanQueue* q) : + _queue(q) + { + } + + void do_buffer(void** buffer, size_t size) { + // tty->print_cr("draining one satb buffer"); + for (size_t i = 0; i < size; ++i) { + void* entry = buffer[i]; + oop obj = oop(entry); + // tty->print_cr("satb buffer entry: "PTR_FORMAT, p2i((HeapWord*) obj)); + if (!oopDesc::is_null(obj)) { + obj = ShenandoahBarrierSet::resolve_oop_static_not_null(obj); + bool pushed = _queue->push(obj); + assert(pushed, "overflow queue should always succeed pushing"); + } + } + } +}; + +class ShenandoahSATBThreadsClosure : public ThreadClosure { + ShenandoahSATBBufferClosure* _satb_cl; + int _thread_parity; + + public: + ShenandoahSATBThreadsClosure(ShenandoahSATBBufferClosure* satb_cl) : + _satb_cl(satb_cl), + _thread_parity(Threads::thread_claim_parity()) {} + + void do_thread(Thread* thread) { + if (thread->is_Java_thread()) { + if (thread->claim_oops_do(true, _thread_parity)) { + JavaThread* jt = (JavaThread*)thread; + jt->satb_mark_queue().apply_closure_and_empty(_satb_cl); + } + } else if (thread->is_VM_thread()) { + if (thread->claim_oops_do(true, _thread_parity)) { + JavaThread::satb_mark_queue_set().shared_satb_queue()->apply_closure_and_empty(_satb_cl); + } + } + } +}; + +void ShenandoahConcurrentMark::drain_satb_buffers(uint worker_id, bool remark) { + + // tty->print_cr("start draining SATB buffers"); + + ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap(); + SCMObjToScanQueue* q = get_queue(worker_id); + ShenandoahSATBBufferClosure cl(q); + + SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); + while (satb_mq_set.apply_closure_to_completed_buffer(&cl)); + + if (remark) { + ShenandoahSATBThreadsClosure tc(&cl); + Threads::threads_do(&tc); + } + + // tty->print_cr("end draining SATB buffers"); + +} + +bool ShenandoahConcurrentMark::drain_one_satb_buffer(uint worker_id) { + + ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap(); + SCMObjToScanQueue* q = get_queue(worker_id); + ShenandoahSATBBufferClosure cl(q); + + SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); + bool result = satb_mq_set.apply_closure_to_completed_buffer(&cl); + return result; +} + +#if TASKQUEUE_STATS +void ShenandoahConcurrentMark::print_taskqueue_stats_hdr(outputStream* const st) { + st->print_raw_cr("GC Task Stats"); + st->print_raw("thr "); TaskQueueStats::print_header(1, st); st->cr(); + st->print_raw("--- "); TaskQueueStats::print_header(2, st); st->cr(); +} + +void ShenandoahConcurrentMark::print_taskqueue_stats(outputStream* const st) const { + print_taskqueue_stats_hdr(st); + ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap(); + TaskQueueStats totals; + const int n = sh->max_conc_workers(); + for (int i = 0; i < n; ++i) { + st->print(INT32_FORMAT_W(3), i); + _task_queues->queue(i)->stats.print(st); + st->print("\n"); + totals += _task_queues->queue(i)->stats; + } + st->print_raw("tot "); totals.print(st); st->cr(); + DEBUG_ONLY(totals.verify()); + +} + +void ShenandoahConcurrentMark::print_push_only_taskqueue_stats(outputStream* const st) const { + print_taskqueue_stats_hdr(st); + ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap(); + TaskQueueStats totals; + const int n = sh->max_conc_workers(); + for (int i = 0; i < n; ++i) { + st->print(INT32_FORMAT_W(3), i); + _task_queues->queue(i)->stats.print(st); + st->print("\n"); + totals += _task_queues->queue(i)->stats; + } + st->print_raw("tot "); totals.print(st); st->cr(); + + DEBUG_ONLY(totals.verify_only_pushes()); +} + +void ShenandoahConcurrentMark::reset_taskqueue_stats() { + ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap(); + const int n = sh->max_conc_workers(); + for (int i = 0; i < n; ++i) { + _task_queues->queue(i)->stats.reset(); + } +} +#endif // TASKQUEUE_STATS + +// Weak Reference Closures +class ShenandoahCMDrainMarkingStackClosure: public VoidClosure { + ShenandoahHeap* _sh; + ShenandoahConcurrentMark* _scm; + uint _worker_id; + int _seed; + +public: + ShenandoahCMDrainMarkingStackClosure(uint worker_id): _worker_id(worker_id), _seed(17) { + _sh = (ShenandoahHeap*) Universe::heap(); + _scm = _sh->concurrentMark(); + } + + + void do_void() { + + SCMObjToScanQueue* q = _scm->get_queue(_worker_id); + ShenandoahMarkObjsClosure cl(q, _sh->need_update_refs()); + while (true) { + if (!_scm->try_queue(q, &cl) && + !_scm->try_draining_an_satb_buffer(_worker_id) && + !_scm->try_to_steal(_worker_id, &cl, &_seed)) { + break; + } + } + } +}; + + +class ShenandoahCMKeepAliveAndDrainClosure: public OopClosure { + SCMObjToScanQueue* _queue; + ShenandoahHeap* _sh; + ShenandoahConcurrentMark* _scm; + + size_t _ref_count; + +public: + ShenandoahCMKeepAliveAndDrainClosure(SCMObjToScanQueue* q) : + _queue(q) { + _sh = (ShenandoahHeap*) Universe::heap(); + _scm = _sh->concurrentMark(); + _ref_count = 0; + } + + virtual void do_oop(oop* p){ do_oop_work(p);} + virtual void do_oop(narrowOop* p) { + assert(false, "narrowOops Aren't implemented"); + } + + + void do_oop_work(oop* p) { + + oop obj; + if (_sh->need_update_refs()) { + obj = _sh->maybe_update_oop_ref(p); + } else { + obj = oopDesc::load_heap_oop(p); + } + + assert(obj == oopDesc::bs()->resolve_oop(obj), "only get updated oops in weak ref processing"); + + if (obj != NULL) { + if (Verbose && ShenandoahTraceWeakReferences) { + gclog_or_tty->print_cr("\twe're looking at location " + "*"PTR_FORMAT" = "PTR_FORMAT, + p2i(p), p2i((void*) obj)); + obj->print(); + } + bool pushed = _queue->push(obj); + assert(pushed, "overflow queue should always succeed pushing"); + + _ref_count++; + } + } + + size_t ref_count() { return _ref_count; } + +}; + +class ShenandoahRefProcTaskProxy : public AbstractGangTask { + +private: + AbstractRefProcTaskExecutor::ProcessTask& _proc_task; + +public: + + ShenandoahRefProcTaskProxy(AbstractRefProcTaskExecutor::ProcessTask& proc_task) : + AbstractGangTask("Process reference objects in parallel"), + _proc_task(proc_task) { + } + + void work(uint worker_id) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahIsAliveClosure is_alive; + ShenandoahCMKeepAliveAndDrainClosure keep_alive(heap->concurrentMark()->get_queue(worker_id)); + ShenandoahCMDrainMarkingStackClosure complete_gc(worker_id); + _proc_task.work(worker_id, is_alive, keep_alive, complete_gc); + } +}; + +class ShenandoahRefEnqueueTaskProxy : public AbstractGangTask { + +private: + AbstractRefProcTaskExecutor::EnqueueTask& _enqueue_task; + +public: + + ShenandoahRefEnqueueTaskProxy(AbstractRefProcTaskExecutor::EnqueueTask& enqueue_task) : + AbstractGangTask("Enqueue reference objects in parallel"), + _enqueue_task(enqueue_task) { + } + + void work(uint worker_id) { + _enqueue_task.work(worker_id); + } +}; + +class ShenandoahRefProcTaskExecutor : public AbstractRefProcTaskExecutor { + +private: + WorkGang* _workers; + +public: + + ShenandoahRefProcTaskExecutor() : _workers(ShenandoahHeap::heap()->conc_workers()) { + } + + // Executes a task using worker threads. + void execute(ProcessTask& task) { + ShenandoahRefProcTaskProxy proc_task_proxy(task); + _workers->run_task(&proc_task_proxy); + } + + void execute(EnqueueTask& task) { + ShenandoahRefEnqueueTaskProxy enqueue_task_proxy(task); + _workers->run_task(&enqueue_task_proxy); + } +}; + + +void ShenandoahConcurrentMark::weak_refs_work() { + ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap(); + ReferenceProcessor* rp = sh->ref_processor(); + + // Setup collector policy for softref cleaning. + bool clear_soft_refs = sh->collector_policy()->use_should_clear_all_soft_refs(true /* bogus arg*/); + if (ShenandoahTraceWeakReferences) { + tty->print_cr("clearing soft refs: %s", BOOL_TO_STR(clear_soft_refs)); + } + rp->setup_policy(clear_soft_refs); + + uint serial_worker_id = 0; + ShenandoahIsAliveClosure is_alive; + ShenandoahCMKeepAliveAndDrainClosure keep_alive(get_queue(serial_worker_id)); + ShenandoahCMDrainMarkingStackClosure complete_gc(serial_worker_id); + ShenandoahRefProcTaskExecutor par_task_executor; + bool processing_is_mt = true; + AbstractRefProcTaskExecutor* executor = (processing_is_mt ? &par_task_executor : NULL); + + if (ShenandoahTraceWeakReferences) { + gclog_or_tty->print_cr("start processing references"); + } + + rp->process_discovered_references(&is_alive, &keep_alive, + &complete_gc, &par_task_executor, + NULL, + ShenandoahHeap::heap()->tracer()->gc_id()); + + if (ShenandoahTraceWeakReferences) { + gclog_or_tty->print_cr("finished processing references, processed "SIZE_FORMAT" refs", keep_alive.ref_count()); + gclog_or_tty->print_cr("start enqueuing references"); + } + + rp->enqueue_discovered_references(executor); + + if (ShenandoahTraceWeakReferences) { + gclog_or_tty->print_cr("finished enqueueing references"); + } + + rp->verify_no_references_recorded(); + assert(!rp->discovery_enabled(), "Post condition"); + + if (ClassUnloadingWithConcurrentMark) { + // Unload classes and purge SystemDictionary. + bool purged_class = SystemDictionary::do_unloading(&is_alive); + // Unload nmethods. + CodeCache::do_unloading(&is_alive, purged_class); + // Prune dead klasses from subklass/sibling/implementor lists. + Klass::clean_weak_klass_links(&is_alive); + // Delete entries from dead interned strings. + StringTable::unlink(&is_alive); + // Clean up unreferenced symbols in symbol table. + SymbolTable::unlink(); + + ClassLoaderDataGraph::purge(); + } +} + +void ShenandoahConcurrentMark::cancel() { + ShenandoahHeap* sh = ShenandoahHeap::heap(); + + // Cancel weak-ref discovery. + if (ShenandoahProcessReferences) { + ReferenceProcessor* rp = sh->ref_processor(); + rp->abandon_partial_discovery(); + rp->disable_discovery(); + } + + // Clean up marking stacks. + SCMObjToScanQueueSet* queues = task_queues(); + for (uint i = 0; i < _max_conc_worker_id; ++i) { + SCMObjToScanQueue* task_queue = queues->queue(i); + task_queue->set_empty(); + task_queue->overflow_stack()->clear(); + } + + // Cancel SATB buffers. + JavaThread::satb_mark_queue_set().abandon_partial_marking(); +} + +SCMObjToScanQueue* ShenandoahConcurrentMark::get_queue(uint worker_id) { + worker_id = worker_id % _max_conc_worker_id; + return _task_queues->queue(worker_id); +} +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/shenandoahConcurrentMark.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,109 @@ +/* +Copyright 2014 Red Hat, Inc. and/or its affiliates. + */ + +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHCONCURRENTMARK_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHCONCURRENTMARK_HPP + +#include "gc/shared/taskqueue.hpp" +#include "gc/shared/workgroup.hpp" + +typedef OverflowTaskQueue<oop, mtGC> OopOverflowTaskQueue; +typedef Padded<OopOverflowTaskQueue> SCMObjToScanQueue; +typedef GenericTaskQueueSet<SCMObjToScanQueue, mtGC> SCMObjToScanQueueSet; + +class ShenandoahConcurrentMark; + +#ifdef ASSERT +class ShenandoahVerifyRootsClosure1 : public OopClosure { + void do_oop(oop* p); + + void do_oop(narrowOop* p) { + Unimplemented(); + } +}; +#endif + +class ShenandoahMarkRefsClosure : public MetadataAwareOopClosure { + SCMObjToScanQueue* _queue; + ShenandoahHeap* _heap; + bool _update_refs; + ShenandoahConcurrentMark* _scm; + +public: + ShenandoahMarkRefsClosure(SCMObjToScanQueue* q, bool update_refs); + + void do_oop(narrowOop* p); + + inline void do_oop(oop* p); + +}; + +class ShenandoahMarkObjsClosure : public ObjectClosure { + ShenandoahHeap* _heap; + size_t* _live_data; + ShenandoahMarkRefsClosure _mark_refs; + +public: + ShenandoahMarkObjsClosure(SCMObjToScanQueue* q, bool update_refs); + + ~ShenandoahMarkObjsClosure(); + + inline void do_object(oop obj); +}; + +class ShenandoahConcurrentMark: public CHeapObj<mtGC> { + +private: + // The per-worker-thread work queues + SCMObjToScanQueueSet* _task_queues; + + bool _aborted; + uint _max_conc_worker_id; + ParallelTaskTerminator* _terminator; + +public: + // We need to do this later when the heap is already created. + void initialize(); + + void mark_from_roots(); + + // Prepares unmarked root objects by marking them and putting + // them into the marking task queue. + void prepare_unmarked_root_objs(); + void prepare_unmarked_root_objs_no_derived_ptrs(bool update_refs); + + void finish_mark_from_roots(); + // Those are only needed public because they're called from closures. + + SCMObjToScanQueue* get_queue(uint worker_id); + inline bool try_queue(SCMObjToScanQueue* q, ShenandoahMarkObjsClosure* cl); + inline bool try_to_steal(uint worker_id, ShenandoahMarkObjsClosure* cl, int *seed); + inline bool try_draining_an_satb_buffer(uint worker_id); + void drain_satb_buffers(uint worker_id, bool remark = false); + SCMObjToScanQueueSet* task_queues() { return _task_queues;} + uint max_conc_worker_id() { return _max_conc_worker_id; } + + void cancel(); + +private: + +#ifdef ASSERT + void verify_roots(); +#endif + + bool drain_one_satb_buffer(uint worker_id); + void weak_refs_work(); + + ParallelTaskTerminator* terminator() { return _terminator;} + +#if TASKQUEUE_STATS + static void print_taskqueue_stats_hdr(outputStream* const st = gclog_or_tty); + void print_taskqueue_stats(outputStream* const st = gclog_or_tty) const; + void print_push_only_taskqueue_stats(outputStream* const st = gclog_or_tty) const; + void reset_taskqueue_stats(); +#endif // TASKQUEUE_STATS + +}; + +#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHCONCURRENTMARK_HPP
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/shenandoahConcurrentMark.inline.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,134 @@ +/* + Copyright 2015 Red Hat, Inc. and/or its affiliates. +*/ + +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHCONCURRENTMARK_INLINE_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHCONCURRENTMARK_INLINE_HPP + +#include "gc/shenandoah/brooksPointer.hpp" +#include "gc/shenandoah/shenandoahConcurrentMark.hpp" +#include "memory/iterator.inline.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/prefetch.inline.hpp" + +void ShenandoahMarkRefsClosure::do_oop(oop* p) { + // We piggy-back reference updating to the marking tasks. +#ifdef ASSERT + oop* old = p; +#endif + oop obj; + if (_update_refs) { + obj = _heap->maybe_update_oop_ref(p); + } else { + obj = oopDesc::load_heap_oop(p); + } + assert(obj == ShenandoahBarrierSet::resolve_oop_static(obj), "need to-space object here"); + +#ifdef ASSERT + if (ShenandoahTraceUpdates) { + if (p != old) + tty->print_cr("Update "PTR_FORMAT" => "PTR_FORMAT" to "PTR_FORMAT" => "PTR_FORMAT, p2i(p), p2i((HeapWord*) *p), p2i(old), p2i((HeapWord*) *old)); + else + tty->print_cr("Not updating "PTR_FORMAT" => "PTR_FORMAT" to "PTR_FORMAT" => "PTR_FORMAT, p2i(p), p2i((HeapWord*) *p), p2i(old), p2i((HeapWord*) *old)); + } +#endif + + // NOTE: We used to assert the following here. This does not always work because + // a concurrent Java thread could change the the field after we updated it. + // oop obj = oopDesc::load_heap_oop(p); + // assert(oopDesc::bs()->resolve_oop(obj) == *p, "we just updated the referrer"); + // assert(obj == NULL || ! _heap->heap_region_containing(obj)->is_dirty(), "must not point to dirty region"); + + // ShenandoahExtendedMarkObjsClosure cl(_heap->ref_processor(), _worker_id); + // ShenandoahMarkObjsClosure mocl(cl, _worker_id); + + if (obj != NULL) { + if (_update_refs) { + Prefetch::write(obj, 128); + } else { + Prefetch::read(obj, 128); + } + +#ifdef ASSERT + uint region_idx = _heap->heap_region_index_containing(obj); + ShenandoahHeapRegion* r = _heap->heap_regions()[region_idx]; + assert(r->bottom() < (HeapWord*) obj && r->top() > (HeapWord*) obj, "object must be in region"); +#endif + + bool pushed = _queue->push(obj); + assert(pushed, "overflow queue should always succeed pushing"); + } +} + +void ShenandoahMarkObjsClosure::do_object(oop obj) { + + assert(obj != NULL, "expect non-null object"); + + assert(obj == ShenandoahBarrierSet::resolve_oop_static_not_null(obj), "expect forwarded obj in queue"); + +#ifdef ASSERT + if (_heap->heap_region_containing(obj)->is_in_collection_set()) { + tty->print_cr("trying to mark obj: "PTR_FORMAT" (%s) in dirty region: ", p2i((HeapWord*) obj), BOOL_TO_STR(_heap->is_marked_current(obj))); + // _heap->heap_region_containing(obj)->print(); + // _heap->print_heap_regions(); + } +#endif + assert(_heap->cancelled_concgc() + || ! _heap->heap_region_containing(obj)->is_in_collection_set(), + "we don't want to mark objects in from-space"); + assert(_heap->is_in(obj), "referenced objects must be in the heap. No?"); + if (_heap->mark_current(obj)) { +#ifdef ASSERT + if (ShenandoahTraceConcurrentMarking) { + tty->print_cr("marked obj: "PTR_FORMAT, p2i((HeapWord*) obj)); + } +#endif + + // Calculate liveness of heap region containing object. + uint region_idx = _heap->heap_region_index_containing(obj); +#ifdef ASSERT + ShenandoahHeapRegion* r = _heap->heap_regions()[region_idx]; + assert(r->bottom() < (HeapWord*) obj && r->top() > (HeapWord*) obj, "object must be in region"); +#endif + _live_data[region_idx] += (obj->size() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE) * HeapWordSize; + obj->oop_iterate(&_mark_refs); + } + +#ifdef ASSERT + else { + if (ShenandoahTraceConcurrentMarking) { + tty->print_cr("failed to mark obj (already marked): "PTR_FORMAT, p2i((HeapWord*) obj)); + } + assert(_heap->is_marked_current(obj), "make sure object is marked"); + } +#endif +} + +inline bool ShenandoahConcurrentMark::try_queue(SCMObjToScanQueue* q, ShenandoahMarkObjsClosure* cl) { + oop obj; + if (q->pop_local(obj)) { + assert(obj != NULL, "Can't mark null"); + cl->do_object(obj); + return true; + } else if (q->pop_overflow(obj)) { + cl->do_object(obj); + return true; + } else { + return false; + } +} + +inline bool ShenandoahConcurrentMark::try_to_steal(uint worker_id, ShenandoahMarkObjsClosure* cl, int *seed) { + oop obj; + if (task_queues()->steal(worker_id, seed, obj)) { + cl->do_object(obj); + return true; + } else + return false; +} + +inline bool ShenandoahConcurrentMark:: try_draining_an_satb_buffer(uint worker_id) { + return drain_one_satb_buffer(worker_id); +} + +#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHCONCURRENTMARK_INLINE_HPP
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/shenandoahConcurrentThread.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,206 @@ +/* +Copyright 2014 Red Hat, Inc. and/or its affiliates. + */ + +#include "gc/shenandoah/shenandoahConcurrentThread.hpp" +#include "gc/shenandoah/shenandoahHeap.hpp" +#include "gc/shenandoah/shenandoahJNICritical.hpp" +#include "gc/shenandoah/vm_operations_shenandoah.hpp" +#include "memory/iterator.hpp" +#include "memory/universe.hpp" +#include "runtime/vmThread.hpp" + +SurrogateLockerThread* ShenandoahConcurrentThread::_slt = NULL; + +ShenandoahConcurrentThread::ShenandoahConcurrentThread() : + ConcurrentGCThread(), + _epoch(0), + _concurrent_mark_started(false), + _concurrent_mark_in_progress(false), + _do_full_gc(false), + _concurrent_mark_aborted(false) +{ + create_and_start(); +} + +ShenandoahConcurrentThread::~ShenandoahConcurrentThread() { + // This is here so that super is called. +} + +void ShenandoahConcurrentThread::run() { + initialize_in_thread(); + + wait_for_universe_init(); + + // Wait until we have the surrogate locker thread in place. + { + MutexLockerEx x(CGC_lock, true); + while(_slt == NULL && !_should_terminate) { + CGC_lock->wait(true, 200); + } + } + + ShenandoahHeap* heap = ShenandoahHeap::heap(); + + while (!_should_terminate) { + if (_do_full_gc) { + { + if (_full_gc_cause == GCCause::_allocation_failure) { + heap->shenandoahPolicy()->record_allocation_failure_gc(); + } else { + heap->shenandoahPolicy()->record_user_requested_gc(); + } + + VM_ShenandoahFullGC full_gc; + heap->jni_critical()->execute_in_vm_thread(&full_gc); + } + MonitorLockerEx ml(ShenandoahFullGC_lock); + _do_full_gc = false; + ml.notify_all(); + } else if (heap->shenandoahPolicy()->should_start_concurrent_mark(heap->used(), + heap->capacity())) + { + + if (ShenandoahGCVerbose) + tty->print("Capacity = "SIZE_FORMAT" Used = "SIZE_FORMAT" doing initMark\n", heap->capacity(), heap->used()); + + if (ShenandoahGCVerbose) tty->print("Starting a mark"); + + VM_ShenandoahInitMark initMark; + VMThread::execute(&initMark); + + if (ShenandoahConcurrentMarking) { + ShenandoahHeap::heap()->concurrentMark()->mark_from_roots(); + + VM_ShenandoahStartEvacuation finishMark; + heap->jni_critical()->execute_in_vm_thread(&finishMark); + } + + if (cm_has_aborted()) { + clear_cm_aborted(); + assert(heap->is_bitmap_clear(), "need to continue with clear mark bitmap"); + assert(! heap->concurrent_mark_in_progress(), "concurrent mark must have terminated"); + continue; + } + if (! _should_terminate) { + // If we're not concurrently evacuating, evacuation is done + // from VM_ShenandoahFinishMark within the VMThread above. + if (ShenandoahConcurrentEvacuation) { + VM_ShenandoahEvacuation evacuation; + evacuation.doit(); + } + } + + if (heap->shenandoahPolicy()->update_refs_early() && ! _should_terminate && ! heap->cancelled_concgc()) { + if (ShenandoahConcurrentUpdateRefs) { + VM_ShenandoahUpdateRefs update_refs; + VMThread::execute(&update_refs); + heap->update_references(); + } + } else { + if (heap->is_evacuation_in_progress()) { + heap->set_evacuation_in_progress(false); + } + heap->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::reset_bitmaps); + heap->reset_mark_bitmap(); + heap->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::reset_bitmaps); + } + + } else { + Thread::current()->_ParkEvent->park(10) ; + // yield(); + } + heap->clear_cancelled_concgc(); + // Make sure the _do_full_gc flag changes are seen. + OrderAccess::storeload(); + } +} + +void ShenandoahConcurrentThread::do_full_gc(GCCause::Cause cause) { + + assert(Thread::current()->is_Java_thread(), "expect Java thread here"); + + MonitorLockerEx ml(ShenandoahFullGC_lock); + schedule_full_gc(); + _full_gc_cause = cause; + while (_do_full_gc) { + ml.wait(); + OrderAccess::storeload(); + } + assert(_do_full_gc == false, "expect full GC to have completed"); +} + +void ShenandoahConcurrentThread::schedule_full_gc() { + _do_full_gc = true; +} + +void ShenandoahConcurrentThread::print() const { + print_on(tty); +} + +void ShenandoahConcurrentThread::print_on(outputStream* st) const { + st->print("Shenandoah Concurrent Thread"); + Thread::print_on(st); + st->cr(); +} + +void ShenandoahConcurrentThread::sleepBeforeNextCycle() { + assert(false, "Wake up in the GC thread that never sleeps :-)"); +} + +void ShenandoahConcurrentThread::set_cm_started() { + assert(!_concurrent_mark_in_progress, "cycle in progress"); + _concurrent_mark_started = true; +} + +void ShenandoahConcurrentThread::clear_cm_started() { + assert(_concurrent_mark_in_progress, "must be starting a cycle"); + _concurrent_mark_started = false; +} + +bool ShenandoahConcurrentThread::cm_started() { + return _concurrent_mark_started; +} + +void ShenandoahConcurrentThread::set_cm_in_progress() { + assert(_concurrent_mark_started, "must be starting a cycle"); + _concurrent_mark_in_progress = true; +} + +void ShenandoahConcurrentThread::clear_cm_in_progress() { + assert(!_concurrent_mark_started, "must not be starting a new cycle"); + _concurrent_mark_in_progress = false; +} + +bool ShenandoahConcurrentThread::cm_in_progress() { + return _concurrent_mark_in_progress; +} + +void ShenandoahConcurrentThread::start() { + create_and_start(); +} + +void ShenandoahConcurrentThread::yield() { + _sts.yield(); +} + +void ShenandoahConcurrentThread::safepoint_synchronize() { + assert(UseShenandoahGC, "just checking"); + _sts.synchronize(); +} + +void ShenandoahConcurrentThread::safepoint_desynchronize() { + assert(UseShenandoahGC, "just checking"); + _sts.desynchronize(); +} + +void ShenandoahConcurrentThread::makeSurrogateLockerThread(TRAPS) { + assert(UseShenandoahGC, "SLT thread needed only for concurrent GC"); + assert(THREAD->is_Java_thread(), "must be a Java thread"); + assert(_slt == NULL, "SLT already created"); + _slt = SurrogateLockerThread::make(THREAD); +} + +void ShenandoahConcurrentThread::shutdown() { + _should_terminate = true; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/shenandoahConcurrentThread.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,84 @@ +/* +Copyright 2014 Red Hat, Inc. and/or its affiliates. + */ +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHCONCURRENTTHREAD_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHCONCURRENTTHREAD_HPP + +#include "gc/shared/concurrentGCThread.hpp" +#include "gc/g1/suspendibleThreadSet.hpp" +#include "gc/shared/gcCause.hpp" +#include "memory/resourceArea.hpp" + +// For now we just want to have a concurrent marking thread. +// Once we have that working we will build a concurrent evacuation thread. + +class ShenandoahConcurrentThread: public ConcurrentGCThread { + friend class VMStructs; + + public: + virtual void run(); + + private: + volatile bool _concurrent_mark_started; + volatile bool _concurrent_mark_in_progress; + volatile bool _concurrent_mark_aborted; + + int _epoch; + + static SurrogateLockerThread* _slt; + static SuspendibleThreadSet _sts; + + bool _do_full_gc; + GCCause::Cause _full_gc_cause; + + void sleepBeforeNextCycle(); + + public: + // Constructor + ShenandoahConcurrentThread(); + ~ShenandoahConcurrentThread(); + + static void makeSurrogateLockerThread(TRAPS); + static SurrogateLockerThread* slt() { return _slt; } + + // Printing + void print_on(outputStream* st) const; + void print() const; + + void set_cm_started(); + void clear_cm_started(); + bool cm_started(); + + void set_cm_in_progress(); + void clear_cm_in_progress(); + bool cm_in_progress(); + + void cm_abort() { _concurrent_mark_aborted = true;} + bool cm_has_aborted() { return _concurrent_mark_aborted;} + void clear_cm_aborted() { _concurrent_mark_aborted = false;} + + void do_full_gc(GCCause::Cause cause); + + void schedule_full_gc(); + + // This flag returns true from the moment a marking cycle is + // initiated (during the initial-mark pause when started() is set) + // to the moment when the cycle completes (just after the next + // marking bitmap has been cleared and in_progress() is + // cleared). While this flag is true we will not start another cycle + // so that cycles do not overlap. We cannot use just in_progress() + // as the CM thread might take some time to wake up before noticing + // that started() is set and set in_progress(). + bool during_cycle() { return cm_started() || cm_in_progress(); } + + char* name() const { return (char*)"ShenandoahConcurrentThread";} + void start(); + void yield(); + + static void safepoint_synchronize(); + static void safepoint_desynchronize(); + + void shutdown(); +}; + +#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHCONCURRENTTHREAD_HPP
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/shenandoahHeap.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,2861 @@ +/* +Copyright 2014 Red Hat, Inc. and/or its affiliates. + */ +#include "precompiled.hpp" +#include "asm/macroAssembler.hpp" + +#include "classfile/symbolTable.hpp" +#include "classfile/stringTable.hpp" + +#include "gc/shared/collectedHeap.inline.hpp" +#include "gc/shared/cmBitMap.inline.hpp" +#include "gc/shared/gcHeapSummary.hpp" +#include "gc/shared/gcTimer.hpp" +#include "gc/shared/gcTrace.hpp" +#include "gc/shared/gcTraceTime.hpp" +#include "gc/shared/isGCActiveMark.hpp" + +#include "gc/shenandoah/brooksPointer.hpp" +#include "gc/shenandoah/shenandoahHumongous.hpp" +#include "gc/shenandoah/shenandoahRootProcessor.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahJNICritical.hpp" +#include "gc/shenandoah/shenandoahBarrierSet.hpp" +#include "gc/shenandoah/vm_operations_shenandoah.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/vmThread.hpp" +#include "memory/iterator.hpp" +#include "memory/oopFactory.hpp" +#include "gc/shared/referenceProcessor.hpp" +#include "gc/shared/space.inline.hpp" +#include "gc/shared/threadLocalAllocBuffer.inline.hpp" +#include "memory/universe.hpp" +#include "utilities/copy.hpp" +#include "gc/shared/vmGCOperations.hpp" +#include "runtime/atomic.inline.hpp" + +#define __ masm-> + +ShenandoahHeap* ShenandoahHeap::_pgc = NULL; + +void ShenandoahHeap::print_heap_locations(HeapWord* start, HeapWord* end) { + HeapWord* cur = NULL; + for (cur = start; cur < end; cur++) { + tty->print_cr(PTR_FORMAT" : "PTR_FORMAT, p2i(cur), p2i(*((HeapWord**) cur))); + } +} + +void ShenandoahHeap::print_heap_objects(HeapWord* start, HeapWord* end) { + HeapWord* cur = NULL; + for (cur = start; cur < end; cur = cur + oop(cur)->size()) { + oop(cur)->print(); + print_heap_locations(cur, cur + oop(cur)->size()); + } +} + +void ShenandoahHeap::print_heap_object(oop p) { + HeapWord* hw = (HeapWord*) p; + print_heap_locations(hw-1, hw+1+p->size()); +} + + +class PrintHeapRegionsClosure : public + ShenandoahHeapRegionClosure { +private: + outputStream* _st; +public: + PrintHeapRegionsClosure() : _st(tty) {} + PrintHeapRegionsClosure(outputStream* st) : _st(st) {} + + bool doHeapRegion(ShenandoahHeapRegion* r) { + r->print_on(_st); + return false; + } +}; + +class PrintHeapObjectsClosure : public ShenandoahHeapRegionClosure { +public: + bool doHeapRegion(ShenandoahHeapRegion* r) { + tty->print_cr("Region "INT32_FORMAT" top = "PTR_FORMAT" used = "SIZE_FORMAT_HEX" free = "SIZE_FORMAT_HEX, + r->region_number(), p2i(r->top()), r->used(), r->free()); + + ShenandoahHeap::heap()->print_heap_objects(r->bottom(), r->top()); + return false; + } +}; + +jint ShenandoahHeap::initialize() { + CollectedHeap::pre_initialize(); + + size_t init_byte_size = collector_policy()->initial_heap_byte_size(); + size_t max_byte_size = collector_policy()->max_heap_byte_size(); + if (ShenandoahGCVerbose) + tty->print_cr("init_byte_size = "SIZE_FORMAT","SIZE_FORMAT_HEX" max_byte_size = "INT64_FORMAT","SIZE_FORMAT_HEX, + init_byte_size, init_byte_size, max_byte_size, max_byte_size); + + Universe::check_alignment(max_byte_size, + ShenandoahHeapRegion::RegionSizeBytes, + "shenandoah heap"); + Universe::check_alignment(init_byte_size, + ShenandoahHeapRegion::RegionSizeBytes, + "shenandoah heap"); + + ReservedSpace heap_rs = Universe::reserve_heap(max_byte_size, + Arguments::conservative_max_heap_alignment()); + initialize_reserved_region((HeapWord*)heap_rs.base(), (HeapWord*) (heap_rs.base() + heap_rs.size())); + + set_barrier_set(new ShenandoahBarrierSet()); + ReservedSpace pgc_rs = heap_rs.first_part(max_byte_size); + _storage.initialize(pgc_rs, init_byte_size); + if (ShenandoahGCVerbose) { + tty->print_cr("Calling initialize on reserved space base = "PTR_FORMAT" end = "PTR_FORMAT, + p2i(pgc_rs.base()), p2i(pgc_rs.base() + pgc_rs.size())); + } + + _num_regions = init_byte_size / ShenandoahHeapRegion::RegionSizeBytes; + _max_regions = max_byte_size / ShenandoahHeapRegion::RegionSizeBytes; + _ordered_regions = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, _max_regions, mtGC); + for (size_t i = 0; i < _max_regions; i++) { + _ordered_regions[i] = NULL; + } + + _initialSize = _num_regions * ShenandoahHeapRegion::RegionSizeBytes; + size_t regionSizeWords = ShenandoahHeapRegion::RegionSizeBytes / HeapWordSize; + assert(init_byte_size == _initialSize, "tautology"); + _free_regions = new ShenandoahHeapRegionSet(_max_regions); + _collection_set = new ShenandoahHeapRegionSet(_max_regions); + + for (size_t i = 0; i < _num_regions; i++) { + ShenandoahHeapRegion* current = new ShenandoahHeapRegion(); + current->initialize_heap_region((HeapWord*) pgc_rs.base() + + regionSizeWords * i, regionSizeWords, i); + _free_regions->append(current); + _ordered_regions[i] = current; + } + _first_region = _ordered_regions[0]; + _first_region_bottom = _first_region->bottom(); + assert((((size_t) _first_region_bottom) & (ShenandoahHeapRegion::RegionSizeBytes - 1)) == 0, err_msg("misaligned heap: "PTR_FORMAT, p2i(_first_region_bottom))); + + _numAllocs = 0; + + if (ShenandoahGCVerbose) { + tty->print("All Regions\n"); + print_heap_regions(); + tty->print("Free Regions\n"); + _free_regions->print(); + } + + // The call below uses stuff (the SATB* things) that are in G1, but probably + // belong into a shared location. + JavaThread::satb_mark_queue_set().initialize(SATB_Q_CBL_mon, + SATB_Q_FL_lock, + 20 /*G1SATBProcessCompletedThreshold */, + Shared_SATB_Q_lock); + + // Reserve space for prev and next bitmap. + size_t bitmap_size = CMBitMap::compute_size(heap_rs.size()); + MemRegion heap_region = MemRegion((HeapWord*) heap_rs.base(), heap_rs.size() / HeapWordSize); + + ReservedSpace bitmap(ReservedSpace::allocation_align_size_up(bitmap_size)); + os::commit_memory_or_exit(bitmap.base(), bitmap.size(), false, err_msg("couldn't allocate mark bitmap")); + MemRegion bitmap_region = MemRegion((HeapWord*) bitmap.base(), bitmap.size() / HeapWordSize); + _mark_bit_map.initialize(heap_region, bitmap_region); + + _next_mark_bit_map = &_mark_bit_map; + reset_mark_bitmap(); + + // Initialize fast collection set test structure. + _in_cset_fast_test_length = _max_regions; + _in_cset_fast_test_base = + NEW_C_HEAP_ARRAY(bool, (size_t) _in_cset_fast_test_length, mtGC); + _in_cset_fast_test = _in_cset_fast_test_base - + ((uintx) pgc_rs.base() >> ShenandoahHeapRegion::RegionSizeShift); + clear_cset_fast_test(); + + _concurrent_gc_thread = new ShenandoahConcurrentThread(); + return JNI_OK; +} + +ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : + CollectedHeap(), + _shenandoah_policy(policy), + _concurrent_mark_in_progress(false), + _evacuation_in_progress(false), + _update_references_in_progress(false), + _free_regions(NULL), + _collection_set(NULL), + _bytesAllocSinceCM(0), + _bytes_allocated_during_cm(0), + _max_allocated_gc(0), + _allocated_last_gc(0), + _used_start_gc(0), + _max_conc_workers((int) MAX2((uint) ConcGCThreads, 1U)), + _max_parallel_workers((int) MAX2((uint) ParallelGCThreads, 1U)), + _ref_processor(NULL), + _in_cset_fast_test(NULL), + _in_cset_fast_test_base(NULL), + _mark_bit_map(), + _cancelled_concgc(false), + _need_update_refs(false), + _need_reset_bitmaps(false), + _jni_critical(new ShenandoahJNICritical()) + +{ + if (ShenandoahLogConfig) { + tty->print_cr("Parallel GC threads: "UINT32_FORMAT, ParallelGCThreads); + tty->print_cr("Concurrent GC threads: "UINT32_FORMAT, ConcGCThreads); + tty->print_cr("Parallel reference processing enabled: %s", BOOL_TO_STR(ParallelRefProcEnabled)); + } + _pgc = this; + _scm = new ShenandoahConcurrentMark(); + _used = 0; + // This is odd. They are concurrent gc threads, but they are also task threads. + // Framework doesn't allow both. + _workers = new WorkGang("Concurrent GC Threads", ParallelGCThreads, + /* are_GC_task_threads */true, + /* are_ConcurrentGC_threads */false); + _conc_workers = new WorkGang("Concurrent GC Threads", ConcGCThreads, + /* are_GC_task_threads */true, + /* are_ConcurrentGC_threads */false); + if ((_workers == NULL) || (_conc_workers == NULL)) { + vm_exit_during_initialization("Failed necessary allocation."); + } else { + _workers->initialize_workers(); + _conc_workers->initialize_workers(); + } +} + +class ResetBitmapTask : public AbstractGangTask { +private: + ShenandoahHeapRegionSet* _regions; + +public: + ResetBitmapTask(ShenandoahHeapRegionSet* regions) : + AbstractGangTask("Parallel Reset Bitmap Task"), + _regions(regions) { + } + + void work(uint worker_id) { + ShenandoahHeapRegion* region = _regions->claim_next(); + ShenandoahHeap* heap = ShenandoahHeap::heap(); + while (region != NULL) { + heap->reset_mark_bitmap_range(region->bottom(), region->end()); + region = _regions->claim_next(); + } + } +}; + +void ShenandoahHeap::reset_mark_bitmap() { + if (ShenandoahTracePhases) { + tty->print_cr("Shenandoah starting concurrent reset bitmaps"); + } + ShenandoahHeapRegionSet regions = ShenandoahHeapRegionSet(_num_regions, _ordered_regions, _num_regions); + ResetBitmapTask task = ResetBitmapTask(®ions); + conc_workers()->set_active_workers(_max_conc_workers); + conc_workers()->run_task(&task); + if (ShenandoahTracePhases) { + tty->print_cr("Shenandoah finishing concurrent reset bitmaps"); + } +} + +void ShenandoahHeap::reset_mark_bitmap_range(HeapWord* from, HeapWord* to) { + _next_mark_bit_map->clearRange(MemRegion(from, to)); +} + +bool ShenandoahHeap::is_bitmap_clear() { + HeapWord* start = _ordered_regions[0]->bottom(); + HeapWord* end = _ordered_regions[_num_regions-1]->end(); + return _next_mark_bit_map->getNextMarkedWordAddress(start, end) == end; +} + +void ShenandoahHeap::print_on(outputStream* st) const { + st->print("Shenandoah Heap"); + st->print(" total = " SIZE_FORMAT " K, used " SIZE_FORMAT " K ", capacity()/ K, used() /K); + st->print("Region size = " SIZE_FORMAT "K ", ShenandoahHeapRegion::RegionSizeBytes / K); + if (_concurrent_mark_in_progress) { + st->print("marking "); + } + if (_evacuation_in_progress) { + st->print("evacuating "); + } + if (_update_references_in_progress) { + st->print("updating-refs "); + } + if (_cancelled_concgc) { + st->print("cancelled "); + } + st->print("\n"); + + if (Verbose) { + print_heap_regions(st); + } +} + +class InitGCLABClosure : public ThreadClosure { +public: + void do_thread(Thread* thread) { + thread->gclab().initialize(true); + } +}; + +void ShenandoahHeap::post_initialize() { + + { + MutexLockerEx ml(Threads_lock); + InitGCLABClosure init_gclabs; + for (JavaThread *thread = Threads::first(); thread != NULL; thread = thread->next()) { + init_gclabs.do_thread(thread); + } + gc_threads_do(&init_gclabs); + } + _scm->initialize(); + + if (ShenandoahProcessReferences) { + ref_processing_init(); + } + _max_workers = MAX(_max_parallel_workers, _max_conc_workers); +} + +class CalculateUsedRegionClosure : public ShenandoahHeapRegionClosure { + size_t sum; +public: + + CalculateUsedRegionClosure() { + sum = 0; + } + + bool doHeapRegion(ShenandoahHeapRegion* r) { + sum = sum + r->used(); + return false; + } + + size_t getResult() { return sum;} +}; + +size_t ShenandoahHeap::calculateUsed() { + CalculateUsedRegionClosure cl; + heap_region_iterate(&cl); + return cl.getResult(); +} + +size_t ShenandoahHeap::calculateFree() { + return capacity() - calculateUsed(); +} + +void ShenandoahHeap::verify_heap_size_consistency() { + + assert(calculateUsed() == used(), + err_msg("heap used size must be consistent heap-used: "SIZE_FORMAT" regions-used: "SIZE_FORMAT, used(), calculateUsed())); +} + +size_t ShenandoahHeap::used() const { + return _used; +} + +void ShenandoahHeap::increase_used(size_t bytes) { + _used += bytes; + // Atomic::add_ptr(bytes, &_used); +} + +void ShenandoahHeap::set_used(size_t bytes) { + _used = bytes; +} + +void ShenandoahHeap::decrease_used(size_t bytes) { + assert(_used >= bytes, "never decrease heap size by more than we've left"); + _used -= bytes; + + // Atomic::add_ptr(-bytes, &_used); +} + +size_t ShenandoahHeap::capacity() const { + return _num_regions * ShenandoahHeapRegion::RegionSizeBytes; + +} + +bool ShenandoahHeap::is_maximal_no_gc() const { + Unimplemented(); + return true; +} + +size_t ShenandoahHeap::max_capacity() const { + return _max_regions * ShenandoahHeapRegion::RegionSizeBytes; +} + +class IsInRegionClosure : public ShenandoahHeapRegionClosure { + const void* _p; + bool _result; +public: + + IsInRegionClosure(const void* p) { + _p = p; + _result = false; + } + + bool doHeapRegion(ShenandoahHeapRegion* r) { + if (r->is_in(_p)) { + _result = true; + return true; + } + return false; + } + + bool result() { return _result;} +}; + +bool ShenandoahHeap::is_in(const void* p) const { + // IsInRegionClosure isIn(p); + // heap_region_iterate(&isIn); + // bool result = isIn.result(); + + // return isIn.result(); + HeapWord* first_region_bottom = _first_region->bottom(); + HeapWord* last_region_end = first_region_bottom + (ShenandoahHeapRegion::RegionSizeBytes / HeapWordSize) * _num_regions; + return p > _first_region_bottom && p < last_region_end; +} + +bool ShenandoahHeap::is_in_partial_collection(const void* p ) { + Unimplemented(); + return false; +} + +bool ShenandoahHeap::is_scavengable(const void* p) { + // nyi(); + // return false; + return true; +} + +HeapWord* ShenandoahHeap::allocate_from_gclab(Thread* thread, size_t size) { + if (UseTLAB) { + HeapWord* obj = thread->gclab().allocate(size); + if (obj != NULL) { + return obj; + } + // Otherwise... + return allocate_from_gclab_slow(thread, size); + } else { + return NULL; + } +} + +HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) { + // Retain tlab and allocate object in shared space if + // the amount free in the tlab is too large to discard. + if (thread->gclab().free() > thread->gclab().refill_waste_limit()) { + thread->gclab().record_slow_allocation(size); + return NULL; + } + + // Discard gclab and allocate a new one. + // To minimize fragmentation, the last GCLAB may be smaller than the rest. + size_t new_gclab_size = thread->gclab().compute_size(size); + + thread->gclab().clear_before_allocation(); + + if (new_gclab_size == 0) { + return NULL; + } + + // Allocate a new GCLAB... + HeapWord* obj = allocate_new_gclab(new_gclab_size); + if (obj == NULL) { + return NULL; + } + + if (ZeroTLAB) { + // ..and clear it. + Copy::zero_to_words(obj, new_gclab_size); + } else { + // ...and zap just allocated object. +#ifdef ASSERT + // Skip mangling the space corresponding to the object header to + // ensure that the returned space is not considered parsable by + // any concurrent GC thread. + size_t hdr_size = oopDesc::header_size(); + Copy::fill_to_words(obj + hdr_size, new_gclab_size - hdr_size, badHeapWordVal); +#endif // ASSERT + } + thread->gclab().fill(obj, obj + size, new_gclab_size); + return obj; +} + +HeapWord* ShenandoahHeap::allocate_new_tlab(size_t word_size) { + return allocate_new_tlab(word_size, true); +} + +HeapWord* ShenandoahHeap::allocate_new_gclab(size_t word_size) { + return allocate_new_tlab(word_size, false); +} + +HeapWord* ShenandoahHeap::allocate_new_tlab(size_t word_size, bool mark) { + HeapWord* result = allocate_memory(word_size); + + if (result != NULL) { + if (mark && (_concurrent_mark_in_progress || + (shenandoahPolicy()->update_refs_early() && _evacuation_in_progress))) { + // We mark the whole tlab here, this way we avoid marking every single + // allocated object. We mark it from the 2nd word, because the 1st word is always + // the brooks ptr of the first object, and it confuses the fast marked-iterator + // if we mark that. + _next_mark_bit_map->parMarkRange(MemRegion(result + BrooksPointer::BROOKS_POINTER_OBJ_SIZE, + word_size - BrooksPointer::BROOKS_POINTER_OBJ_SIZE)); + } + assert(! heap_region_containing(result)->is_in_collection_set(), "Never allocate in dirty region"); + _bytesAllocSinceCM += word_size * HeapWordSize; + +#ifdef ASSERT + if (ShenandoahTraceTLabs) + tty->print_cr("allocating new tlab of size "SIZE_FORMAT" at addr "PTR_FORMAT, word_size, p2i(result)); +#endif + + } + return result; +} + +ShenandoahHeap* ShenandoahHeap::heap() { + assert(_pgc != NULL, "Unitialized access to ShenandoahHeap::heap()"); + assert(_pgc->kind() == CollectedHeap::ShenandoahHeap, "not a shenandoah heap"); + return _pgc; +} + +class VM_ShenandoahVerifyHeap: public VM_GC_Operation { +public: + VM_ShenandoahVerifyHeap(unsigned int gc_count_before, + unsigned int full_gc_count_before, + GCCause::Cause cause) + : VM_GC_Operation(gc_count_before, cause, full_gc_count_before) { } + virtual VMOp_Type type() const { return VMOp_G1CollectFull; } + virtual void doit() { + if (ShenandoahGCVerbose) + tty->print_cr("verifying heap"); + Universe::heap()->ensure_parsability(false); + Universe::verify(); + } + virtual const char* name() const { + return "Shenandoah verify trigger"; + } +}; + +class FindEmptyRegionClosure: public ShenandoahHeapRegionClosure { + ShenandoahHeapRegion* _result; + size_t _required_size; +public: + + FindEmptyRegionClosure(size_t required_size) : _required_size(required_size) { + _result = NULL; + } + + bool doHeapRegion(ShenandoahHeapRegion* r) { + if ((! r->is_in_collection_set()) && r->free() >= _required_size) { + _result = r; + return true; + } + return false; + } + ShenandoahHeapRegion* result() { return _result;} + +}; + +HeapWord* ShenandoahHeap::allocate_memory(size_t word_size) { + HeapWord* result = NULL; + result = allocate_memory_with_lock(word_size); + + if (result == NULL && ! Thread::current()->is_evacuating()) { // Allocation failed, try full-GC, then retry allocation. + // tty->print_cr("failed to allocate "SIZE_FORMAT " bytes, free regions:", word_size * HeapWordSize); + // _free_regions->print(); + collect(GCCause::_allocation_failure); + result = allocate_memory_with_lock(word_size); + } + + return result; +} + +HeapWord* ShenandoahHeap::allocate_memory_with_lock(size_t word_size) { + return allocate_memory_shenandoah_lock(word_size); +} + +HeapWord* ShenandoahHeap::allocate_memory_heap_lock(size_t word_size) { + ShouldNotReachHere(); + MutexLocker ml(Heap_lock); + return allocate_memory_work(word_size); +} + +HeapWord* ShenandoahHeap::allocate_memory_shenandoah_lock(size_t word_size) { + MutexLockerEx ml(ShenandoahHeap_lock, true); + return allocate_memory_work(word_size); +} + +ShenandoahHeapRegion* ShenandoahHeap::check_skip_humongous(ShenandoahHeapRegion* region) const { + while (region != NULL && region->is_humongous()) { + region = _free_regions->get_next(); + } + return region; +} + +ShenandoahHeapRegion* ShenandoahHeap::get_next_region_skip_humongous() const { + ShenandoahHeapRegion* next = _free_regions->get_next(); + return check_skip_humongous(next); +} + +ShenandoahHeapRegion* ShenandoahHeap::get_current_region_skip_humongous() const { + ShenandoahHeapRegion* current = _free_regions->current(); + return check_skip_humongous(current); +} + + +ShenandoahHeapRegion* ShenandoahHeap::check_grow_heap(ShenandoahHeapRegion* current) { + if (current == NULL) { + if (grow_heap_by()) { + current = _free_regions->get_next(); + assert(current != NULL, "After successfully growing the heap we should have a region"); + assert(! current->is_humongous(), "new region must not be humongous"); + } else { + current = NULL; // No more room to make a new region. OOM. + } + } + return current; +} + +ShenandoahHeapRegion* ShenandoahHeap::get_current_region() { + ShenandoahHeapRegion* current = get_current_region_skip_humongous(); + return check_grow_heap(current); +} + +ShenandoahHeapRegion* ShenandoahHeap::get_next_region() { + ShenandoahHeapRegion* current = get_next_region_skip_humongous(); + return check_grow_heap(current); +} + + +HeapWord* ShenandoahHeap::allocate_memory_work(size_t word_size) { + + if (word_size * HeapWordSize > ShenandoahHeapRegion::RegionSizeBytes) { + assert(! Thread::current()->is_evacuating(), "no humongous allocation for evacuating thread"); + return allocate_large_memory(word_size); + } + + ShenandoahHeapRegion* my_current_region = get_current_region(); + if (my_current_region == NULL) { + return NULL; // No more room to make a new region. OOM. + } + assert(my_current_region != NULL, "should have a region at this point"); + +#ifdef ASSERT + if (my_current_region->is_in_collection_set()) { + print_heap_regions(); + } +#endif + assert(! my_current_region->is_in_collection_set(), "never get targetted regions in free-lists"); + assert(! my_current_region->is_humongous(), "never attempt to allocate from humongous object regions"); + + HeapWord* result; + + result = my_current_region->par_allocate(word_size); + while (result == NULL && my_current_region != NULL) { + // 2nd attempt. Try next region. + size_t remaining = my_current_region->free(); + my_current_region = get_next_region(); + if (my_current_region == NULL) { + return NULL; // No more room to make a new region. OOM. + } + _free_regions->decrease_available(remaining); + assert(my_current_region != NULL, "should have a region at this point"); + assert(! my_current_region->is_in_collection_set(), "never get targetted regions in free-lists"); + assert(! my_current_region->is_humongous(), "never attempt to allocate from humongous object regions"); + result = my_current_region->par_allocate(word_size); + } + + if (result != NULL) { + my_current_region->increase_live_data(word_size * HeapWordSize); + increase_used(word_size * HeapWordSize); + _free_regions->decrease_available(word_size * HeapWordSize); + } + return result; +} + +HeapWord* ShenandoahHeap::allocate_large_memory(size_t words) { + if (ShenandoahTraceHumongous) { + gclog_or_tty->print_cr("allocating humongous object of size: "SIZE_FORMAT" KB", (words * HeapWordSize) / K); + } + + uint required_regions = ShenandoahHumongous::required_regions(words * HeapWordSize); + + assert(required_regions <= _max_regions, "sanity check"); + + HeapWord* result; + ShenandoahHeapRegion* free_regions[required_regions]; + + bool success = find_contiguous_free_regions(required_regions, free_regions); + if (! success) { + success = allocate_contiguous_free_regions(required_regions, free_regions); + } + if (! success) { + result = NULL; // Throw OOM, we cannot allocate the huge object. + } else { + // Initialize huge object flags in the regions. + size_t live = words * HeapWordSize; + free_regions[0]->set_humongous_start(true); + free_regions[0]->increase_live_data(live); + + for (uint i = 0; i < required_regions; i++) { + if (i == 0) { + free_regions[0]->set_humongous_start(true); + } else { + free_regions[i]->set_humongous_continuation(true); + } + free_regions[i]->set_top(free_regions[i]->end()); + increase_used(ShenandoahHeapRegion::RegionSizeBytes); + } + _free_regions->decrease_available(ShenandoahHeapRegion::RegionSizeBytes * required_regions); + result = free_regions[0]->bottom(); + } + return result; +} + +bool ShenandoahHeap::find_contiguous_free_regions(uint num_free_regions, ShenandoahHeapRegion** free_regions) { + if (ShenandoahTraceHumongous) { + gclog_or_tty->print_cr("trying to find "UINT32_FORMAT" contiguous free regions", num_free_regions); + } + uint free_regions_index = 0; + for (uint regions_index = 0; regions_index < _num_regions; regions_index++) { + // Claim a free region. + ShenandoahHeapRegion* region = _ordered_regions[regions_index]; + bool free = false; + if (region != NULL) { + if (region->free() == ShenandoahHeapRegion::RegionSizeBytes) { + assert(! region->is_humongous(), "don't reuse occupied humongous regions"); + free = true; + } + } + if (! free) { + // Not contiguous, reset search + free_regions_index = 0; + continue; + } + assert(free_regions_index < num_free_regions, "array bounds"); + free_regions[free_regions_index] = region; + free_regions_index++; + + if (free_regions_index == num_free_regions) { + if (ShenandoahTraceHumongous) { + gclog_or_tty->print_cr("found "UINT32_FORMAT" contiguous free regions:", num_free_regions); + for (uint i = 0; i < num_free_regions; i++) { + gclog_or_tty->print(UINT32_FORMAT": " , i); + free_regions[i]->print_on(gclog_or_tty); + } + } + return true; + } + + } + if (ShenandoahTraceHumongous) { + gclog_or_tty->print_cr("failed to find "UINT32_FORMAT" free regions", num_free_regions); + } + return false; +} + +bool ShenandoahHeap::allocate_contiguous_free_regions(uint num_free_regions, ShenandoahHeapRegion** free_regions) { + // We need to be smart here to avoid interleaved allocation of regions when concurrently + // allocating for large objects. We get the new index into regions array using CAS, where can + // subsequently safely allocate new regions. + int new_regions_index = ensure_new_regions(num_free_regions); + if (new_regions_index == -1) { + return false; + } + + int last_new_region = new_regions_index + num_free_regions; + + // Now we can allocate new regions at the found index without being scared that + // other threads allocate in the same contiguous region. + if (ShenandoahGCVerbose) { + tty->print_cr("allocate contiguous regions:"); + } + for (int i = new_regions_index; i < last_new_region; i++) { + ShenandoahHeapRegion* region = new ShenandoahHeapRegion(); + HeapWord* start = _first_region_bottom + (ShenandoahHeapRegion::RegionSizeBytes / HeapWordSize) * i; + region->initialize_heap_region(start, ShenandoahHeapRegion::RegionSizeBytes / HeapWordSize, i); + _ordered_regions[i] = region; + uint index = i - new_regions_index; + assert(index < num_free_regions, "array bounds"); + free_regions[index] = region; + + if (ShenandoahGCVerbose) { + region->print(); + } + } + return true; +} + +HeapWord* ShenandoahHeap::mem_allocate_locked(size_t size, + bool* gc_overhead_limit_was_exceeded) { + + // This was used for allocation while holding the Heap_lock. + // HeapWord* filler = allocate_memory(BrooksPointer::BROOKS_POINTER_OBJ_SIZE + size); + + HeapWord* filler = allocate_memory(BrooksPointer::BROOKS_POINTER_OBJ_SIZE + size); + HeapWord* result = filler + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; + if (filler != NULL) { + initialize_brooks_ptr(filler, result); + _bytesAllocSinceCM += size * HeapWordSize; +#ifdef ASSERT + if (ShenandoahTraceAllocations) { + if (*gc_overhead_limit_was_exceeded) + tty->print("gc_overhead_limit_was_exceeded"); + tty->print_cr("mem_allocate_locked object of size "SIZE_FORMAT" uat addr "PTR_FORMAT, size, p2i(result)); + } +#endif + + assert(! heap_region_containing(result)->is_in_collection_set(), "never allocate in targetted region"); + if (_concurrent_mark_in_progress || + (shenandoahPolicy()->update_refs_early() && _evacuation_in_progress)) { + mark_current_no_checks(oop(result)); + } + + return result; + } else { + tty->print_cr("Out of memory. Requested number of words: "SIZE_FORMAT" used heap: "INT64_FORMAT", bytes allocated since last CM: "INT64_FORMAT, size, used(), _bytesAllocSinceCM); + { + MutexLockerEx ml(ShenandoahHeap_lock, true); + print_heap_regions(); + tty->print("Printing "SIZE_FORMAT" free regions:\n", _free_regions->length()); + _free_regions->print(); + } + assert(false, "Out of memory"); + return NULL; + } +} + +class PrintOopContents: public OopClosure { +public: + void do_oop(oop* o) { + oop obj = *o; + tty->print_cr("References oop "PTR_FORMAT, p2i((HeapWord*) obj)); + obj->print(); + } + + void do_oop(narrowOop* o) { + assert(false, "narrowOops aren't implemented"); + } +}; + +HeapWord* ShenandoahHeap::mem_allocate(size_t size, + bool* gc_overhead_limit_was_exceeded) { + +#ifdef ASSERT + if (ShenandoahVerify && _numAllocs > 1000000) { + _numAllocs = 0; + // VM_ShenandoahVerifyHeap op(0, 0, GCCause::_allocation_failure); + // if (Thread::current()->is_VM_thread()) { + // op.doit(); + // } else { + // // ...and get the VM thread to execute it. + // VMThread::execute(&op); + // } + } + _numAllocs++; +#endif + + // MutexLockerEx ml(ShenandoahHeap_lock, true); + HeapWord* result = mem_allocate_locked(size, gc_overhead_limit_was_exceeded); + return result; +} + +class ParallelEvacuateRegionObjectClosure : public ObjectClosure { +private: + ShenandoahHeap* _heap; + Thread* _thread; + public: + ParallelEvacuateRegionObjectClosure(ShenandoahHeap* heap) : + _heap(heap), _thread(Thread::current()) { + } + + void do_object(oop p) { + +#ifdef ASSERT + if (ShenandoahTraceEvacuations) { + tty->print_cr("Calling ParallelEvacuateRegionObjectClosure on "PTR_FORMAT, p2i((HeapWord*) p)); + } +#endif + + if (_heap->is_marked_current(p) && p == ShenandoahBarrierSet::resolve_oop_static_not_null(p)) { + _heap->evacuate_object(p, _thread); + } + } +}; + +//fixme +void ShenandoahHeap::initialize_brooks_ptr(HeapWord* filler, HeapWord* obj, bool new_obj) { + BrooksPointer brooks_ptr = BrooksPointer::get(oop(obj)); + brooks_ptr.set_forwardee(oop(obj)); +} + +void ShenandoahHeap::initialize_brooks_ptr(oop p) { + BrooksPointer brooks_ptr = BrooksPointer::get(p); + brooks_ptr.set_forwardee(p); +} + +class VerifyEvacuatedObjectClosure : public ObjectClosure { + +public: + + void do_object(oop p) { + if (ShenandoahHeap::heap()->is_marked_current(p)) { + oop p_prime = oopDesc::bs()->resolve_oop(p); + assert(p != p_prime, "Should point to evacuated copy"); +#ifdef ASSERT + if (p->klass() != p_prime->klass()) { + tty->print_cr("copy has different class than original:"); + p->klass()->print_on(tty); + p_prime->klass()->print_on(tty); + } +#endif + assert(p->klass() == p_prime->klass(), err_msg("Should have the same class p: "PTR_FORMAT", p_prime: "PTR_FORMAT, p2i((HeapWord*) p), p2i((HeapWord*) p_prime))); + // assert(p->mark() == p_prime->mark(), "Should have the same mark"); + assert(p->size() == p_prime->size(), "Should be the same size"); + assert(p_prime == oopDesc::bs()->resolve_oop(p_prime), "One forward once"); + } + } +}; + +void ShenandoahHeap::verify_evacuated_region(ShenandoahHeapRegion* from_region) { + if (ShenandoahGCVerbose) { + tty->print("Verifying From Region\n"); + from_region->print(); + } + + VerifyEvacuatedObjectClosure verify_evacuation; + from_region->object_iterate_interruptible(&verify_evacuation, false); +} + +void ShenandoahHeap::parallel_evacuate_region(ShenandoahHeapRegion* from_region) { + + assert(from_region->getLiveData() > 0, "all-garbage regions are reclaimed earlier"); + + ParallelEvacuateRegionObjectClosure evacuate_region(this); + +#ifdef ASSERT + if (ShenandoahGCVerbose) { + tty->print_cr("parallel_evacuate_region starting from_region "INT32_FORMAT": free_regions = "SIZE_FORMAT, from_region->region_number(), _free_regions->available_regions()); + } +#endif + + marked_object_iterate(from_region, &evacuate_region); + +#ifdef ASSERT + if (ShenandoahVerify && ! cancelled_concgc()) { + verify_evacuated_region(from_region); + } + if (ShenandoahGCVerbose) { + tty->print_cr("parallel_evacuate_region after from_region = "INT32_FORMAT": free_regions = "SIZE_FORMAT, from_region->region_number(), _free_regions->available_regions()); + } +#endif +} + +class ParallelEvacuationTask : public AbstractGangTask { +private: + ShenandoahHeap* _sh; + ShenandoahHeapRegionSet* _cs; + +public: + ParallelEvacuationTask(ShenandoahHeap* sh, + ShenandoahHeapRegionSet* cs) : + AbstractGangTask("Parallel Evacuation Task"), + _cs(cs), + _sh(sh) {} + + void work(uint worker_id) { + + ShenandoahHeapRegion* from_hr = _cs->claim_next(); + + while (from_hr != NULL) { + if (ShenandoahGCVerbose) { + tty->print_cr("Thread "INT32_FORMAT" claimed Heap Region "INT32_FORMAT, + worker_id, + from_hr->region_number()); + from_hr->print(); + } + + assert(from_hr->getLiveData() > 0, "all-garbage regions are reclaimed early"); + _sh->parallel_evacuate_region(from_hr); + + if (_sh->cancelled_concgc()) { + if (ShenandoahTracePhases) { + tty->print_cr("Cancelled concurrent evacuation"); + } + break; + } + from_hr = _cs->claim_next(); + } + + Thread::current()->gclab().make_parsable(true); + } +}; + +class RecycleDirtyRegionsClosure: public ShenandoahHeapRegionClosure { +private: + ShenandoahHeap* _heap; + size_t _bytes_reclaimed; +public: + RecycleDirtyRegionsClosure() : _heap(ShenandoahHeap::heap()) {} + + bool doHeapRegion(ShenandoahHeapRegion* r) { + + // If evacuation has been cancelled, we can't recycle regions, we only + // clear their collection-set status. + if (_heap->cancelled_concgc()) { + r->set_is_in_collection_set(false); + return false; + } + + if (r->is_in_collection_set()) { + // tty->print_cr("recycling region "INT32_FORMAT":", r->region_number()); + // r->print_on(tty); + // tty->print_cr(" "); + _heap->decrease_used(r->used()); + _bytes_reclaimed += r->used(); + r->recycle(); + _heap->free_regions()->append(r); + } + + return false; + } + size_t bytes_reclaimed() { return _bytes_reclaimed;} + void clear_bytes_reclaimed() {_bytes_reclaimed = 0;} +}; + +void ShenandoahHeap::recycle_dirty_regions() { + RecycleDirtyRegionsClosure cl; + cl.clear_bytes_reclaimed(); + + heap_region_iterate(&cl); + + _shenandoah_policy->record_bytes_reclaimed(cl.bytes_reclaimed()); + clear_cset_fast_test(); +} + +ShenandoahHeapRegionSet* ShenandoahHeap::free_regions() { + return _free_regions; +} + +void ShenandoahHeap::print_heap_regions(outputStream* st) const { + PrintHeapRegionsClosure pc1(st); + heap_region_iterate(&pc1); +} + +class PrintAllRefsOopClosure: public ExtendedOopClosure { +private: + int _index; + const char* _prefix; + +public: + PrintAllRefsOopClosure(const char* prefix) : _index(0), _prefix(prefix) {} + + void do_oop(oop* p) { + oop o = *p; + if (o != NULL) { + if (ShenandoahHeap::heap()->is_in(o) && o->is_oop()) { + tty->print_cr("%s "INT32_FORMAT" ("PTR_FORMAT")-> "PTR_FORMAT" (marked: %s) (%s "PTR_FORMAT")", _prefix, _index, p2i(p), p2i((HeapWord*) o), BOOL_TO_STR(ShenandoahHeap::heap()->is_marked_current(o)), o->klass()->internal_name(), p2i(o->klass())); + } else { + // tty->print_cr("%s "INT32_FORMAT" ("PTR_FORMAT" dirty: %s) -> "PTR_FORMAT" (not in heap, possibly corrupted or dirty (%s))", _prefix, _index, p2i(p), BOOL_TO_STR(ShenandoahHeap::heap()->heap_region_containing(p)->is_in_collection_set()), p2i((HeapWord*) o), BOOL_TO_STR(ShenandoahHeap::heap()->heap_region_containing(o)->is_in_collection_set())); + tty->print_cr("%s "INT32_FORMAT" ("PTR_FORMAT" dirty -> "PTR_FORMAT" (not in heap, possibly corrupted or dirty)", _prefix, _index, p2i(p), p2i((HeapWord*) o)); + } + } else { + tty->print_cr("%s "INT32_FORMAT" ("PTR_FORMAT") -> "PTR_FORMAT, _prefix, _index, p2i(p), p2i((HeapWord*) o)); + } + _index++; + } + + void do_oop(narrowOop* p) { + Unimplemented(); + } + +}; + +class PrintAllRefsObjectClosure : public ObjectClosure { + const char* _prefix; + +public: + PrintAllRefsObjectClosure(const char* prefix) : _prefix(prefix) {} + + void do_object(oop p) { + if (ShenandoahHeap::heap()->is_in(p)) { + tty->print_cr("%s object "PTR_FORMAT" (marked: %s) (%s "PTR_FORMAT") refers to:", _prefix, p2i((HeapWord*) p), BOOL_TO_STR(ShenandoahHeap::heap()->is_marked_current(p)), p->klass()->internal_name(), p2i(p->klass())); + PrintAllRefsOopClosure cl(_prefix); + p->oop_iterate(&cl); + } + } +}; + +void ShenandoahHeap::print_all_refs(const char* prefix) { + tty->print_cr("printing all references in the heap"); + tty->print_cr("root references:"); + + ensure_parsability(false); + + PrintAllRefsOopClosure cl(prefix); + roots_iterate(&cl); + + tty->print_cr("heap references:"); + PrintAllRefsObjectClosure cl2(prefix); + object_iterate(&cl2); +} + +class VerifyAfterMarkingOopClosure: public ExtendedOopClosure { +private: + ShenandoahHeap* _heap; + +public: + VerifyAfterMarkingOopClosure() : + _heap(ShenandoahHeap::heap()) { } + + void do_oop(oop* p) { + oop o = *p; + if (o != NULL) { + if (! _heap->is_marked_current(o)) { + _heap->print_heap_regions(); + _heap->print_all_refs("post-mark"); + tty->print_cr("oop not marked, although referrer is marked: "PTR_FORMAT": in_heap: %s, is_marked: %s", + p2i((HeapWord*) o), BOOL_TO_STR(_heap->is_in(o)), BOOL_TO_STR(_heap->is_marked_current(o))); + _heap->print_heap_locations((HeapWord*) o, (HeapWord*) o + o->size()); + + tty->print_cr("oop class: %s", o->klass()->internal_name()); + if (_heap->is_in(p)) { + oop referrer = oop(_heap->heap_region_containing(p)->block_start_const(p)); + tty->print_cr("Referrer starts at addr "PTR_FORMAT, p2i((HeapWord*) referrer)); + referrer->print(); + _heap->print_heap_locations((HeapWord*) referrer, (HeapWord*) referrer + referrer->size()); + } + tty->print_cr("heap region containing object:"); + _heap->heap_region_containing(o)->print(); + tty->print_cr("heap region containing referrer:"); + _heap->heap_region_containing(p)->print(); + tty->print_cr("heap region containing forwardee:"); + _heap->heap_region_containing(oopDesc::bs()->resolve_oop(o))->print(); + } + assert(o->is_oop(), "oop must be an oop"); + assert(Metaspace::contains(o->klass()), "klass pointer must go to metaspace"); + if (! (o == oopDesc::bs()->resolve_oop(o))) { + tty->print_cr("oops has forwardee: p: "PTR_FORMAT" (%s), o = "PTR_FORMAT" (%s), new-o: "PTR_FORMAT" (%s)", p2i(p), BOOL_TO_STR(_heap->heap_region_containing(p)->is_in_collection_set()), p2i((HeapWord*) o), BOOL_TO_STR(_heap->heap_region_containing(o)->is_in_collection_set()), p2i((HeapWord*) oopDesc::bs()->resolve_oop(o)), BOOL_TO_STR(_heap->heap_region_containing(oopDesc::bs()->resolve_oop(o))->is_in_collection_set())); + tty->print_cr("oop class: %s", o->klass()->internal_name()); + } + assert(o == oopDesc::bs()->resolve_oop(o), "oops must not be forwarded"); + assert(! _heap->heap_region_containing(o)->is_in_collection_set(), "references must not point to dirty heap regions"); + assert(_heap->is_marked_current(o), "live oops must be marked current"); + } + } + + void do_oop(narrowOop* p) { + Unimplemented(); + } + +}; + +class IterateMarkedCurrentObjectsClosure: public ObjectClosure { +private: + ShenandoahHeap* _heap; + ExtendedOopClosure* _cl; +public: + IterateMarkedCurrentObjectsClosure(ExtendedOopClosure* cl) : + _heap(ShenandoahHeap::heap()), _cl(cl) {}; + + void do_object(oop p) { + if (_heap->is_marked_current(p)) { + p->oop_iterate(_cl); + } + } + +}; + +class IterateMarkedObjectsClosure: public ObjectClosure { +private: + ShenandoahHeap* _heap; + ExtendedOopClosure* _cl; +public: + IterateMarkedObjectsClosure(ExtendedOopClosure* cl) : + _heap(ShenandoahHeap::heap()), _cl(cl) {}; + + void do_object(oop p) { + if (_heap->is_marked_current(p)) { + p->oop_iterate(_cl); + } + } + +}; + +void ShenandoahHeap::verify_heap_after_marking() { + + verify_heap_size_consistency(); + + if (ShenandoahGCVerbose) { + tty->print("verifying heap after marking\n"); + } + ensure_parsability(false); + VerifyAfterMarkingOopClosure cl; + roots_iterate(&cl); + + IterateMarkedCurrentObjectsClosure marked_oops(&cl); + object_iterate(&marked_oops); +} + +void ShenandoahHeap::prepare_for_concurrent_evacuation() { + if (!cancelled_concgc()) { + + recycle_dirty_regions(); + + ensure_parsability(true); + + // NOTE: This needs to be done during a stop the world pause, because + // putting regions into the collection set concurrently with Java threads + // will create a race. In particular, acmp could fail because when we + // resolve the first operand, the containing region might not yet be in + // the collection set, and thus return the original oop. When the 2nd + // operand gets resolved, the region could be in the collection set + // and the oop gets evacuated. If both operands have originally been + // the same, we get false negatives. + ShenandoahHeapRegionSet regions = ShenandoahHeapRegionSet(_num_regions, _ordered_regions, _num_regions); + regions.reclaim_humongous_regions(); + _collection_set->clear(); + _free_regions->clear(); + _shenandoah_policy->choose_collection_and_free_sets(®ions, _collection_set, _free_regions); + + if (PrintGCTimeStamps) { + gclog_or_tty->print("Collection set used = " SIZE_FORMAT " K live = " SIZE_FORMAT " K reclaimable = " SIZE_FORMAT " K\n", + _collection_set->used() / K, _collection_set->live_data() / K, _collection_set->garbage() / K); + } + + if (_collection_set->length() == 0) + cancel_concgc(); + + _bytesAllocSinceCM = 0; + + Universe::update_heap_info_at_gc(); + } +} + + +class ShenandoahUpdateRootsClosure: public ExtendedOopClosure { + + void do_oop(oop* p) { + ShenandoahHeap::heap()->maybe_update_oop_ref(p); + } + + void do_oop(narrowOop* p) { + Unimplemented(); + } +}; + +void ShenandoahHeap::update_roots() { + + COMPILER2_PRESENT(DerivedPointerTable::clear()); + + assert(SafepointSynchronize::is_at_safepoint(), "Only iterate roots while world is stopped"); + + ShenandoahUpdateRootsClosure cl; + CodeBlobToOopClosure blobsCl(&cl, false); + CLDToOopClosure cldCl(&cl); + + ClassLoaderDataGraph::clear_claimed_marks(); + + { + ShenandoahRootProcessor rp(this, 1); + rp.process_all_roots(&cl, &cldCl, &blobsCl); + ShenandoahIsAliveClosure is_alive; + JNIHandles::weak_oops_do(&is_alive, &cl); + } + + COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); +} + +class ShenandoahUpdateObjectsClosure : public ObjectClosure { + ShenandoahHeap* _heap; + +public: + ShenandoahUpdateObjectsClosure() : + _heap(ShenandoahHeap::heap()) { + } + + void do_object(oop p) { + ShenandoahUpdateRootsClosure refs_cl; + assert(ShenandoahHeap::heap()->is_in(p), "only update objects in heap (where else?)"); + + if (_heap->is_marked_current(p)) { + p->oop_iterate(&refs_cl); + } + } + +}; + +class ParallelUpdateRefsTask : public AbstractGangTask { +private: + ShenandoahHeapRegionSet* _regions; + +public: + ParallelUpdateRefsTask(ShenandoahHeapRegionSet* regions) : + AbstractGangTask("Parallel Update References Task"), + _regions(regions) { + } + + void work(uint worker_id) { + ShenandoahUpdateObjectsClosure update_refs_cl; + ShenandoahHeapRegion* region = _regions->claim_next(); + ShenandoahHeap* heap = ShenandoahHeap::heap(); + while (region != NULL && ! heap->cancelled_concgc()) { + if ((! region->is_in_collection_set()) && ! region->is_humongous_continuation()) { + heap->marked_object_iterate_careful(region, &update_refs_cl); + } + heap->reset_mark_bitmap_range(region->bottom(), region->end()); + region = _regions->claim_next(); + } + if (ShenandoahTracePhases && heap->cancelled_concgc()) { + tty->print_cr("Cancelled concurrent update references"); + } + } +}; + +class RetireTLABClosure : public ThreadClosure { +private: + bool _retire; + +public: + RetireTLABClosure(bool retire) : _retire(retire) { + } + + void do_thread(Thread* thread) { + thread->gclab().make_parsable(_retire); + } +}; + +void ShenandoahHeap::ensure_parsability(bool retire_tlabs) { + if (UseTLAB) { + CollectedHeap::ensure_parsability(retire_tlabs); + + RetireTLABClosure cl(retire_tlabs); + for (JavaThread *thread = Threads::first(); thread != NULL; thread = thread->next()) { + cl.do_thread(thread); + } + gc_threads_do(&cl); + } +} + +void ShenandoahHeap::prepare_for_update_references() { + ensure_parsability(true); + + ShenandoahHeapRegionSet regions = ShenandoahHeapRegionSet(_num_regions, _ordered_regions, _num_regions); + regions.set_concurrent_iteration_safe_limits(); + + if (ShenandoahVerifyReadsToFromSpace) { + set_from_region_protection(false); + + // We need to update the roots so that they are ok for C2 when returning from the safepoint. + update_roots(); + + set_from_region_protection(true); + + } else { + // We need to update the roots so that they are ok for C2 when returning from the safepoint. + update_roots(); + } + + set_update_references_in_progress(true); +} + +void ShenandoahHeap::update_references() { + + ShenandoahHeapRegionSet regions = ShenandoahHeapRegionSet(_num_regions, _ordered_regions, _num_regions); + ParallelUpdateRefsTask task = ParallelUpdateRefsTask(®ions); + conc_workers()->set_active_workers(_max_conc_workers); + _shenandoah_policy->record_phase_start(ShenandoahCollectorPolicy::conc_uprefs); + conc_workers()->run_task(&task); + _shenandoah_policy->record_phase_end(ShenandoahCollectorPolicy::conc_uprefs); + conc_workers()->set_active_workers(_max_conc_workers); + + if (! cancelled_concgc()) { + VM_ShenandoahUpdateRootRefs update_roots; + if (ShenandoahConcurrentUpdateRefs) { + VMThread::execute(&update_roots); + } else { + update_roots.doit(); + } + + _allocated_last_gc = used() - _used_start_gc; + size_t max_allocated_gc = MAX2(_max_allocated_gc, _allocated_last_gc); + /* + tty->print_cr("prev max_allocated_gc: "SIZE_FORMAT", new max_allocated_gc: "SIZE_FORMAT", allocated_last_gc: "SIZE_FORMAT" diff %f", _max_allocated_gc, max_allocated_gc, _allocated_last_gc, ((double) max_allocated_gc/ (double) _allocated_last_gc)); + */ + _max_allocated_gc = max_allocated_gc; + + // Update-references completed, no need to update-refs during marking. + set_need_update_refs(false); + } + + Universe::update_heap_info_at_gc(); + + set_update_references_in_progress(false); +} + + +class ShenandoahEvacuateUpdateRootsClosure: public ExtendedOopClosure { +private: + ShenandoahHeap* _heap; + Thread* _thread; +public: + ShenandoahEvacuateUpdateRootsClosure() : + _heap(ShenandoahHeap::heap()), _thread(Thread::current()) { + } + + void do_oop(oop* p) { + assert(_heap->is_evacuation_in_progress(), "Only do this when evacuation is in progress"); + + oop obj = oopDesc::load_heap_oop(p); + if (obj != NULL && _heap->in_cset_fast_test((HeapWord*) obj)) { + assert(_heap->is_marked_current(obj), err_msg("only evacuate marked objects %d %d", _heap->is_marked_current(obj), _heap->is_marked_current(ShenandoahBarrierSet::resolve_oop_static_not_null(obj)))); + oop resolved = ShenandoahBarrierSet::resolve_oop_static_not_null(obj); + if (resolved == obj) { + resolved = _heap->evacuate_object(obj, _thread); + } + oopDesc::store_heap_oop(p, resolved); + } +#ifdef ASSERT + else if (! oopDesc::is_null(obj)) { + // tty->print_cr("not updating root at: "PTR_FORMAT" with object: "PTR_FORMAT", is_in_heap: %s, is_in_cset: %s, is_marked: %s", p2i(p), p2i((HeapWord*) obj), BOOL_TO_STR(_heap->is_in(obj)), BOOL_TO_STR(_heap->in_cset_fast_test(obj)), BOOL_TO_STR(_heap->is_marked_current(obj))); + } +#endif + } + + void do_oop(narrowOop* p) { + Unimplemented(); + } +}; + +class ShenandoahEvacuateUpdateStrongRootsTask : public AbstractGangTask { + ShenandoahRootProcessor* _rp; +public: + + ShenandoahEvacuateUpdateStrongRootsTask(ShenandoahRootProcessor* rp) : + AbstractGangTask("Shenandoah evacuate and update strong roots"), + _rp(rp) + { + // Nothing else to do. + } + + void work(uint worker_id) { + ShenandoahEvacuateUpdateRootsClosure cl; + CodeBlobToOopClosure blobsCl(&cl, false); + CLDToOopClosure cldCl(&cl); + + _rp->process_all_roots(&cl, &cldCl, &blobsCl); + } +}; + +class ShenandoahEvacuateUpdateWeakRootsTask : public AbstractGangTask { +public: + + ShenandoahEvacuateUpdateWeakRootsTask() : AbstractGangTask("Shenandoah evacuate and update weak roots") { + // Nothing else to do. + } + + void work(uint worker_id) { + ShenandoahEvacuateUpdateRootsClosure cl; + ShenandoahIsAliveClosure is_alive; + JNIHandles::weak_oops_do(&is_alive, &cl); + + ShenandoahHeap* heap = ShenandoahHeap::heap(); + if (ShenandoahProcessReferences) { + heap->ref_processor()->weak_oops_do(&cl); + } + } +}; + +void ShenandoahHeap::evacuate_and_update_roots() { + + COMPILER2_PRESENT(DerivedPointerTable::clear()); + + if (ShenandoahVerifyReadsToFromSpace) { + set_from_region_protection(false); + } + + assert(SafepointSynchronize::is_at_safepoint(), "Only iterate roots while world is stopped"); + ClassLoaderDataGraph::clear_claimed_marks(); + + { + ShenandoahRootProcessor rp(this, _max_parallel_workers); + ShenandoahEvacuateUpdateStrongRootsTask strong_roots_task(&rp); + workers()->set_active_workers(_max_parallel_workers); + workers()->run_task(&strong_roots_task); + } + + // We process weak roots using only 1 worker thread, multi-threaded weak roots + // processing is not implemented yet. We can't use the VMThread itself, because + // we need to grab the Heap_lock. + { + ShenandoahEvacuateUpdateWeakRootsTask weak_roots_task; + workers()->set_active_workers(1); + workers()->run_task(&weak_roots_task); + workers()->set_active_workers(_max_parallel_workers); + } + + if (ShenandoahVerifyReadsToFromSpace) { + set_from_region_protection(true); + } + + COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); + +} + + +void ShenandoahHeap::do_evacuation() { + assert(Thread::current()->is_VM_thread() || ShenandoahConcurrentEvacuation, "Only evacuate from VMThread unless we do concurrent evacuation"); + + parallel_evacuate(); + + if (! ShenandoahConcurrentEvacuation) { + // We need to make sure that after leaving the safepoint, all + // GC roots are up-to-date. This is an assumption built into + // the hotspot compilers, especially C2, that allows it to + // do optimizations like lifting barriers outside of a loop. + + if (ShenandoahVerifyReadsToFromSpace) { + set_from_region_protection(false); + + update_roots(); + + set_from_region_protection(true); + + } else { + update_roots(); + } + } + + if (ShenandoahVerify && ! cancelled_concgc()) { + VM_ShenandoahVerifyHeapAfterEvacuation verify_after_evacuation; + if (Thread::current()->is_VM_thread()) { + verify_after_evacuation.doit(); + } else { + VMThread::execute(&verify_after_evacuation); + } + } + +} + +void ShenandoahHeap::parallel_evacuate() { + + if (! cancelled_concgc()) { + assert(Thread::current()->is_VM_thread() || ShenandoahConcurrentEvacuation, "Only evacuate from VMThread unless we do concurrent evacuation"); + + if (ShenandoahGCVerbose) { + tty->print_cr("starting parallel_evacuate"); + // PrintHeapRegionsClosure pc1; + // heap_region_iterate(&pc1); + } + + _shenandoah_policy->record_phase_start(ShenandoahCollectorPolicy::conc_evac); + + if (ShenandoahGCVerbose) { + tty->print("Printing all available regions"); + print_heap_regions(); + } + + if (ShenandoahPrintCollectionSet) { + tty->print("Printing collection set which contains "SIZE_FORMAT" regions:\n", _collection_set->length()); + _collection_set->print(); + + tty->print("Printing free set which contains "SIZE_FORMAT" regions:\n", _free_regions->length()); + _free_regions->print(); + + // if (_collection_set->length() == 0) + // print_heap_regions(); + } + + ParallelEvacuationTask evacuationTask = ParallelEvacuationTask(this, _collection_set); + + conc_workers()->set_active_workers(_max_conc_workers); + conc_workers()->run_task(&evacuationTask); + //workers()->set_active_workers(_max_parallel_workers); + + if (ShenandoahGCVerbose) { + + tty->print("Printing postgc collection set which contains "SIZE_FORMAT" regions:\n", _collection_set->available_regions()); + _collection_set->print(); + + tty->print("Printing postgc free regions which contain "SIZE_FORMAT" free regions:\n", _free_regions->available_regions()); + _free_regions->print(); + + tty->print_cr("finished parallel_evacuate"); + print_heap_regions(); + + tty->print_cr("all regions after evacuation:"); + print_heap_regions(); + } + + _shenandoah_policy->record_phase_end(ShenandoahCollectorPolicy::conc_evac); + } +} + +class VerifyEvacuationClosure: public ExtendedOopClosure { +private: + ShenandoahHeap* _heap; + ShenandoahHeapRegion* _from_region; + +public: + VerifyEvacuationClosure(ShenandoahHeapRegion* from_region) : + _heap(ShenandoahHeap::heap()), _from_region(from_region) { } + + void do_oop(oop* p) { + oop heap_oop = oopDesc::load_heap_oop(p); + if (! oopDesc::is_null(heap_oop)) { + guarantee(! _from_region->is_in(heap_oop), err_msg("no references to from-region allowed after evacuation: "PTR_FORMAT, p2i((HeapWord*) heap_oop))); + } + } + + void do_oop(narrowOop* p) { + Unimplemented(); + } + +}; + +void ShenandoahHeap::roots_iterate(ExtendedOopClosure* cl) { + + assert(SafepointSynchronize::is_at_safepoint(), "Only iterate roots while world is stopped"); + + CodeBlobToOopClosure blobsCl(cl, false); + CLDToOopClosure cldCl(cl); + + ClassLoaderDataGraph::clear_claimed_marks(); + + ShenandoahRootProcessor rp(this, 1); + rp.process_all_roots(cl, &cldCl, &blobsCl); +} + +void ShenandoahHeap::weak_roots_iterate(ExtendedOopClosure* cl) { + if (ShenandoahProcessReferences) { + ref_processor()->weak_oops_do(cl); + } + ShenandoahAlwaysTrueClosure always_true; + JNIHandles::weak_oops_do(&always_true, cl); +} + +void ShenandoahHeap::verify_evacuation(ShenandoahHeapRegion* from_region) { + + VerifyEvacuationClosure rootsCl(from_region); + roots_iterate(&rootsCl); + +} + +bool ShenandoahHeap::supports_tlab_allocation() const { + return true; +} + + +size_t ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const { + ShenandoahHeapRegion* current = get_current_region_skip_humongous(); + if (current == NULL) + return 0; + else if (current->free() > MinTLABSize) { + return current->free(); + } else { + return MinTLABSize; + } +} + +size_t ShenandoahHeap::max_tlab_size() const { + return ShenandoahHeapRegion::RegionSizeBytes; +} + +class ResizeGCLABClosure : public ThreadClosure { +public: + void do_thread(Thread* thread) { + thread->gclab().resize(); + } +}; + +void ShenandoahHeap::resize_all_tlabs() { + CollectedHeap::resize_all_tlabs(); + + if (PrintTLAB && Verbose) { + tty->print_cr("Resizing Shenandoah GCLABs..."); + } + + ResizeGCLABClosure cl; + for (JavaThread *thread = Threads::first(); thread != NULL; thread = thread->next()) { + cl.do_thread(thread); + } + gc_threads_do(&cl); + + if (PrintTLAB && Verbose) { + tty->print_cr("Done resizing Shenandoah GCLABs..."); + } +} + +class AccumulateStatisticsGCLABClosure : public ThreadClosure { +public: + void do_thread(Thread* thread) { + thread->gclab().accumulate_statistics(); + thread->gclab().initialize_statistics(); + } +}; + +void ShenandoahHeap::accumulate_statistics_all_gclabs() { + + AccumulateStatisticsGCLABClosure cl; + for (JavaThread *thread = Threads::first(); thread != NULL; thread = thread->next()) { + cl.do_thread(thread); + } + gc_threads_do(&cl); +} + +bool ShenandoahHeap::can_elide_tlab_store_barriers() const { + return true; +} + +oop ShenandoahHeap::new_store_pre_barrier(JavaThread* thread, oop new_obj) { + // Overridden to do nothing. + return new_obj; +} + +bool ShenandoahHeap::can_elide_initializing_store_barrier(oop new_obj) { + return true; +} + +bool ShenandoahHeap::card_mark_must_follow_store() const { + return false; +} + +bool ShenandoahHeap::supports_heap_inspection() const { + return false; +} + +size_t ShenandoahHeap::unsafe_max_alloc() { + return ShenandoahHeapRegion::RegionSizeBytes / HeapWordSize; +} + +void ShenandoahHeap::collect(GCCause::Cause cause) { + if (GCCause::is_user_requested_gc(cause)) { + if (! DisableExplicitGC) { + if (ShenandoahTraceFullGC) { + gclog_or_tty->print_cr("Shenandoah-full-gc: requested full GC"); + } + cancel_concgc(); + _concurrent_gc_thread->do_full_gc(cause); + } + } else if (cause == GCCause::_allocation_failure) { + + if (ShenandoahTraceFullGC) { + gclog_or_tty->print_cr("Shenandoah-full-gc: full GC for allocation failure heap free: "SIZE_FORMAT", available: "SIZE_FORMAT, capacity() - used(), free_regions()->available()); + } + cancel_concgc(); + collector_policy()->set_should_clear_all_soft_refs(true); + _concurrent_gc_thread->do_full_gc(cause); + + } else if (cause == GCCause::_gc_locker) { + + if (ShenandoahTraceJNICritical) { + gclog_or_tty->print_cr("Resuming deferred evacuation after JNI critical regions"); + } + + jni_critical()->notify_jni_critical(); + } +} + +void ShenandoahHeap::do_full_collection(bool clear_all_soft_refs) { + //assert(false, "Shouldn't need to do full collections"); +} + +AdaptiveSizePolicy* ShenandoahHeap::size_policy() { + Unimplemented(); + return NULL; + +} + +ShenandoahCollectorPolicy* ShenandoahHeap::collector_policy() const { + return _shenandoah_policy; +} + + +HeapWord* ShenandoahHeap::block_start(const void* addr) const { + Space* sp = space_containing(addr); + if (sp != NULL) { + return sp->block_start(addr); + } + return NULL; +} + +size_t ShenandoahHeap::block_size(const HeapWord* addr) const { + Space* sp = space_containing(addr); + assert(sp != NULL, "block_size of address outside of heap"); + return sp->block_size(addr); +} + +bool ShenandoahHeap::block_is_obj(const HeapWord* addr) const { + Space* sp = space_containing(addr); + return sp->block_is_obj(addr); +} + +jlong ShenandoahHeap::millis_since_last_gc() { + return 0; +} + +void ShenandoahHeap::prepare_for_verify() { + if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) { + ensure_parsability(false); + } +} + +void ShenandoahHeap::print_gc_threads_on(outputStream* st) const { + workers()->print_worker_threads_on(st); + conc_workers()->print_worker_threads_on(st); +} + +void ShenandoahHeap::gc_threads_do(ThreadClosure* tcl) const { + workers()->threads_do(tcl); + conc_workers()->threads_do(tcl); +} + +void ShenandoahHeap::print_tracing_info() const { + if (PrintGCDetails) { + _shenandoah_policy->print_tracing_info(); + } +} + +class ShenandoahVerifyRootsClosure: public ExtendedOopClosure { +private: + ShenandoahHeap* _heap; + VerifyOption _vo; + bool _failures; +public: + // _vo == UsePrevMarking -> use "prev" marking information, + // _vo == UseNextMarking -> use "next" marking information, + // _vo == UseMarkWord -> use mark word from object header. + ShenandoahVerifyRootsClosure(VerifyOption vo) : + _heap(ShenandoahHeap::heap()), + _vo(vo), + _failures(false) { } + + bool failures() { return _failures; } + + void do_oop(oop* p) { + if (*p != NULL) { + oop heap_oop = oopDesc::load_heap_oop(p); + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + if (!obj->is_oop()) { + { // Just for debugging. + gclog_or_tty->print_cr("Root location "PTR_FORMAT + "verified "PTR_FORMAT, p2i(p), p2i((void*) obj)); + // obj->print_on(gclog_or_tty); + } + } + guarantee(obj->is_oop(), "is_oop"); + } + } + + void do_oop(narrowOop* p) { + Unimplemented(); + } + +}; + +class ShenandoahVerifyHeapClosure: public ObjectClosure { +private: + ShenandoahVerifyRootsClosure _rootsCl; +public: + ShenandoahVerifyHeapClosure(ShenandoahVerifyRootsClosure rc) : + _rootsCl(rc) {}; + + void do_object(oop p) { + _rootsCl.do_oop(&p); + } +}; + +class ShenandoahVerifyKlassClosure: public KlassClosure { + OopClosure *_oop_closure; + public: + ShenandoahVerifyKlassClosure(OopClosure* cl) : _oop_closure(cl) {} + void do_klass(Klass* k) { + k->oops_do(_oop_closure); + } +}; + +void ShenandoahHeap::verify(bool silent , VerifyOption vo) { + if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) { + + ShenandoahVerifyRootsClosure rootsCl(vo); + + assert(Thread::current()->is_VM_thread(), + "Expected to be executed serially by the VM thread at this point"); + + roots_iterate(&rootsCl); + + bool failures = rootsCl.failures(); + if (ShenandoahGCVerbose) + gclog_or_tty->print("verify failures: %s", BOOL_TO_STR(failures)); + + ShenandoahVerifyHeapClosure heapCl(rootsCl); + + object_iterate(&heapCl); + // TODO: Implement rest of it. +#ifdef ASSERT_DISABLED + verify_live(); +#endif + } else { + if (!silent) gclog_or_tty->print("(SKIPPING roots, heapRegions, remset) "); + } +} +size_t ShenandoahHeap::tlab_capacity(Thread *thr) const { + return _free_regions->available(); +} + +class ShenandoahIterateObjectClosureRegionClosure: public ShenandoahHeapRegionClosure { + ObjectClosure* _cl; +public: + ShenandoahIterateObjectClosureRegionClosure(ObjectClosure* cl) : _cl(cl) {} + bool doHeapRegion(ShenandoahHeapRegion* r) { + r->object_iterate_interruptible(_cl, false); + return false; + } +}; + +class ShenandoahIterateUpdateClosure: public ShenandoahHeapRegionClosure { + ObjectClosure* _cl; +public: + ShenandoahIterateUpdateClosure(ObjectClosure *cl) : _cl(cl) {} + bool doHeapRegion(ShenandoahHeapRegion* r) { + if ((! r->is_in_collection_set()) && !r->is_humongous_continuation()) { + r->object_iterate_interruptible(_cl, false); + } + return false; + } +}; + +void ShenandoahHeap::cleanup_after_cancelconcgc() { + if (need_update_refs()) { + ShenandoahUpdateObjectsClosure update_refs_cl; + ShenandoahIterateUpdateClosure blk(&update_refs_cl); + heap_region_iterate(&blk, false, false); + } +} + +class ShenandoahIterateObjectClosureCarefulRegionClosure: public ShenandoahHeapRegionClosure { + ObjectClosureCareful* _cl; +public: + ShenandoahIterateObjectClosureCarefulRegionClosure(ObjectClosureCareful* cl) : _cl(cl) {} + bool doHeapRegion(ShenandoahHeapRegion* r) { + r->object_iterate_careful(_cl); + return false; + } +}; + +void ShenandoahHeap::object_iterate(ObjectClosure* cl) { + ShenandoahIterateObjectClosureRegionClosure blk(cl); + heap_region_iterate(&blk, false, true); +} + +void ShenandoahHeap::object_iterate_careful(ObjectClosureCareful* cl) { + ShenandoahIterateObjectClosureCarefulRegionClosure blk(cl); + heap_region_iterate(&blk, false, true); +} + +void ShenandoahHeap::safe_object_iterate(ObjectClosure* cl) { + Unimplemented(); +} + +void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region, ObjectClosure* cl) { + marked_object_iterate(region, cl, region->bottom(), region->top()); +} + +void ShenandoahHeap::marked_object_iterate_careful(ShenandoahHeapRegion* region, ObjectClosure* cl) { + marked_object_iterate(region, cl, region->bottom(), region->concurrent_iteration_safe_limit()); +} + +void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region, ObjectClosure* cl, + HeapWord* addr, HeapWord* limit) { + addr += BrooksPointer::BROOKS_POINTER_OBJ_SIZE; + HeapWord* last_addr = NULL; + size_t last_size = 0; + while (addr < limit) { + addr = _next_mark_bit_map->getNextMarkedWordAddress(addr, limit); + if (addr < limit) { + oop obj = oop(addr); + assert(is_marked_current(obj), "object expected to be marked"); + cl->do_object(obj); + last_addr = addr; + last_size = obj->size(); + addr += obj->size() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; + } else { + break; + } + } +} + +class ShenandoahIterateOopClosureRegionClosure : public ShenandoahHeapRegionClosure { + MemRegion _mr; + ExtendedOopClosure* _cl; + bool _skip_unreachable_objects; +public: + ShenandoahIterateOopClosureRegionClosure(ExtendedOopClosure* cl, bool skip_unreachable_objects) : + _cl(cl), _skip_unreachable_objects(skip_unreachable_objects) {} + ShenandoahIterateOopClosureRegionClosure(MemRegion mr, ExtendedOopClosure* cl) + :_mr(mr), _cl(cl) {} + bool doHeapRegion(ShenandoahHeapRegion* r) { + r->oop_iterate_skip_unreachable(_cl, _skip_unreachable_objects); + return false; + } +}; + +void ShenandoahHeap::oop_iterate(ExtendedOopClosure* cl, bool skip_dirty_regions, bool skip_unreachable_objects) { + ShenandoahIterateOopClosureRegionClosure blk(cl, skip_unreachable_objects); + heap_region_iterate(&blk, skip_dirty_regions, true); +} + +void ShenandoahHeap::oop_iterate(MemRegion mr, + ExtendedOopClosure* cl) { + ShenandoahIterateOopClosureRegionClosure blk(mr, cl); + heap_region_iterate(&blk, false, true); +} + +void ShenandoahHeap::object_iterate_since_last_GC(ObjectClosure* cl) { + Unimplemented(); +} + +class SpaceClosureRegionClosure: public ShenandoahHeapRegionClosure { + SpaceClosure* _cl; +public: + SpaceClosureRegionClosure(SpaceClosure* cl) : _cl(cl) {} + bool doHeapRegion(ShenandoahHeapRegion* r) { + _cl->do_space(r); + return false; + } +}; + +void ShenandoahHeap::space_iterate(SpaceClosure* cl) { + SpaceClosureRegionClosure blk(cl); + heap_region_iterate(&blk); +} + +ShenandoahHeapRegion* +ShenandoahHeap::heap_region_containing(const void* addr) const { + uint index = heap_region_index_containing(addr); + ShenandoahHeapRegion* result = _ordered_regions[index]; +#ifdef ASSERT + if (!(addr >= result->bottom() && addr < result->end())) { + tty->print_cr("heap region does not contain address, first_region_bottom: "PTR_FORMAT", real bottom of first region: "PTR_FORMAT", num_regions: "SIZE_FORMAT, p2i(_first_region_bottom), p2i(_ordered_regions[0]->bottom()), _num_regions); + } +#endif + assert(addr >= result->bottom() && addr < result->end(), "address must be in found region"); + return result; +} + +Space* ShenandoahHeap::space_containing(const void* oop) const { + Space* res = heap_region_containing(oop); + return res; +} + +void ShenandoahHeap::gc_prologue(bool b) { + Unimplemented(); +} + +void ShenandoahHeap::gc_epilogue(bool b) { + Unimplemented(); +} + +// Apply blk->doHeapRegion() on all committed regions in address order, +// terminating the iteration early if doHeapRegion() returns true. +void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure* blk, bool skip_dirty_regions, bool skip_humongous_continuation) const { + for (size_t i = 0; i < _num_regions; i++) { + ShenandoahHeapRegion* current = _ordered_regions[i]; + if (skip_humongous_continuation && current->is_humongous_continuation()) { + continue; + } + if (skip_dirty_regions && current->is_in_collection_set()) { + continue; + } + if (blk->doHeapRegion(current)) { + return; + } + } +} + +/** + * Maybe we need that at some point... +oop* ShenandoahHeap::resolve_oop_ptr(oop* p) { + if (is_in(p) && heap_region_containing(p)->is_dirty()) { + // If the reference is in an object in from-space, we need to first + // find its to-space counterpart. + // TODO: This here is slow (linear search inside region). Make it faster. + oop from_space_oop = oop_containing_oop_ptr(p); + HeapWord* to_space_obj = (HeapWord*) oopDesc::bs()->resolve_oop(from_space_oop); + return (oop*) (to_space_obj + ((HeapWord*) p - ((HeapWord*) from_space_oop))); + } else { + return p; + } +} + +oop ShenandoahHeap::oop_containing_oop_ptr(oop* p) { + HeapWord* from_space_ref = (HeapWord*) p; + ShenandoahHeapRegion* region = heap_region_containing(from_space_ref); + HeapWord* from_space_obj = NULL; + for (HeapWord* curr = region->bottom(); curr < from_space_ref; ) { + oop curr_obj = (oop) curr; + if (curr < from_space_ref && from_space_ref < (curr + curr_obj->size())) { + from_space_obj = curr; + break; + } else { + curr += curr_obj->size(); + } + } + assert (from_space_obj != NULL, "must not happen"); + oop from_space_oop = (oop) from_space_obj; + assert (from_space_oop->is_oop(), "must be oop"); + assert(ShenandoahBarrierSet::is_brooks_ptr(oop(((HeapWord*) from_space_oop) - BrooksPointer::BROOKS_POINTER_OBJ_SIZE)), "oop must have a brooks ptr"); + return from_space_oop; +} + */ + +class ClearLivenessClosure : public ShenandoahHeapRegionClosure { + ShenandoahHeap* sh; +public: + ClearLivenessClosure(ShenandoahHeap* heap) : sh(heap) { } + + bool doHeapRegion(ShenandoahHeapRegion* r) { + r->clearLiveData(); + return false; + } +}; + + +void ShenandoahHeap::start_concurrent_marking() { + + accumulate_statistics_all_tlabs(); + + set_concurrent_mark_in_progress(true); + // We need to reset all TLABs because we'd lose marks on all objects allocated in them. + if (UseTLAB) { + ensure_parsability(true); + } + + _shenandoah_policy->record_bytes_allocated(_bytesAllocSinceCM); + _used_start_gc = used(); + +#ifdef ASSERT + if (ShenandoahDumpHeapBeforeConcurrentMark) { + ensure_parsability(false); + print_all_refs("pre-mark"); + } +#endif + + ClearLivenessClosure clc(this); + heap_region_iterate(&clc); + + // print_all_refs("pre -mark"); + + // oopDesc::_debug = true; + + concurrentMark()->prepare_unmarked_root_objs(); + + // print_all_refs("pre-mark2"); +} + + +class VerifyLivenessClosure : public ExtendedOopClosure { + + ShenandoahHeap* _sh; + +public: + VerifyLivenessClosure() : _sh ( ShenandoahHeap::heap() ) {} + + template<class T> void do_oop_nv(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + guarantee(_sh->heap_region_containing(obj)->is_in_collection_set() == (obj != oopDesc::bs()->resolve_oop(obj)), + err_msg("forwarded objects can only exist in dirty (from-space) regions is_dirty: %s, is_forwarded: %s", + BOOL_TO_STR(_sh->heap_region_containing(obj)->is_in_collection_set()), + BOOL_TO_STR(obj != oopDesc::bs()->resolve_oop(obj))) + ); + obj = oopDesc::bs()->resolve_oop(obj); + guarantee(! _sh->heap_region_containing(obj)->is_in_collection_set(), "forwarded oops must not point to dirty regions"); + guarantee(obj->is_oop(), "is_oop"); + ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap(); + if (! sh->is_marked_current(obj)) { + sh->print_on(tty); + } + assert(sh->is_marked_current(obj), err_msg("Referenced Objects should be marked obj: "PTR_FORMAT", marked: %s, is_in_heap: %s", + p2i((HeapWord*) obj), BOOL_TO_STR(sh->is_marked_current(obj)), BOOL_TO_STR(sh->is_in(obj)))); + } + } + + void do_oop(oop* p) { do_oop_nv(p); } + void do_oop(narrowOop* p) { do_oop_nv(p); } + +}; + +void ShenandoahHeap::verify_live() { + + VerifyLivenessClosure cl; + roots_iterate(&cl); + + IterateMarkedObjectsClosure marked_oops(&cl); + object_iterate(&marked_oops); + +} + +class VerifyAfterEvacuationClosure : public ExtendedOopClosure { + + ShenandoahHeap* _sh; + +public: + VerifyAfterEvacuationClosure() : _sh ( ShenandoahHeap::heap() ) {} + + template<class T> void do_oop_nv(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + guarantee(_sh->heap_region_containing(obj)->is_in_collection_set() == (obj != oopDesc::bs()->resolve_oop(obj)), + err_msg("forwarded objects can only exist in dirty (from-space) regions is_dirty: %s, is_forwarded: %s obj-klass: %s, marked: %s", + BOOL_TO_STR(_sh->heap_region_containing(obj)->is_in_collection_set()), + BOOL_TO_STR(obj != oopDesc::bs()->resolve_oop(obj)), obj->klass()->external_name(), BOOL_TO_STR(_sh->is_marked_current(obj))) + ); + obj = oopDesc::bs()->resolve_oop(obj); + guarantee(! _sh->heap_region_containing(obj)->is_in_collection_set(), "forwarded oops must not point to dirty regions"); + guarantee(obj->is_oop(), "is_oop"); + guarantee(Metaspace::contains(obj->klass()), "klass pointer must go to metaspace"); + } + } + + void do_oop(oop* p) { do_oop_nv(p); } + void do_oop(narrowOop* p) { do_oop_nv(p); } + +}; + +class VerifyAfterUpdateRefsClosure : public ExtendedOopClosure { + + ShenandoahHeap* _sh; + +public: + VerifyAfterUpdateRefsClosure() : _sh ( ShenandoahHeap::heap() ) {} + + template<class T> void do_oop_nv(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + guarantee((! _sh->heap_region_containing(obj)->is_in_collection_set()), + err_msg("no live reference must point to from-space, is_marked: %s", + BOOL_TO_STR(_sh->is_marked_current(obj)))); + if (obj != oopDesc::bs()->resolve_oop(obj) && _sh->is_in(p)) { + tty->print_cr("top-limit: "PTR_FORMAT", p: "PTR_FORMAT, p2i(_sh->heap_region_containing(p)->concurrent_iteration_safe_limit()), p2i(p)); + } + guarantee(obj == oopDesc::bs()->resolve_oop(obj), "no live reference must point to forwarded object"); + guarantee(obj->is_oop(), "is_oop"); + guarantee(Metaspace::contains(obj->klass()), "klass pointer must go to metaspace"); + } + } + + void do_oop(oop* p) { do_oop_nv(p); } + void do_oop(narrowOop* p) { do_oop_nv(p); } + +}; + +void ShenandoahHeap::verify_heap_after_evacuation() { + + verify_heap_size_consistency(); + + ensure_parsability(false); + + VerifyAfterEvacuationClosure cl; + roots_iterate(&cl); + + IterateMarkedCurrentObjectsClosure marked_oops(&cl); + object_iterate(&marked_oops); + +} + +class VerifyRegionsAfterUpdateRefsClosure : public ShenandoahHeapRegionClosure { +public: + bool doHeapRegion(ShenandoahHeapRegion* r) { + assert(! r->is_in_collection_set(), "no region must be in collection set"); + assert(! ShenandoahHeap::heap()->in_cset_fast_test(r->bottom()), "no region must be in collection set"); + return false; + } +}; + +void ShenandoahHeap::verify_regions_after_update_refs() { + VerifyRegionsAfterUpdateRefsClosure verify_regions; + heap_region_iterate(&verify_regions); +} + +void ShenandoahHeap::verify_heap_after_update_refs() { + + verify_heap_size_consistency(); + + ensure_parsability(false); + + VerifyAfterUpdateRefsClosure cl; + + roots_iterate(&cl); + weak_roots_iterate(&cl); + oop_iterate(&cl, true, true); + +} + +void ShenandoahHeap::stop_concurrent_marking() { + assert(concurrent_mark_in_progress(), "How else could we get here?"); + if (! cancelled_concgc()) { + // If we needed to update refs, and concurrent marking has been cancelled, + // we need to finish updating references. + set_need_update_refs(false); + } + set_concurrent_mark_in_progress(false); + + if (ShenandoahGCVerbose) { + print_heap_regions(); + } + +#ifdef ASSERT + if (ShenandoahVerify && ! _cancelled_concgc) { + verify_heap_after_marking(); + } + +#endif +} + +bool ShenandoahHeap::concurrent_mark_in_progress() { + return _concurrent_mark_in_progress; +} + +void ShenandoahHeap::set_concurrent_mark_in_progress(bool in_progress) { + if (ShenandoahTracePhases) { + if (in_progress) { + gclog_or_tty->print_cr("Shenandoah starting concurrent marking, heap used: "SIZE_FORMAT" MB", used() / M); + } else { + gclog_or_tty->print_cr("Shenandoah finishing concurrent marking, heap used: "SIZE_FORMAT" MB", used() / M); + } + } + + _concurrent_mark_in_progress = in_progress; + JavaThread::satb_mark_queue_set().set_active_all_threads(in_progress, ! in_progress); +} + +void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) { + if (ShenandoahTracePhases) { + if (ShenandoahConcurrentEvacuation) { + if (in_progress) { + gclog_or_tty->print_cr("Shenandoah starting concurrent evacuation, heap used: "SIZE_FORMAT" MB", used() / M); + } else { + gclog_or_tty->print_cr("Shenandoah finishing concurrent evacuation, heap used: "SIZE_FORMAT" MB", used() / M); + } + } else { + if (in_progress) { + gclog_or_tty->print_cr("Shenandoah starting non-concurrent evacuation"); + } else { + gclog_or_tty->print_cr("Shenandoah finishing non-concurrent evacuation"); + } + } + } + JavaThread::set_evacuation_in_progress_all_threads(in_progress); + _evacuation_in_progress = in_progress; + OrderAccess::fence(); +} + +bool ShenandoahHeap::is_evacuation_in_progress() { + return _evacuation_in_progress; +} + +bool ShenandoahHeap::is_update_references_in_progress() { + return _update_references_in_progress; +} + +void ShenandoahHeap::set_update_references_in_progress(bool update_refs_in_progress) { + if (ShenandoahTracePhases) { + if (ShenandoahConcurrentUpdateRefs) { + if (update_refs_in_progress) { + gclog_or_tty->print_cr("Shenandoah starting concurrent reference-updating"); + } else { + gclog_or_tty->print_cr("Shenandoah finishing concurrent reference-updating"); + } + } else { + if (update_refs_in_progress) { + gclog_or_tty->print_cr("Shenandoah starting non-concurrent reference-updating"); + } else { + gclog_or_tty->print_cr("Shenandoah finishing non-concurrent reference-updating"); + } + } + } + _update_references_in_progress = update_refs_in_progress; +} + +void ShenandoahHeap::post_allocation_collector_specific_setup(HeapWord* hw) { + oop obj = oop(hw); + + // Assuming for now that objects can't be created already locked + assert(! obj->has_displaced_mark(), "hopefully new objects don't have displaced mark"); + // tty->print_cr("post_allocation_collector_specific_setup:: "PTR_FORMAT, p2i(obj)); + + if (_concurrent_mark_in_progress + || (shenandoahPolicy()->update_refs_early() && _evacuation_in_progress)) { + mark_current_no_checks(obj); + } +} + +void ShenandoahHeap::verify_copy(oop p,oop c){ + assert(p != oopDesc::bs()->resolve_oop(p), "forwarded correctly"); + assert(oopDesc::bs()->resolve_oop(p) == c, "verify pointer is correct"); + if (p->klass() != c->klass()) { + print_heap_regions(); + } + assert(p->klass() == c->klass(), err_msg("verify class p-size: "INT32_FORMAT" c-size: "INT32_FORMAT, p->size(), c->size())); + assert(p->size() == c->size(), "verify size"); + // Object may have been locked between copy and verification + // assert(p->mark() == c->mark(), "verify mark"); + assert(c == oopDesc::bs()->resolve_oop(c), "verify only forwarded once"); + } + +void ShenandoahHeap::oom_during_evacuation() { + // tty->print_cr("Out of memory during evacuation, cancel evacuation, schedule full GC"); + // We ran out of memory during evacuation. Cancel evacuation, and schedule a full-GC. + collector_policy()->set_should_clear_all_soft_refs(true); + concurrent_thread()->schedule_full_gc(); + cancel_concgc(); + + if ((! Thread::current()->is_GC_task_thread()) && (! Thread::current()->is_ConcurrentGC_thread())) { + tty->print_cr("OOM during evacuation. Let Java thread wait until evacuation settlded.."); + while (_evacuation_in_progress) { // wait. + Thread::current()->_ParkEvent->park(1) ; + } + } + +} + +void ShenandoahHeap::copy_object(oop p, HeapWord* s) { + HeapWord* filler = s; + assert(s != NULL, "allocation of brooks pointer must not fail"); + HeapWord* copy = s + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; + + guarantee(copy != NULL, "allocation of copy object must not fail"); + Copy::aligned_disjoint_words((HeapWord*) p, copy, p->size()); + initialize_brooks_ptr(filler, copy); + +#ifdef ASSERT + if (ShenandoahTraceEvacuations) { + tty->print_cr("copy object from "PTR_FORMAT" to: "PTR_FORMAT, p2i((HeapWord*) p), p2i(copy)); + } +#endif +} + +oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { + ShenandoahHeapRegion* hr; + size_t required; + +#ifdef ASSERT + if (ShenandoahVerifyReadsToFromSpace) { + hr = heap_region_containing(p); + { + hr->memProtectionOff(); + required = BrooksPointer::BROOKS_POINTER_OBJ_SIZE + p->size(); + hr->memProtectionOn(); + } + } else { + required = BrooksPointer::BROOKS_POINTER_OBJ_SIZE + p->size(); + } +#else + required = BrooksPointer::BROOKS_POINTER_OBJ_SIZE + p->size(); +#endif + + assert(! heap_region_containing(p)->is_humongous(), "never evacuate humongous objects"); + + // Don't even attempt to evacuate anything if evacuation has been cancelled. + if (_cancelled_concgc) { + return ShenandoahBarrierSet::resolve_oop_static(p); + } + + bool alloc_from_gclab = true; + thread->set_evacuating(true); + HeapWord* filler = allocate_from_gclab(thread, required); + if (filler == NULL) { + filler = allocate_memory(required); + alloc_from_gclab = false; + } + thread->set_evacuating(false); + + if (filler == NULL) { + oom_during_evacuation(); + // If this is a Java thread, it should have waited + // until all GC threads are done, and then we + // return the forwardee. + oop resolved = ShenandoahBarrierSet::resolve_oop_static(p); + return resolved; + } + + HeapWord* copy = filler + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; + +#ifdef ASSERT + if (ShenandoahVerifyReadsToFromSpace) { + hr->memProtectionOff(); + copy_object(p, filler); + hr->memProtectionOn(); + } else { + copy_object(p, filler); + } +#else + copy_object(p, filler); +#endif + + HeapWord* result = BrooksPointer::get(p).cas_forwardee((HeapWord*) p, copy); + + oop return_val; + if (result == (HeapWord*) p) { + return_val = oop(copy); + + if (shenandoahPolicy()->update_refs_early()) { + mark_current(return_val); + } + +#ifdef ASSERT + if (ShenandoahTraceEvacuations) { + tty->print("Copy of "PTR_FORMAT" to "PTR_FORMAT" succeeded \n", p2i((HeapWord*) p), p2i(copy)); + } + assert(return_val->is_oop(), "expect oop"); + assert(p->klass() == return_val->klass(), err_msg("Should have the same class p: "PTR_FORMAT", copy: "PTR_FORMAT, p2i((HeapWord*) p), p2i((HeapWord*) copy))); +#endif + } else { + if (alloc_from_gclab) { + thread->gclab().rollback(required); + } +#ifdef ASSERT + if (ShenandoahTraceEvacuations) { + tty->print_cr("Copy of "PTR_FORMAT" to "PTR_FORMAT" failed, use other: "PTR_FORMAT, p2i((HeapWord*) p), p2i(copy), p2i((HeapWord*) result)); + } +#endif + return_val = (oopDesc*) result; + } + + return return_val; +} + +HeapWord* ShenandoahHeap::tlab_post_allocation_setup(HeapWord* obj, bool new_obj) { + HeapWord* result = obj + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; + initialize_brooks_ptr(obj, result, new_obj); + return result; +} + +uint ShenandoahHeap::oop_extra_words() { + return BrooksPointer::BROOKS_POINTER_OBJ_SIZE; +} + +bool ShenandoahHeap::grow_heap_by() { + int new_region_index = ensure_new_regions(1); + if (new_region_index != -1) { + ShenandoahHeapRegion* new_region = new ShenandoahHeapRegion(); + HeapWord* start = _first_region_bottom + (ShenandoahHeapRegion::RegionSizeBytes / HeapWordSize) * new_region_index; + new_region->initialize_heap_region(start, ShenandoahHeapRegion::RegionSizeBytes / HeapWordSize, new_region_index); + if (ShenandoahGCVerbose) { + tty->print_cr("allocating new region at index: "INT32_FORMAT, new_region_index); + new_region->print(); + } + _ordered_regions[new_region_index] = new_region; + _free_regions->append(new_region); + return true; + } else { + return false; + } +} + +int ShenandoahHeap::ensure_new_regions(int new_regions) { + + size_t num_regions = _num_regions; + size_t new_num_regions = num_regions + new_regions; + if (new_num_regions >= _max_regions) { + // Not enough regions left. + return -1; + } + + size_t expand_size = new_regions * ShenandoahHeapRegion::RegionSizeBytes; + if (ShenandoahGCVerbose) { + tty->print_cr("expanding storage by "SIZE_FORMAT_HEX" bytes, for "INT32_FORMAT" new regions", expand_size, new_regions); + } + bool success = _storage.expand_by(expand_size); + assert(success, "should always be able to expand by requested size"); + + _num_regions = new_num_regions; + + return num_regions; + +} + +#ifndef CC_INTERP +void ShenandoahHeap::compile_prepare_oop(MacroAssembler* masm, Register obj) { + __ incrementq(obj, BrooksPointer::BROOKS_POINTER_OBJ_SIZE * HeapWordSize); + __ movptr(Address(obj, -1 * HeapWordSize), obj); +} +#endif + +bool ShenandoahIsAliveClosure:: do_object_b(oop obj) { + + ShenandoahHeap* sh = ShenandoahHeap::heap(); + if (sh->need_update_refs()) { + obj = ShenandoahBarrierSet::resolve_oop_static(obj); + } + +#ifdef ASSERT + if (obj != ShenandoahBarrierSet::resolve_oop_static(obj)) { + ShenandoahHeap* sh = ShenandoahHeap::heap(); + } +#endif + assert(obj == ShenandoahBarrierSet::resolve_oop_static(obj), "needs to be in to-space"); + + HeapWord* addr = (HeapWord*) obj; + + if (ShenandoahTraceWeakReferences) { + + if (addr != NULL) { + if(sh->is_in(addr)) { + if (sh->is_obj_ill(obj)) { + HandleMark hm; + tty->print_cr("ShenandoahIsAliveClosure Found an ill object "PTR_FORMAT, p2i((HeapWord*) obj)); + obj->print(); + } + else + tty->print_cr("found a healthy object "PTR_FORMAT, p2i((HeapWord*) obj)); + + } else { + tty->print_cr("found an object outside the heap "PTR_FORMAT, p2i((HeapWord*) obj)); + } + } else { + tty->print_cr("found a null object "PTR_FORMAT, p2i((HeapWord*) obj)); + } + } + + return addr != NULL && sh->is_marked_current(obj); //(!sh->is_in(addr) || !sh->is_obj_ill(obj)); +} + +void ShenandoahHeap::ref_processing_init() { + MemRegion mr = reserved_region(); + + // Concurrent Mark ref processor +// _ref_processor = +// new ReferenceProcessor(mr, // span +// ParallelRefProcEnabled && (ParallelGCThreads > 1), +// // mt processing +// (int) ParallelGCThreads, +// // degree of mt processing +// (ParallelGCThreads > 1) || (ConcGCThreads > 1), +// // mt discovery +// (int) MAX2(ParallelGCThreads, ConcGCThreads), +// // degree of mt discovery +// false, +// // Reference discovery is not atomic +// &isAlive); +// // is alive closure +// // (for efficiency/performance) + _ref_processor = + new ReferenceProcessor(mr, // span + ParallelRefProcEnabled && (ConcGCThreads > 1), + // mt processing + (int) ConcGCThreads, + // degree of mt processing + (ConcGCThreads > 1), + // mt discovery + (int) ConcGCThreads, + // degree of mt discovery + false, + // Reference discovery is not atomic + &isAlive); + // is alive closure + // (for efficiency/performance) + + + +} + +#ifdef ASSERT +void ShenandoahHeap::set_from_region_protection(bool protect) { + for (uint i = 0; i < _num_regions; i++) { + ShenandoahHeapRegion* region = _ordered_regions[i]; + if (region != NULL && region->is_in_collection_set()) { + if (protect) { + region->memProtectionOn(); + } else { + region->memProtectionOff(); + } + } + } +} +#endif + +void ShenandoahHeap::acquire_pending_refs_lock() { + _concurrent_gc_thread->slt()->manipulatePLL(SurrogateLockerThread::acquirePLL); +} + +void ShenandoahHeap::release_pending_refs_lock() { + _concurrent_gc_thread->slt()->manipulatePLL(SurrogateLockerThread::releaseAndNotifyPLL); +} + +ShenandoahHeapRegion** ShenandoahHeap::heap_regions() { + return _ordered_regions; +} + +size_t ShenandoahHeap::num_regions() { + return _num_regions; +} + +size_t ShenandoahHeap::max_regions() { + return _max_regions; +} + +GCTracer* ShenandoahHeap::tracer() { + return collector_policy()->tracer(); +} + +size_t ShenandoahHeap::tlab_used(Thread* thread) const { + return _free_regions->used(); +} + +void ShenandoahHeap::cancel_concgc() { + // only report it once + if (!_cancelled_concgc) { + if (ShenandoahTracePhases) { + tty->print_cr("Cancelling GC"); + } + _cancelled_concgc = true; + OrderAccess::fence(); + _shenandoah_policy->report_concgc_cancelled(); + } + + if ((! Thread::current()->is_GC_task_thread()) && (! Thread::current()->is_ConcurrentGC_thread())) { + while (_evacuation_in_progress) { // wait. + Thread::current()->_ParkEvent->park(1) ; + } + } +} + +bool ShenandoahHeap::cancelled_concgc() { + bool cancelled = _cancelled_concgc; + return cancelled; +} + +void ShenandoahHeap::clear_cancelled_concgc() { + _cancelled_concgc = false; +} + +int ShenandoahHeap::max_workers() { + return _max_workers; +} + +int ShenandoahHeap::max_parallel_workers() { + return _max_parallel_workers; +} +int ShenandoahHeap::max_conc_workers() { + return _max_conc_workers; +} + +void ShenandoahHeap::shutdown() { + // We set this early here, to let GC threads terminate before we ask the concurrent thread + // to terminate, which would otherwise block until all GC threads come to finish normally. + _cancelled_concgc = true; + _concurrent_gc_thread->shutdown(); + cancel_concgc(); +} + +class ShenandoahStringSymbolTableUnlinkTask : public AbstractGangTask { +private: + BoolObjectClosure* _is_alive; + int _initial_string_table_size; + int _initial_symbol_table_size; + + bool _process_strings; + int _strings_processed; + int _strings_removed; + + bool _process_symbols; + int _symbols_processed; + int _symbols_removed; + +public: + ShenandoahStringSymbolTableUnlinkTask(BoolObjectClosure* is_alive, bool process_strings, bool process_symbols) : + AbstractGangTask("String/Symbol Unlinking"), + _is_alive(is_alive), + _process_strings(process_strings), _strings_processed(0), _strings_removed(0), + _process_symbols(process_symbols), _symbols_processed(0), _symbols_removed(0) { + + _initial_string_table_size = StringTable::the_table()->table_size(); + _initial_symbol_table_size = SymbolTable::the_table()->table_size(); + if (process_strings) { + StringTable::clear_parallel_claimed_index(); + } + if (process_symbols) { + SymbolTable::clear_parallel_claimed_index(); + } + } + + ~ShenandoahStringSymbolTableUnlinkTask() { + guarantee(!_process_strings || StringTable::parallel_claimed_index() >= _initial_string_table_size, + err_msg("claim value %d after unlink less than initial string table size %d", + StringTable::parallel_claimed_index(), _initial_string_table_size)); + guarantee(!_process_symbols || SymbolTable::parallel_claimed_index() >= _initial_symbol_table_size, + err_msg("claim value %d after unlink less than initial symbol table size %d", + SymbolTable::parallel_claimed_index(), _initial_symbol_table_size)); + + if (G1TraceStringSymbolTableScrubbing) { + gclog_or_tty->print_cr("Cleaned string and symbol table, " + "strings: "SIZE_FORMAT" processed, "SIZE_FORMAT" removed, " + "symbols: "SIZE_FORMAT" processed, "SIZE_FORMAT" removed", + strings_processed(), strings_removed(), + symbols_processed(), symbols_removed()); + } + } + + void work(uint worker_id) { + int strings_processed = 0; + int strings_removed = 0; + int symbols_processed = 0; + int symbols_removed = 0; + if (_process_strings) { + StringTable::possibly_parallel_unlink(_is_alive, &strings_processed, &strings_removed); + Atomic::add(strings_processed, &_strings_processed); + Atomic::add(strings_removed, &_strings_removed); + } + if (_process_symbols) { + SymbolTable::possibly_parallel_unlink(&symbols_processed, &symbols_removed); + Atomic::add(symbols_processed, &_symbols_processed); + Atomic::add(symbols_removed, &_symbols_removed); + } + } + + size_t strings_processed() const { return (size_t)_strings_processed; } + size_t strings_removed() const { return (size_t)_strings_removed; } + + size_t symbols_processed() const { return (size_t)_symbols_processed; } + size_t symbols_removed() const { return (size_t)_symbols_removed; } +}; + +void ShenandoahHeap::unlink_string_and_symbol_table(BoolObjectClosure* is_alive, bool process_strings, bool process_symbols) { + + workers()->set_active_workers(_max_parallel_workers); + ShenandoahStringSymbolTableUnlinkTask shenandoah_unlink_task(is_alive, process_strings, process_symbols); + workers()->run_task(&shenandoah_unlink_task); + + // if (G1StringDedup::is_enabled()) { + // G1StringDedup::unlink(is_alive); + // } +} + +bool ShenandoahHeap::is_obj_ill(const oop obj) const { + return ! is_marked_current(obj); +} + +void ShenandoahHeap::set_need_update_refs(bool need_update_refs) { + _need_update_refs = need_update_refs; +} + +ShenandoahJNICritical* ShenandoahHeap::jni_critical() { + return _jni_critical; +} + +ShenandoahHeapRegion* ShenandoahHeap::next_compaction_region(const ShenandoahHeapRegion* r) { + int region_idx = r->region_number() + 1; + ShenandoahHeapRegion* next = _ordered_regions[region_idx]; + guarantee(next->region_number() == region_idx, "region number must match"); + while (next->is_humongous()) { + region_idx = next->region_number() + 1; + next = _ordered_regions[region_idx]; + guarantee(next->region_number() == region_idx, "region number must match"); + } + return next; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/shenandoahHeap.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,446 @@ +/* +Copyright 2014 Red Hat, Inc. and/or its affiliates. + */ +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP + +#include "gc/shenandoah/shenandoahCollectorPolicy.hpp" +#include "gc/shenandoah/shenandoahConcurrentMark.hpp" +#include "gc/shenandoah/shenandoahConcurrentThread.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.hpp" +#include "gc/shenandoah/shenandoahHeapRegionSet.hpp" + +#include "gc/shared/cmBitMap.hpp" +#include "gc/g1/heapRegionBounds.inline.hpp" + +#include "gc/shared/barrierSet.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/space.hpp" +#include "oops/oop.hpp" +#include "oops/markOop.hpp" + + +class SpaceClosure; +class GCTracer; + +class ShenandoahJNICritical; + +class ShenandoahJNICritical; + +class ShenandoahAlwaysTrueClosure : public BoolObjectClosure { +public: + bool do_object_b(oop p) { return true; } +}; + + +class ShenandoahIsAliveClosure: public BoolObjectClosure { + +public: + bool do_object_b(oop obj); +}; + + +class ShenandoahHeapRegionClosure : public StackObj { + bool _complete; + void incomplete() {_complete = false;} + +public: + ShenandoahHeapRegionClosure(): _complete(true) {} + + // typically called on each region until it returns true; + virtual bool doHeapRegion(ShenandoahHeapRegion* r) = 0; + + bool complete() { return _complete;} +}; + +// A "ShenandoahHeap" is an implementation of a java heap for HotSpot. +// It uses a new pauseless GC algorithm based on Brooks pointers. +// Derived from G1 + +// +// CollectedHeap +// SharedHeap +// ShenandoahHeap + +class ShenandoahHeap : public CollectedHeap { + +private: + + static ShenandoahHeap* _pgc; + ShenandoahCollectorPolicy* _shenandoah_policy; + VirtualSpace _storage; + ShenandoahHeapRegion* _first_region; + HeapWord* _first_region_bottom; + // Ordered array of regions (name confusing with _regions) + ShenandoahHeapRegion** _ordered_regions; + + // Sortable array of regions + ShenandoahHeapRegionSet* _free_regions; + ShenandoahHeapRegionSet* _collection_set; + ShenandoahHeapRegion* _currentAllocationRegion; + ShenandoahConcurrentMark* _scm; + + + + ShenandoahConcurrentThread* _concurrent_gc_thread; + + size_t _num_regions; + size_t _max_regions; + size_t _initialSize; +#ifndef NDEBUG + uint _numAllocs; +#endif + WorkGangBarrierSync barrierSync; + int _max_parallel_workers; + int _max_conc_workers; + int _max_workers; + + WorkGang* _conc_workers; + WorkGang* _workers; + + + volatile size_t _used; + + CMBitMap _mark_bit_map; + CMBitMap* _next_mark_bit_map; + + bool* _in_cset_fast_test; + bool* _in_cset_fast_test_base; + uint _in_cset_fast_test_length; + + bool _cancelled_concgc; + + ShenandoahJNICritical* _jni_critical; + +public: + size_t _bytesAllocSinceCM; + size_t _bytes_allocated_during_cm; + size_t _bytes_allocated_during_cm_start; + size_t _max_allocated_gc; + size_t _allocated_last_gc; + size_t _used_start_gc; + +public: + ShenandoahHeap(ShenandoahCollectorPolicy* policy); + HeapWord* allocate_from_gclab(Thread* thread, size_t size); + HeapWord* allocate_from_gclab_slow(Thread* thread, size_t size); + HeapWord* allocate_new_tlab(size_t word_size); + HeapWord* allocate_new_gclab(size_t word_size); +private: + HeapWord* allocate_new_tlab(size_t word_size, bool mark); +public: + HeapWord* allocate_memory(size_t word_size); + + bool find_contiguous_free_regions(uint num_free_regions, ShenandoahHeapRegion** free_regions); + bool allocate_contiguous_free_regions(uint num_free_regions, ShenandoahHeapRegion** free_regions); + + // For now we are ignoring eden. + inline bool should_alloc_in_eden(size_t size) { return false;} + void print_on(outputStream* st) const ; + + ShenandoahHeap::Name kind() const { + return CollectedHeap::ShenandoahHeap; + } + + static ShenandoahHeap* heap(); + + ShenandoahCollectorPolicy *shenandoahPolicy() { return _shenandoah_policy;} + + jint initialize(); + static size_t conservative_max_heap_alignment() { + return HeapRegionBounds::max_size(); + } + + void post_initialize(); + size_t capacity() const; + size_t used() const; + bool is_maximal_no_gc() const; + size_t max_capacity() const; + virtual bool is_in(const void* p) const; + bool is_in_partial_collection(const void* p); + bool is_scavengable(const void* addr); + virtual HeapWord* mem_allocate(size_t size, bool* what); + HeapWord* mem_allocate_locked(size_t size, bool* what); + virtual size_t unsafe_max_alloc(); + bool can_elide_tlab_store_barriers() const; + virtual oop new_store_pre_barrier(JavaThread* thread, oop new_obj); + bool can_elide_initializing_store_barrier(oop new_obj); + bool card_mark_must_follow_store() const; + bool supports_heap_inspection() const; + void collect(GCCause::Cause); + void do_full_collection(bool clear_all_soft_refs); + AdaptiveSizePolicy* size_policy(); + ShenandoahCollectorPolicy* collector_policy() const; + + void ensure_parsability(bool retire_tlabs); + + void add_free_region(ShenandoahHeapRegion* r) {_free_regions->append(r);} + void clear_free_regions() {_free_regions->clear();} + + void oop_iterate(ExtendedOopClosure* cl, bool skip_dirty_regions, + bool skip_unreachable_objects); + void oop_iterate(ExtendedOopClosure* cl) { + oop_iterate(cl, false, false); + } + + void roots_iterate(ExtendedOopClosure* cl); + void weak_roots_iterate(ExtendedOopClosure* cl); + + void object_iterate(ObjectClosure* cl); + void object_iterate_careful(ObjectClosureCareful* cl); + void object_iterate_no_from_space(ObjectClosure* cl); + void safe_object_iterate(ObjectClosure* cl); + + void marked_object_iterate(ShenandoahHeapRegion* region, ObjectClosure* cl); + void marked_object_iterate_careful(ShenandoahHeapRegion* region, ObjectClosure* cl); +private: + void marked_object_iterate(ShenandoahHeapRegion* region, ObjectClosure* cl, HeapWord* start, HeapWord* limit); + +public: + HeapWord* block_start(const void* addr) const; + size_t block_size(const HeapWord* addr) const; + bool block_is_obj(const HeapWord* addr) const; + jlong millis_since_last_gc(); + void prepare_for_verify(); + void print_gc_threads_on(outputStream* st) const; + void gc_threads_do(ThreadClosure* tcl) const; + void print_tracing_info() const; + void verify(bool silent, VerifyOption vo); + bool supports_tlab_allocation() const; + virtual size_t tlab_capacity(Thread *thr) const; + void oop_iterate(MemRegion mr, ExtendedOopClosure* ecl); + void object_iterate_since_last_GC(ObjectClosure* cl); + void space_iterate(SpaceClosure* scl); + virtual size_t unsafe_max_tlab_alloc(Thread *thread) const; + virtual size_t max_tlab_size() const; + + void resize_all_tlabs(); + void accumulate_statistics_all_gclabs(); + + HeapWord* tlab_post_allocation_setup(HeapWord* obj, bool new_obj); + + uint oop_extra_words(); + +#ifndef CC_INTERP + void compile_prepare_oop(MacroAssembler* masm, Register obj = rax); +#endif + + Space* space_containing(const void* oop) const; + void gc_prologue(bool b); + void gc_epilogue(bool b); + + void heap_region_iterate(ShenandoahHeapRegionClosure* blk, bool skip_dirty_regions = false, bool skip_humongous_continuation = false) const; + ShenandoahHeapRegion* heap_region_containing(const void* addr) const; + inline uint heap_region_index_containing(const void* addr) const; + +/** + * Maybe we need that at some point... + + oop* resolve_oop_ptr(oop* p); + + oop oop_containing_oop_ptr(oop* p); + +*/ + + void temp(); + + volatile unsigned int _concurrent_mark_in_progress; + + volatile unsigned int _evacuation_in_progress; + volatile bool _update_references_in_progress; + bool _need_update_refs; + bool _need_reset_bitmaps; + + void start_concurrent_marking(); + void stop_concurrent_marking(); + ShenandoahConcurrentMark* concurrentMark() { return _scm;} + ShenandoahConcurrentThread* concurrent_thread() { return _concurrent_gc_thread; } + + ShenandoahJNICritical* jni_critical(); + + size_t bump_object_age(HeapWord* start, HeapWord* end); + + inline bool mark_current(oop obj) const; + inline bool mark_current_no_checks(oop obj) const; + inline bool is_marked_current(oop obj) const; + + ReferenceProcessor* _ref_processor; + bool is_marked_prev(oop obj) const; + + bool is_obj_ill(const oop obj) const; + + void reset_mark_bitmap(); + void reset_mark_bitmap_range(HeapWord* from, HeapWord* to); + + bool is_bitmap_clear(); + + virtual void post_allocation_collector_specific_setup(HeapWord* obj); + + void mark_object_live(oop obj, bool enqueue); + + void prepare_for_concurrent_evacuation(); + void do_evacuation(); + void parallel_evacuate(); + + void initialize_brooks_ptr(HeapWord* brooks_ptr, HeapWord* object, bool new_obj = true); + void initialize_brooks_ptr(oop p); + + inline oop maybe_update_oop_ref(oop* p); + void evacuate_region(ShenandoahHeapRegion* from_region, ShenandoahHeapRegion* to_region); + void parallel_evacuate_region(ShenandoahHeapRegion* from_region); + void verify_evacuated_region(ShenandoahHeapRegion* from_region); + + void print_heap_regions(outputStream* st = tty) const; + + void print_all_refs(const char* prefix); + + void print_heap_objects(HeapWord* start, HeapWord* end); + void print_heap_locations(HeapWord* start, HeapWord* end); + void print_heap_object(oop p); + + oop evacuate_object(oop src, Thread* thread); + bool is_in_collection_set(const void* p) { + return heap_region_containing(p)->is_in_collection_set(); + } + + void copy_object(oop p, HeapWord* s); + void verify_copy(oop p, oop c); + // void assign_brooks_pointer(oop p, HeapWord* filler, HeapWord* copy); + void verify_heap_size_consistency(); + void verify_heap_after_marking(); + void verify_heap_after_evacuation(); + void verify_heap_after_update_refs(); + void verify_regions_after_update_refs(); + + static ByteSize ordered_regions_offset() { return byte_offset_of(ShenandoahHeap, _ordered_regions); } + static ByteSize first_region_bottom_offset() { return byte_offset_of(ShenandoahHeap, _first_region_bottom); } + + void cleanup_after_cancelconcgc(); + void increase_used(size_t bytes); + void decrease_used(size_t bytes); + void set_used(size_t bytes); + + int ensure_new_regions(int num_new_regions); + + void set_evacuation_in_progress(bool in_progress); + bool is_evacuation_in_progress(); + + bool is_update_references_in_progress(); + void set_update_references_in_progress(bool update_refs_in_progress); + + inline bool need_update_refs() const; + void set_need_update_refs(bool update_refs); + + ReferenceProcessor* ref_processor() { return _ref_processor;} + virtual void ref_processing_init(); + ShenandoahIsAliveClosure isAlive; + void evacuate_and_update_roots(); + void prepare_for_update_references(); + + void update_references(); + + ShenandoahHeapRegionSet* free_regions(); + + void update_roots(); + + void acquire_pending_refs_lock(); + void release_pending_refs_lock(); + + int max_workers(); + int max_conc_workers(); + int max_parallel_workers(); + WorkGang* conc_workers() const{ return _conc_workers;} + WorkGang* workers() const{ return _workers;} + + ShenandoahHeapRegion** heap_regions(); + size_t num_regions(); + size_t max_regions(); + + ShenandoahHeapRegion* next_compaction_region(const ShenandoahHeapRegion* r); + + void recycle_dirty_regions(); + + void register_region_with_in_cset_fast_test(ShenandoahHeapRegion* r) { + assert(_in_cset_fast_test_base != NULL, "sanity"); + assert(r->is_in_collection_set(), "invariant"); + uint index = r->region_number(); + assert(index < _in_cset_fast_test_length, "invariant"); + assert(!_in_cset_fast_test_base[index], "invariant"); + _in_cset_fast_test_base[index] = true; + } + bool in_cset_fast_test(HeapWord* obj) { + assert(_in_cset_fast_test != NULL, "sanity"); + if (is_in(obj)) { + // no need to subtract the bottom of the heap from obj, + // _in_cset_fast_test is biased + uintx index = ((uintx) obj) >> ShenandoahHeapRegion::RegionSizeShift; + bool ret = _in_cset_fast_test[index]; + // let's make sure the result is consistent with what the slower + // test returns + assert( ret || !is_in_collection_set(obj), "sanity"); + assert(!ret || is_in_collection_set(obj), "sanity"); + return ret; + } else { + return false; + } + } + + static address in_cset_fast_test_addr() { + return (address) (ShenandoahHeap::heap()->_in_cset_fast_test); + } + + void clear_cset_fast_test() { + assert(_in_cset_fast_test_base != NULL, "sanity"); + memset(_in_cset_fast_test_base, false, + (size_t) _in_cset_fast_test_length * sizeof(bool)); + } + + GCTracer* tracer(); + ShenandoahHeapRegionSet* collection_set() { return _collection_set; } + size_t tlab_used(Thread* ignored) const; + +private: + + bool grow_heap_by(); + + void verify_evacuation(ShenandoahHeapRegion* from_region); + void set_concurrent_mark_in_progress(bool in_progress); + + void oom_during_evacuation(); + void cancel_concgc(); +public: + bool cancelled_concgc(); + void clear_cancelled_concgc(); + + void shutdown(); + + bool concurrent_mark_in_progress(); + size_t calculateUsed(); + size_t calculateFree(); + +private: + void verify_live(); + void verify_liveness_after_concurrent_mark(); + + HeapWord* allocate_memory_with_lock(size_t word_size); + HeapWord* allocate_memory_heap_lock(size_t word_size); + HeapWord* allocate_memory_shenandoah_lock(size_t word_size); + HeapWord* allocate_memory_work(size_t word_size); + HeapWord* allocate_large_memory(size_t word_size); + ShenandoahHeapRegion* check_skip_humongous(ShenandoahHeapRegion* region) const; + ShenandoahHeapRegion* get_next_region_skip_humongous() const; + ShenandoahHeapRegion* get_current_region_skip_humongous() const; + ShenandoahHeapRegion* check_grow_heap(ShenandoahHeapRegion* current); + ShenandoahHeapRegion* get_next_region(); + ShenandoahHeapRegion* get_current_region(); + + void set_from_region_protection(bool protect); + +public: + // Delete entries for dead interned string and clean up unreferenced symbols + // in symbol table, possibly in parallel. + void unlink_string_and_symbol_table(BoolObjectClosure* is_alive, bool unlink_strings = true, bool unlink_symbols = true); + +}; + +#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/shenandoahHeap.inline.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,102 @@ +/* +Copyright 2015 Red Hat, Inc. and/or its affiliates. + */ + +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_INLINE_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_INLINE_HPP + +#include "gc/shared/cmBitMap.inline.hpp" +#include "gc/shenandoah/shenandoahBarrierSet.hpp" +#include "gc/shenandoah/shenandoahHeap.hpp" +#include "runtime/atomic.inline.hpp" + +/* + * Marks the object. Returns true if the object has not been marked before and has + * been marked by this thread. Returns false if the object has already been marked, + * or if a competing thread succeeded in marking this object. + */ +inline bool ShenandoahHeap::mark_current(oop obj) const { +#ifdef ASSERT + if (obj != oopDesc::bs()->resolve_oop(obj)) { + tty->print_cr("heap region containing obj:"); + ShenandoahHeapRegion* obj_region = heap_region_containing(obj); + obj_region->print(); + tty->print_cr("heap region containing forwardee:"); + ShenandoahHeapRegion* forward_region = heap_region_containing(oopDesc::bs()->resolve_oop(obj)); + forward_region->print(); + } +#endif + + assert(obj == oopDesc::bs()->resolve_oop(obj), "only mark forwarded copy of objects"); + return mark_current_no_checks(obj); +} + +inline bool ShenandoahHeap::mark_current_no_checks(oop obj) const { + return _next_mark_bit_map->parMark((HeapWord*) obj); +} + +inline bool ShenandoahHeap::is_marked_current(oop obj) const { + return _next_mark_bit_map->isMarked((HeapWord*) obj); +} + +inline bool ShenandoahHeap::need_update_refs() const { + return _need_update_refs; +} + +inline uint ShenandoahHeap::heap_region_index_containing(const void* addr) const { + uintptr_t region_start = ((uintptr_t) addr); // & ~(ShenandoahHeapRegion::RegionSizeBytes - 1); + uintptr_t index = (region_start - (uintptr_t) _first_region_bottom) >> ShenandoahHeapRegion::RegionSizeShift; +#ifdef ASSERT + if (!(index < _num_regions)) { + tty->print_cr("heap region does not contain address, first_region_bottom: "PTR_FORMAT", real bottom of first region: "PTR_FORMAT", num_regions: "SIZE_FORMAT", region_size: "SIZE_FORMAT, p2i(_first_region_bottom), p2i(_ordered_regions[0]->bottom()), _num_regions, ShenandoahHeapRegion::RegionSizeBytes); + } +#endif + assert(index < _num_regions, "heap region index must be in range"); + return index; +} + +oop ShenandoahHeap::maybe_update_oop_ref(oop* p) { + + assert((! is_in(p)) || (! heap_region_containing(p)->is_in_collection_set()), + "never update refs in from-space, unless evacuation has been cancelled"); + + oop heap_oop = oopDesc::load_heap_oop(p); // read p + if (! oopDesc::is_null(heap_oop)) { + +#ifdef ASSERT + if (! is_in(heap_oop)) { + print_heap_regions(); + tty->print_cr("object not in heap: "PTR_FORMAT", referenced by: "PTR_FORMAT, p2i((HeapWord*) heap_oop), p2i(p)); + assert(is_in(heap_oop), "object must be in heap"); + } +#endif + assert(is_in(heap_oop), "only ever call this on objects in the heap"); + assert((! (is_in(p) && heap_region_containing(p)->is_in_collection_set())), "we don't want to update references in from-space"); + oop forwarded_oop = ShenandoahBarrierSet::resolve_oop_static_not_null(heap_oop); // read brooks ptr + if (forwarded_oop != heap_oop) { + // tty->print_cr("updating old ref: "PTR_FORMAT" pointing to "PTR_FORMAT" to new ref: "PTR_FORMAT, p2i(p), p2i(heap_oop), p2i(forwarded_oop)); + assert(forwarded_oop->is_oop(), "oop required"); + assert(is_in(forwarded_oop), "forwardee must be in heap"); + assert(! heap_region_containing(forwarded_oop)->is_in_collection_set(), "forwardee must not be in collection set"); + // If this fails, another thread wrote to p before us, it will be logged in SATB and the + // reference be updated later. + oop result = (oop) Atomic::cmpxchg_ptr(forwarded_oop, p, heap_oop); + + if (result == heap_oop) { // CAS successful. + return forwarded_oop; + } else { + return result; + } + } else { + return forwarded_oop; + } + /* + else { + tty->print_cr("not updating ref: "PTR_FORMAT, p2i(heap_oop)); + } + */ + } + return NULL; +} + +#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_INLINE_HPP
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/shenandoahHeapRegion.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,339 @@ +/* +Copyright 2014 Red Hat, Inc. and/or its affiliates. + */ + +#include "memory/allocation.hpp" +#include "gc/g1/heapRegionBounds.inline.hpp" +#include "gc/shenandoah/brooksPointer.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shared/space.inline.hpp" +#include "memory/universe.hpp" +#include "runtime/mutexLocker.hpp" +#include "runtime/os.hpp" + +size_t ShenandoahHeapRegion::RegionSizeShift = 0; +size_t ShenandoahHeapRegion::RegionSizeBytes = 0; + +jint ShenandoahHeapRegion::initialize_heap_region(HeapWord* start, + size_t regionSizeWords, int index) { + + reserved = MemRegion((HeapWord*) start, regionSizeWords); + ContiguousSpace::initialize(reserved, true, false); + liveData = 0; + _is_in_collection_set = false; + _region_number = index; +#ifdef ASSERT + _mem_protection_level = 1; // Off, level 1. +#endif + return JNI_OK; +} + +int ShenandoahHeapRegion::region_number() const { + return _region_number; +} + +bool ShenandoahHeapRegion::rollback_allocation(uint size) { + set_top(top() - size); + return true; +} + +void ShenandoahHeapRegion::clearLiveData() { + setLiveData(0); +} + +void ShenandoahHeapRegion::setLiveData(size_t s) { + Atomic::store_ptr(s, (intptr_t*) &liveData); +} + +void ShenandoahHeapRegion::increase_live_data(size_t s) { + size_t new_live_data = Atomic::add(s, &liveData); + assert(new_live_data <= used() || is_humongous(), "can't have more live data than used"); +} + +size_t ShenandoahHeapRegion::getLiveData() const { + return liveData; +} + +size_t ShenandoahHeapRegion::garbage() const { + assert(used() >= getLiveData() || is_humongous(), err_msg("Live Data must be a subset of used() live: "SIZE_FORMAT" used: "SIZE_FORMAT, getLiveData(), used())); + size_t result = used() - getLiveData(); + return result; +} + +bool ShenandoahHeapRegion::is_in_collection_set() const { + return _is_in_collection_set; +} + +#include <sys/mman.h> + +#ifdef ASSERT + +void ShenandoahHeapRegion::memProtectionOn() { + /* + tty->print_cr("protect memory on region level: "INT32_FORMAT, _mem_protection_level); + print(tty); + */ + MutexLockerEx ml(ShenandoahMemProtect_lock, true); + assert(_mem_protection_level >= 1, "invariant"); + + if (--_mem_protection_level == 0) { + if (ShenandoahVerifyWritesToFromSpace) { + assert(! ShenandoahVerifyReadsToFromSpace, "can't verify from-space reads when verifying from-space writes"); + os::protect_memory((char*) bottom(), end() - bottom(), os::MEM_PROT_READ); + } else { + assert(ShenandoahVerifyReadsToFromSpace, "need to be verifying reads here"); + assert(! ShenandoahConcurrentEvacuation, "concurrent evacuation needs to be turned off for verifying from-space-reads"); + os::protect_memory((char*) bottom(), end() - bottom(), os::MEM_PROT_NONE); + } + } +} + +void ShenandoahHeapRegion::memProtectionOff() { + /* + tty->print_cr("unprotect memory on region level: "INT32_FORMAT, _mem_protection_level); + print(tty); + */ + MutexLockerEx ml(ShenandoahMemProtect_lock, true); + assert(_mem_protection_level >= 0, "invariant"); + if (_mem_protection_level++ == 0) { + os::protect_memory((char*) bottom(), end() - bottom(), os::MEM_PROT_RW); + } +} + +#endif + +void ShenandoahHeapRegion::set_is_in_collection_set(bool b) { + assert(! (is_humongous() && b), "never ever enter a humongous region into the collection set"); + + _is_in_collection_set = b; + + if (b) { + // tty->print_cr("registering region in fast-cset"); + // print(); + ShenandoahHeap::heap()->register_region_with_in_cset_fast_test(this); + } + +#ifdef ASSERT + if (ShenandoahVerifyWritesToFromSpace || ShenandoahVerifyReadsToFromSpace) { + if (b) { + memProtectionOn(); + assert(_mem_protection_level == 0, "need to be protected here"); + } else { + assert(_mem_protection_level == 0, "need to be protected here"); + memProtectionOff(); + } + } +#endif +} + +ByteSize ShenandoahHeapRegion::is_in_collection_set_offset() { + return byte_offset_of(ShenandoahHeapRegion, _is_in_collection_set); +} + +void ShenandoahHeapRegion::print_on(outputStream* st) const { + st->print_cr("ShenandoahHeapRegion: "PTR_FORMAT"/"INT32_FORMAT, p2i(this), _region_number); + + if (is_in_collection_set()) + st->print("C"); + if (is_humongous_start()) { + st->print("H"); + } + if (is_humongous_continuation()) { + st->print("h"); + } + //else + st->print(" "); + + st->print_cr("live = "SIZE_FORMAT" garbage = "SIZE_FORMAT" bottom = "PTR_FORMAT" end = "PTR_FORMAT" top = "PTR_FORMAT, + getLiveData(), garbage(), p2i(bottom()), p2i(end()), p2i(top())); +} + + +class SkipUnreachableObjectToOopClosure: public ObjectClosure { + ExtendedOopClosure* _cl; + bool _skip_unreachable_objects; + ShenandoahHeap* _heap; + +public: + SkipUnreachableObjectToOopClosure(ExtendedOopClosure* cl, bool skip_unreachable_objects) : + _cl(cl), _skip_unreachable_objects(skip_unreachable_objects), _heap(ShenandoahHeap::heap()) {} + + void do_object(oop obj) { + + if ((! _skip_unreachable_objects) || _heap->is_marked_current(obj)) { + if (_skip_unreachable_objects) { + assert(_heap->is_marked_current(obj), "obj must be live"); + } + obj->oop_iterate(_cl); + } + + } +}; + +void ShenandoahHeapRegion::object_iterate_interruptible(ObjectClosure* blk, bool allow_cancel) { + HeapWord* p = bottom() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; + ShenandoahHeap* heap = ShenandoahHeap::heap(); + while (p < top() && !(allow_cancel && heap->cancelled_concgc())) { + blk->do_object(oop(p)); +#ifdef ASSERT + if (ShenandoahVerifyReadsToFromSpace) { + memProtectionOff(); + p += oop(p)->size() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; + memProtectionOn(); + } else { + p += oop(p)->size() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; + } +#else + p += oop(p)->size() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; +#endif + } +} + +HeapWord* ShenandoahHeapRegion::object_iterate_careful(ObjectClosureCareful* blk) { + HeapWord * limit = concurrent_iteration_safe_limit(); + assert(limit <= top(), "sanity check"); + for (HeapWord* p = bottom() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; p < limit;) { + size_t size = blk->do_object_careful(oop(p)); + if (size == 0) { + return p; // failed at p + } else { + p += size + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; + } + } + return NULL; // all done +} + +void ShenandoahHeapRegion::oop_iterate_skip_unreachable(ExtendedOopClosure* cl, bool skip_unreachable_objects) { + SkipUnreachableObjectToOopClosure cl2(cl, skip_unreachable_objects); + object_iterate_interruptible(&cl2, false); +} + +void ShenandoahHeapRegion::fill_region() { + ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap(); + + if (free() > (BrooksPointer::BROOKS_POINTER_OBJ_SIZE + CollectedHeap::min_fill_size())) { + HeapWord* filler = allocate(BrooksPointer::BROOKS_POINTER_OBJ_SIZE); + HeapWord* obj = allocate(end() - top()); + sh->fill_with_object(obj, end() - obj); + sh->initialize_brooks_ptr(filler, obj); + } +} + +void ShenandoahHeapRegion::set_humongous_start(bool start) { + _humongous_start = start; +} + +void ShenandoahHeapRegion::set_humongous_continuation(bool continuation) { + _humongous_continuation = continuation; +} + +bool ShenandoahHeapRegion::is_humongous() const { + return _humongous_start || _humongous_continuation; +} + +bool ShenandoahHeapRegion::is_humongous_start() const { + return _humongous_start; +} + +bool ShenandoahHeapRegion::is_humongous_continuation() const { + return _humongous_continuation; +} + +void ShenandoahHeapRegion::do_reset() { + ContiguousSpace::initialize(reserved, true, false); + clearLiveData(); + _humongous_start = false; + _humongous_continuation = false; +} + +void ShenandoahHeapRegion::recycle() { + do_reset(); + set_is_in_collection_set(false); +} + +void ShenandoahHeapRegion::reset() { + assert(_mem_protection_level == 1, "needs to be unprotected here"); + do_reset(); + _is_in_collection_set = false; +} + +HeapWord* ShenandoahHeapRegion::block_start_const(const void* p) const { + assert(MemRegion(bottom(), end()).contains(p), + err_msg("p ("PTR_FORMAT") not in space ["PTR_FORMAT", "PTR_FORMAT")", + p2i(p), p2i(bottom()), p2i(end()))); + if (p >= top()) { + return top(); + } else { + HeapWord* last = bottom() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; + HeapWord* cur = last; + while (cur <= p) { + last = cur; + cur += oop(cur)->size() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; + } + assert(oop(last)->is_oop(), + err_msg(PTR_FORMAT" should be an object start", p2i(last))); + return last; + } +} + +void ShenandoahHeapRegion::setup_heap_region_size(size_t initial_heap_size, size_t max_heap_size) { + uintx region_size = ShenandoahHeapRegionSize; + if (FLAG_IS_DEFAULT(ShenandoahHeapRegionSize)) { + size_t average_heap_size = (initial_heap_size + max_heap_size) / 2; + region_size = MAX2(average_heap_size / HeapRegionBounds::target_number(), + (uintx) HeapRegionBounds::min_size()); + } + + int region_size_log = log2_long((jlong) region_size); + // Recalculate the region size to make sure it's a power of + // 2. This means that region_size is the largest power of 2 that's + // <= what we've calculated so far. + region_size = ((uintx)1 << region_size_log); + + // Now make sure that we don't go over or under our limits. + if (region_size < HeapRegionBounds::min_size()) { + region_size = HeapRegionBounds::min_size(); + } else if (region_size > HeapRegionBounds::max_size()) { + region_size = HeapRegionBounds::max_size(); + } + + // And recalculate the log. + region_size_log = log2_long((jlong) region_size); + + // Now, set up the globals. + guarantee(RegionSizeShift == 0, "we should only set it once"); + RegionSizeShift = region_size_log; + + guarantee(RegionSizeBytes == 0, "we should only set it once"); + RegionSizeBytes = (size_t)region_size; + + if (ShenandoahLogConfig) { + tty->print_cr("Region size in bytes: "SIZE_FORMAT, RegionSizeBytes); + tty->print_cr("Region size shift: "SIZE_FORMAT, RegionSizeShift); + tty->print_cr("Initial number of regions: "SIZE_FORMAT, initial_heap_size / RegionSizeBytes); + tty->print_cr("Maximum number of regions: "SIZE_FORMAT, max_heap_size / RegionSizeBytes); + } +} + +CompactibleSpace* ShenandoahHeapRegion::next_compaction_space() const { + return ShenandoahHeap::heap()->next_compaction_region(this); +} + +void ShenandoahHeapRegion::prepare_for_compaction(CompactPoint* cp) { + scan_and_forward(this, cp); +} + +void ShenandoahHeapRegion::adjust_pointers() { + // Check first is there is any work to do. + if (used() == 0) { + return; // Nothing to do. + } + + scan_and_adjust_pointers(this); +} + +void ShenandoahHeapRegion::compact() { + scan_and_compact(this); +} +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/shenandoahHeapRegion.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,142 @@ +/* +Copyright 2014 Red Hat, Inc. and/or its affiliates. + */ +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAPREGION_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAPREGION_HPP + +#include "gc/shared/space.hpp" +#include "memory/universe.hpp" +#include "utilities/sizes.hpp" + +class ShenandoahHeapRegion : public ContiguousSpace { + + // Allow scan_and_forward to call (private) overrides for auxiliary functions on this class + template <typename SpaceType> + friend void CompactibleSpace::scan_and_forward(SpaceType* space, CompactPoint* cp); + template <typename SpaceType> + friend void CompactibleSpace::scan_and_adjust_pointers(SpaceType* space); + template <typename SpaceType> + friend void CompactibleSpace::scan_and_compact(SpaceType* space); + +private: + // Auxiliary functions for scan_and_forward support. + // See comments for CompactibleSpace for more information. + inline HeapWord* scan_limit() const { + return top(); + } + + inline bool scanned_block_is_obj(const HeapWord* addr) const { + return true; // Always true, since scan_limit is top + } + + inline size_t scanned_block_size(const HeapWord* addr) const { + oop obj = oop(addr+1); + size_t size = obj->size() + 1; + return size; + } + + // Auxiliary functions for scan_and_{forward,adjust_pointers,compact} support. + inline size_t adjust_obj_size(size_t size) const { + return size + 1; + } + + inline size_t obj_size(const HeapWord* addr) const { + return oop(addr+1)->size() + 1; + } + + inline oop make_oop(HeapWord* addr) const { + return oop(addr+1); + } +public: + static size_t RegionSizeBytes; + static size_t RegionSizeShift; + +private: + int _region_number; + volatile size_t liveData; + MemRegion reserved; + bool _is_in_collection_set; + + bool _humongous_start; + bool _humongous_continuation; + +#ifdef ASSERT + int _mem_protection_level; +#endif + +public: + static void setup_heap_region_size(size_t initial_heap_size, size_t max_heap_size); + + jint initialize_heap_region(HeapWord* start, size_t regionSize, int index); + + + int region_number() const; + + // Roll back the previous allocation of an object with specified size. + // Returns TRUE when successful, FALSE if not successful or not supported. + bool rollback_allocation(uint size); + + void clearLiveData(); + void setLiveData(size_t s); + void increase_live_data(size_t s); + + size_t getLiveData() const; + + void print_on(outputStream* st) const; + + size_t garbage() const; + + void recycle(); + void reset(); + + void oop_iterate_skip_unreachable(ExtendedOopClosure* cl, bool skip_unreachable_objects); + + void object_iterate_interruptible(ObjectClosure* blk, bool allow_cancel); + + HeapWord* object_iterate_careful(ObjectClosureCareful* cl); + + HeapWord* block_start_const(const void* p) const; + + // Just before GC we need to fill the current region. + void fill_region(); + + bool is_in_collection_set() const; + + void set_is_in_collection_set(bool b); + + void set_humongous_start(bool start); + void set_humongous_continuation(bool continuation); + + bool is_humongous() const; + bool is_humongous_start() const; + bool is_humongous_continuation() const; + +#ifdef ASSERT + void memProtectionOn(); + void memProtectionOff(); +#endif + + static ByteSize is_in_collection_set_offset(); + // The following are for humongous regions. We need to save the + markOop saved_mark_word; + void save_mark_word(oop obj) {saved_mark_word = obj->mark();} + markOop mark_word() {return saved_mark_word;} + + virtual CompactibleSpace* next_compaction_space() const; + + // Override for scan_and_forward support. + void prepare_for_compaction(CompactPoint* cp); + void adjust_pointers(); + void compact(); + + virtual oop compact_oop(HeapWord* addr) const { + return oop(addr + 1); + } +private: + void do_reset(); + +}; + + + +#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAPREGION_HPP
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/shenandoahHeapRegionSet.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,354 @@ +/* +Copyright 2014 Red Hat, Inc. and/or its affiliates. + */ +#include "gc/shenandoah/brooksPointer.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.hpp" +#include "gc/shenandoah/shenandoahHeapRegionSet.hpp" +#include "gc/shenandoah/shenandoahHumongous.hpp" +#include "memory/resourceArea.hpp" +#include "utilities/quickSort.hpp" + +ShenandoahHeapRegionSet::ShenandoahHeapRegionSet(size_t max_regions) : + _max_regions(max_regions), + _regions(NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, max_regions, mtGC)), + _garbage_threshold(ShenandoahHeapRegion::RegionSizeBytes / 2), + _free_threshold(ShenandoahHeapRegion::RegionSizeBytes / 2), + _available(0), _used(0) +{ + + _next = &_regions[0]; + _current = NULL; + _next_free = &_regions[0]; +} + +ShenandoahHeapRegionSet::ShenandoahHeapRegionSet(size_t max_regions, ShenandoahHeapRegion** regions, size_t num_regions) : + _max_regions(num_regions), + _regions(NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, max_regions, mtGC)), + _garbage_threshold(ShenandoahHeapRegion::RegionSizeBytes / 2), + _free_threshold(ShenandoahHeapRegion::RegionSizeBytes / 2) { + + // Make copy of the regions array so that we can sort without destroying the original. + memcpy(_regions, regions, sizeof(ShenandoahHeapRegion*) * num_regions); + + _next = &_regions[0]; + _current = NULL; + _next_free = &_regions[num_regions]; +} + +ShenandoahHeapRegionSet::~ShenandoahHeapRegionSet() { + FREE_C_HEAP_ARRAY(ShenandoahHeapRegion*, _regions); +} + +int compareHeapRegionsByGarbage(ShenandoahHeapRegion* a, ShenandoahHeapRegion* b) { + if (a == NULL) { + if (b == NULL) { + return 0; + } else { + return 1; + } + } else if (b == NULL) { + return -1; + } + + size_t garbage_a = a->garbage(); + size_t garbage_b = b->garbage(); + + if (garbage_a > garbage_b) + return -1; + else if (garbage_a < garbage_b) + return 1; + else return 0; +} + +ShenandoahHeapRegion* ShenandoahHeapRegionSet::current() { + ShenandoahHeapRegion** current = _current; + if (current == NULL) { + return get_next(); + } else { + return *(limit_region(current)); + } +} + +size_t ShenandoahHeapRegionSet::length() { + return _next_free - _regions; +} + +size_t ShenandoahHeapRegionSet::available_regions() { + return (_regions + _max_regions) - _next_free; +} + +void ShenandoahHeapRegionSet::append(ShenandoahHeapRegion* region) { + assert(_next_free < _regions + _max_regions, "need space for additional regions"); + assert(SafepointSynchronize::is_at_safepoint() || ShenandoahHeap_lock->owned_by_self() || ! Universe::is_fully_initialized(), "only append regions to list while world is stopped"); + + // Grab next slot. + ShenandoahHeapRegion** next_free = _next_free; + _next_free++; + + // Insert new region into slot. + *next_free = region; + + _available += region->free(); +} + +void ShenandoahHeapRegionSet::clear() { + _current = NULL; + _next = _regions; + _next_free = _regions; + _available = 0; + _used = 0; +} + +ShenandoahHeapRegion* ShenandoahHeapRegionSet::claim_next() { + ShenandoahHeapRegion** next = (ShenandoahHeapRegion**) Atomic::add_ptr(sizeof(ShenandoahHeapRegion**), &_next); + next--; + if (next < _next_free) { + return *next; + } else { + return NULL; + } +} + +ShenandoahHeapRegion* ShenandoahHeapRegionSet::get_next() { + + ShenandoahHeapRegion** next = _next; + if (next < _next_free) { + _current = next; + _next++; + return *next; + } else { + return NULL; + } +} + +ShenandoahHeapRegion** ShenandoahHeapRegionSet::limit_region(ShenandoahHeapRegion** region) { + if (region >= _next_free) { + return NULL; + } else { + return region; + } +} + +void ShenandoahHeapRegionSet::print() { + for (ShenandoahHeapRegion** i = _regions; i < _next_free; i++) { + if (i == _current) { + tty->print_cr("C->"); + } + if (i == _next) { + tty->print_cr("N->"); + } + (*i)->print(); + } +} + +void ShenandoahHeapRegionSet::choose_collection_and_free_sets(ShenandoahHeapRegionSet* col_set, ShenandoahHeapRegionSet* free_set) { + col_set->choose_collection_set(_regions, length()); + free_set->choose_free_set(_regions, length()); + // assert(col_set->length() > 0 && free_set->length() > 0, "Better have some regions in the collection and free sets"); + +} + +void ShenandoahHeapRegionSet::choose_collection_and_free_sets_min_garbage(ShenandoahHeapRegionSet* col_set, ShenandoahHeapRegionSet* free_set, size_t min_garbage) { + col_set->choose_collection_set_min_garbage(_regions, length(), min_garbage); + free_set->choose_free_set(_regions, length()); + // assert(col_set->length() > 0 && free_set->length() > 0, "Better have some regions in the collection and free sets"); +} + +void ShenandoahHeapRegionSet::choose_collection_set(ShenandoahHeapRegion** regions, size_t length) { + + clear(); + + assert(length <= _max_regions, "must not blow up array"); + + ShenandoahHeapRegion** tmp = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, length, mtGC); + + memcpy(tmp, regions, sizeof(ShenandoahHeapRegion*) * length); + + QuickSort::sort<ShenandoahHeapRegion*>(tmp, length, compareHeapRegionsByGarbage, false); + + ShenandoahHeapRegion** r = tmp; + ShenandoahHeapRegion** end = tmp + length; + + // We don't want the current allocation region in the collection set because a) it is still being allocated into and b) This is where the write barriers will allocate their copies. + + while (r < end) { + ShenandoahHeapRegion* region = *r; + if (region->garbage() > _garbage_threshold && ! region->is_humongous()) { + // tty->print("choose region %d with garbage = " SIZE_FORMAT " and live = " SIZE_FORMAT " and _garbage_threshold = " SIZE_FORMAT "\n", + // region->region_number(), region->garbage(), region->getLiveData(), _garbage_threshold); + + assert(! region->is_humongous(), "no humongous regions in collection set"); + + if (region->getLiveData() == 0) { + // We can recycle it right away and put it in the free set. + ShenandoahHeap::heap()->decrease_used(region->used()); + region->recycle(); + } else { + append(region); + region->set_is_in_collection_set(true); + } + // } else { + // tty->print("rejected region %d with garbage = " SIZE_FORMAT " and live = " SIZE_FORMAT " and _garbage_threshold = " SIZE_FORMAT "\n", + // region->region_number(), region->garbage(), region->getLiveData(), _garbage_threshold); + } + r++; + } + + FREE_C_HEAP_ARRAY(ShenandoahHeapRegion*, tmp); + +} + +void ShenandoahHeapRegionSet::choose_collection_set_min_garbage(ShenandoahHeapRegion** regions, size_t length, size_t min_garbage) { + + clear(); + + assert(length <= _max_regions, "must not blow up array"); + + ShenandoahHeapRegion** tmp = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, length, mtGC); + + memcpy(tmp, regions, sizeof(ShenandoahHeapRegion*) * length); + + QuickSort::sort<ShenandoahHeapRegion*>(tmp, length, compareHeapRegionsByGarbage, false); + + ShenandoahHeapRegion** r = tmp; + ShenandoahHeapRegion** end = tmp + length; + + // We don't want the current allocation region in the collection set because a) it is still being allocated into and b) This is where the write barriers will allocate their copies. + + size_t garbage = 0; + while (r < end && garbage < min_garbage) { + ShenandoahHeapRegion* region = *r; + if (region->garbage() > _garbage_threshold && ! region->is_humongous()) { + append(region); + garbage += region->garbage(); + region->set_is_in_collection_set(true); + } + r++; + } + + FREE_C_HEAP_ARRAY(ShenandoahHeapRegion*, tmp); + + /* + tty->print_cr("choosen region with "SIZE_FORMAT" garbage given "SIZE_FORMAT" min_garbage", garbage, min_garbage); + */ +} + + +void ShenandoahHeapRegionSet::choose_free_set(ShenandoahHeapRegion** regions, size_t length) { + + clear(); + ShenandoahHeapRegion** end = regions + length; + + for (ShenandoahHeapRegion** r = regions; r < end; r++) { + ShenandoahHeapRegion* region = *r; + if ((! region->is_in_collection_set()) + && (! region->is_humongous())) { + append(region); + } + } +} + +void ShenandoahHeapRegionSet::reclaim_humongous_regions() { + + ShenandoahHeap* heap = ShenandoahHeap::heap(); + for (ShenandoahHeapRegion** r = _regions; r < _next_free; r++) { + // We can immediately reclaim humongous objects/regions that are no longer reachable. + ShenandoahHeapRegion* region = *r; + if (region->is_humongous_start()) { + oop humongous_obj = oop(region->bottom() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE); + if (! heap->is_marked_current(humongous_obj)) { + reclaim_humongous_region_at(r); + } + } + } + +} + +void ShenandoahHeapRegionSet::reclaim_humongous_region_at(ShenandoahHeapRegion** r) { + assert((*r)->is_humongous_start(), "reclaim regions starting with the first one"); + + oop humongous_obj = oop((*r)->bottom() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE); + size_t size = humongous_obj->size() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; + uint required_regions = ShenandoahHumongous::required_regions(size * HeapWordSize); + + if (ShenandoahTraceHumongous) { + tty->print_cr("reclaiming "UINT32_FORMAT" humongous regions for object of size: "SIZE_FORMAT" words", required_regions, size); + } + + assert((*r)->getLiveData() == 0, "liveness must be zero"); + + for (ShenandoahHeapRegion** i = r; i < r + required_regions; i++) { + ShenandoahHeapRegion* region = *i; + + assert(i == r ? region->is_humongous_start() : region->is_humongous_continuation(), + "expect correct humongous start or continuation"); + + if (ShenandoahTraceHumongous) { + region->print(); + } + + region->reset(); + ShenandoahHeap::heap()->decrease_used(ShenandoahHeapRegion::RegionSizeBytes); + } +} + +void ShenandoahHeapRegionSet::set_concurrent_iteration_safe_limits() { + for (ShenandoahHeapRegion** i = _regions; i < _next_free; i++) { + ShenandoahHeapRegion* region = *i; + region->set_concurrent_iteration_safe_limit(region->top()); + } +} + +size_t ShenandoahHeapRegionSet::garbage() { + size_t garbage = 0; + for (ShenandoahHeapRegion** i = _regions; i < _next_free; i++) { + ShenandoahHeapRegion* region = *i; + garbage += region->garbage(); + } + return garbage; +} + +size_t ShenandoahHeapRegionSet::used() { + size_t used = 0; + for (ShenandoahHeapRegion** i = _regions; i < _next_free; i++) { + ShenandoahHeapRegion* region = *i; + used += region->used(); + } + return used; +} + +size_t ShenandoahHeapRegionSet::live_data() { + size_t live = 0; + for (ShenandoahHeapRegion** i = _regions; i < _next_free; i++) { + ShenandoahHeapRegion* region = *i; + live += region->getLiveData(); + } + return live; +} + +void ShenandoahHeapRegionSet::decrease_available(size_t num_bytes) { + assert(_available >= num_bytes, "can't use more than available"); + _available -= num_bytes; + _used += num_bytes; +} + +size_t ShenandoahHeapRegionSet::available() const { +#ifdef ASSERT + // Need to grab the lock here, because the different counters are updated + // within the lock. Otherwise we sometimes get inconsistent results. + { + MutexLockerEx ml(ShenandoahHeap_lock, true); + assert(ShenandoahHeap::heap()->capacity() - ShenandoahHeap::heap()->used()>= _available, + err_msg("must not be > heap free, capacity: "SIZE_FORMAT", used: "SIZE_FORMAT", available: "SIZE_FORMAT, + ShenandoahHeap::heap()->capacity(), ShenandoahHeap::heap()->used(), _available) + ); + } +#endif + return _available; +} + +size_t ShenandoahHeapRegionSet::used() const { + assert(ShenandoahHeap::heap()->used() >= _used, "must not be > heap used"); + return _used; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/shenandoahHeapRegionSet.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,102 @@ +/* +Copyright 2014 Red Hat, Inc. and/or its affiliates. + */ +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAPREGIONSET_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAPREGIONSET_HPP + +#include "gc/shenandoah/shenandoahHeapRegion.hpp" + + +class ShenandoahHeapRegionSet : public CHeapObj<mtGC> { +private: + ShenandoahHeapRegion** _regions; + // current region to be returned from get_next() + ShenandoahHeapRegion** _current; + ShenandoahHeapRegion** _next; + + // last inserted region. + ShenandoahHeapRegion** _next_free; + ShenandoahHeapRegion** _concurrent_next_free; + + // Maximum size of the set. + const size_t _max_regions; + + size_t _garbage_threshold; + size_t _free_threshold; + + size_t _available; + size_t _used; + + void choose_collection_set(ShenandoahHeapRegion** regions, size_t length); + void choose_collection_set_min_garbage(ShenandoahHeapRegion** regions, size_t length, size_t min_garbage); + void choose_free_set(ShenandoahHeapRegion** regions, size_t length); + +public: + ShenandoahHeapRegionSet(size_t max_regions); + + ShenandoahHeapRegionSet(size_t max_regions, ShenandoahHeapRegion** regions, size_t num_regions); + + ~ShenandoahHeapRegionSet(); + + void set_garbage_threshold(size_t minimum_garbage) { _garbage_threshold = minimum_garbage;} + void set_free_threshold(size_t minimum_free) { _free_threshold = minimum_free;} + + /** + * Appends a region to the set. This is implemented to be concurrency-safe. + */ + void append(ShenandoahHeapRegion* region); + + void clear(); + + size_t length(); + size_t used_regions() { + return _current - _regions; + } + size_t available_regions(); + void print(); + + size_t garbage(); + size_t used(); + size_t live_data(); + size_t reclaimed() {return _reclaimed;} + + /** + * Returns a pointer to the current region. + */ + ShenandoahHeapRegion* current(); + + /** + * Gets the next region for allocation (from free-list). + * If multiple threads are competing, one will succeed to + * increment to the next region, the others will fail and return + * the region that the succeeding thread got. + */ + ShenandoahHeapRegion* get_next(); + + /** + * Claims next region for processing. This is implemented to be concurrency-safe. + */ + ShenandoahHeapRegion* claim_next(); + + void choose_collection_and_free_sets(ShenandoahHeapRegionSet* col_set, ShenandoahHeapRegionSet* free_set); + void choose_collection_and_free_sets_min_garbage(ShenandoahHeapRegionSet* col_set, ShenandoahHeapRegionSet* free_set, size_t min_garbage); + + // Check for unreachable humongous regions and reclaim them. + void reclaim_humongous_regions(); + + void set_concurrent_iteration_safe_limits(); + + void decrease_available(size_t num_bytes); + + size_t available() const; + size_t used() const; + +private: + void reclaim_humongous_region_at(ShenandoahHeapRegion** r); + + ShenandoahHeapRegion** limit_region(ShenandoahHeapRegion** region); + size_t _reclaimed; + +}; + +#endif //SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAPREGIONSET_HPP
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/shenandoahHumongous.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,18 @@ + +/* +Copyright 2015 Red Hat, Inc. and/or its affiliates. + */ +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHHUMONGOUS_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHHUMONGOUS_HPP + +#include "gc/shenandoah/shenandoahHeapRegion.hpp" + +class ShenandoahHumongous : public AllStatic { + +public: + static uint required_regions(size_t bytes) { + return (bytes + ShenandoahHeapRegion::RegionSizeBytes - 1) / ShenandoahHeapRegion::RegionSizeBytes; + } +}; + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/shenandoahJNICritical.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,102 @@ +/* +Copyright 2015 Red Hat, Inc. and/or its affiliates. + */ + +#include "gc/shenandoah/shenandoahJNICritical.hpp" +#include "gc/shenandoah/shenandoahHeap.hpp" + +#include "gc/shared/gcLocker.hpp" +#include "runtime/mutexLocker.hpp" +#include "runtime/thread.hpp" +#include "runtime/vmThread.hpp" + +class VM_ShenandoahJNICriticalOperation : public VM_Operation { +private: + VM_Operation* _target; +public: + VM_ShenandoahJNICriticalOperation(VM_Operation* target); + VMOp_Type type() const; + bool doit_prologue(); + void doit_epilogue(); + void doit(); + const char* name() const; +}; + +ShenandoahJNICritical::ShenandoahJNICritical() : _op_waiting_for_jni_critical(NULL) { +} + +/* + * This is called by the Java thread who leaves the last JNI critical block. + */ +void ShenandoahJNICritical::notify_jni_critical() { + assert(Thread::current()->is_Java_thread(), "call only from Java thread"); + assert(_op_waiting_for_jni_critical != NULL, "must be waiting for jni critical notification"); + + MonitorLockerEx ml(ShenandoahJNICritical_lock, true); + + VMThread::execute(_op_waiting_for_jni_critical); + _op_waiting_for_jni_critical = NULL; + + ml.notify_all(); + +} + +/* + * This is called by the VM thread, if it determines that the task must wait + * for JNI critical regions to be left. + */ +void ShenandoahJNICritical::set_waiting_for_jni_before_gc(VM_Operation* op) { + assert(Thread::current()->is_VM_thread(), "call only from VM thread"); + _op_waiting_for_jni_critical = op; +} + +/** + * This is called by the Shenandoah concurrent thread in order + * to execute a VM_Operation on the VM thread, that needs to perform + * a JNI critical region check. + */ +void ShenandoahJNICritical::execute_in_vm_thread(VM_Operation* op) { + MonitorLockerEx ml(ShenandoahJNICritical_lock, true); + VM_ShenandoahJNICriticalOperation jni_op(op); + VMThread::execute(&jni_op); + while (_op_waiting_for_jni_critical != NULL) { + ml.wait(true); + } +} + + +VM_ShenandoahJNICriticalOperation::VM_ShenandoahJNICriticalOperation(VM_Operation* target) + : _target(target) { +} + +VM_Operation::VMOp_Type VM_ShenandoahJNICriticalOperation::type() const { + return _target->type(); +} + +const char* VM_ShenandoahJNICriticalOperation::name() const { + return _target->name(); +} + +bool VM_ShenandoahJNICriticalOperation::doit_prologue() { + return _target->doit_prologue(); +} + +void VM_ShenandoahJNICriticalOperation::doit_epilogue() { + _target->doit_epilogue(); +} + +void VM_ShenandoahJNICriticalOperation::doit() { + if (! GC_locker::check_active_before_gc()) { + _target->doit(); + } else { + + if (ShenandoahTraceJNICritical) { + gclog_or_tty->print_cr("Deferring JNI critical op because of active JNI critical regions"); + } + + // This makes the GC background thread wait, and kick off evacuation as + // soon as JNI notifies us that critical regions have all been left. + ShenandoahHeap *sh = ShenandoahHeap::heap(); + sh->jni_critical()->set_waiting_for_jni_before_gc(this); + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/shenandoahJNICritical.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,22 @@ +/* +Copyright 2015 Red Hat, Inc. and/or its affiliates. + */ +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHJNICRITICAL_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHJNICRITICAL_HPP + +#include "gc/shared/vmGCOperations.hpp" +#include "memory/allocation.hpp" + +class ShenandoahJNICritical : public CHeapObj<mtGC> { +private: + VM_Operation* _op_waiting_for_jni_critical; + +public: + ShenandoahJNICritical(); + void notify_jni_critical(); + void set_waiting_for_jni_before_gc(VM_Operation* op); + void execute_in_vm_thread(VM_Operation* op); +}; + + +#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHJNICRITICAL_HPP
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/shenandoahMarkCompact.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,432 @@ +/* + Copyright 2014 Red Hat, Inc. and/or its affiliates. +*/ + +#include "code/codeCache.hpp" +#include "gc/shared/isGCActiveMark.hpp" +#include "gc/shenandoah/brooksPointer.hpp" +#include "gc/shenandoah/shenandoahMarkCompact.hpp" +#include "gc/shenandoah/shenandoahBarrierSet.hpp" +#include "gc/shenandoah/shenandoahHeap.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahRootProcessor.hpp" +#include "gc/shenandoah/vm_operations_shenandoah.hpp" +#include "gc/serial/markSweep.inline.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/biasedLocking.hpp" +#include "runtime/thread.hpp" +#include "utilities/copy.hpp" +#include "gc/shared/taskqueue.inline.hpp" +#include "gc/shared/workgroup.hpp" + + + +void ShenandoahMarkCompact::allocate_stacks() { + MarkSweep::_preserved_count_max = 0; + MarkSweep::_preserved_marks = NULL; + MarkSweep::_preserved_count = 0; +} + +void ShenandoahMarkCompact::do_mark_compact() { + + COMPILER2_PRESENT(DerivedPointerTable::clear()); + + ShenandoahHeap* _heap = ShenandoahHeap::heap(); + + assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint"); + IsGCActiveMark is_active; + + assert(Thread::current()->is_VM_thread(), "Do full GC only while world is stopped"); + assert(_heap->is_bitmap_clear(), "require cleared bitmap"); + assert(!_heap->concurrent_mark_in_progress(), "can't do full-GC while marking is in progress"); + assert(!_heap->is_evacuation_in_progress(), "can't do full-GC while evacuation is in progress"); + assert(!_heap->is_update_references_in_progress(), "can't do full-GC while updating of references is in progress"); + BarrierSet* _old_barrier_set = oopDesc::bs(); + + oopDesc::set_bs(new ShenandoahMarkCompactBarrierSet()); + + _heap->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::full_gc); + + // We need to clear the is_in_collection_set flag in all regions. + ShenandoahHeapRegion** regions = _heap->heap_regions(); + size_t num_regions = _heap->num_regions(); + for (size_t i = 0; i < num_regions; i++) { + regions[i]->set_is_in_collection_set(false); + } + _heap->clear_cset_fast_test(); + + if (ShenandoahVerify) { + // Full GC should only be called between regular concurrent cycles, therefore + // those verifications should be valid. + _heap->verify_heap_after_evacuation(); + _heap->verify_heap_after_update_refs(); + } + + if (ShenandoahTraceFullGC) { + gclog_or_tty->print_cr("Shenandoah-full-gc: start with heap used: "SIZE_FORMAT" MB", _heap->used() / M); + gclog_or_tty->print_cr("Shenandoah-full-gc: phase 1: marking the heap"); + // _heap->print_heap_regions(); + } + + if (UseTLAB) { + _heap->ensure_parsability(true); + } + + _heap->cleanup_after_cancelconcgc(); + + ReferenceProcessor* rp = _heap->ref_processor(); + + // hook up weak ref data so it can be used during Mark-Sweep + assert(MarkSweep::ref_processor() == NULL, "no stomping"); + assert(rp != NULL, "should be non-NULL"); + assert(rp == ShenandoahHeap::heap()->ref_processor(), "Precondition"); + bool clear_all_softrefs = true; //fixme + MarkSweep::_ref_processor = rp; + rp->setup_policy(clear_all_softrefs); + + CodeCache::gc_prologue(); + allocate_stacks(); + + // We should save the marks of the currently locked biased monitors. + // The marking doesn't preserve the marks of biased objects. + BiasedLocking::preserve_marks(); + + phase1_mark_heap(); + + if (ShenandoahTraceFullGC) { + gclog_or_tty->print_cr("Shenandoah-full-gc: phase 2: calculating target addresses"); + } + phase2_calculate_target_addresses(); + + if (ShenandoahTraceFullGC) { + gclog_or_tty->print_cr("Shenandoah-full-gc: phase 3: updating references"); + } + + // Don't add any more derived pointers during phase3 + COMPILER2_PRESENT(DerivedPointerTable::set_active(false)); + + phase3_update_references(); + + if (ShenandoahTraceFullGC) { + gclog_or_tty->print_cr("Shenandoah-full-gc: phase 4: compacting objects"); + } + + phase4_compact_objects(); + + + MarkSweep::restore_marks(); + BiasedLocking::restore_marks(); + GenMarkSweep::deallocate_stacks(); + + CodeCache::gc_epilogue(); + JvmtiExport::gc_epilogue(); + + // refs processing: clean slate + MarkSweep::_ref_processor = NULL; + + + if (ShenandoahVerify) { + _heap->verify_heap_after_evacuation(); + _heap->verify_heap_after_update_refs(); + } + + if (ShenandoahTraceFullGC) { + gclog_or_tty->print_cr("Shenandoah-full-gc: finish with heap used: "SIZE_FORMAT" MB", _heap->used() / M); + } + + _heap->_bytesAllocSinceCM = 0; + + oopDesc::set_bs(_old_barrier_set); + + _heap->set_need_update_refs(false); + + COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); + + _heap->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::full_gc); +} + +class UpdateRefsClosure: public ExtendedOopClosure { +public: + virtual void do_oop(oop* p) { + oop obj = oopDesc::load_heap_oop(p); + if (! oopDesc::is_null(obj)) { + ShenandoahBarrierSet::resolve_and_update_oop_static(p, obj); + } + } + virtual void do_oop(narrowOop* p) { + Unimplemented(); + } +}; + +void ShenandoahMarkCompact::phase1_mark_heap() { + ShenandoahHeap* _heap = ShenandoahHeap::heap(); + ReferenceProcessor* rp = _heap->ref_processor(); + + MarkSweep::_ref_processor = rp; + + // First, update _all_ references in GC roots to point to to-space. + { + // Need cleared claim bits for the roots processing + /* + ClassLoaderDataGraph::clear_claimed_marks(); + UpdateRefsClosure uprefs; + CLDToOopClosure cld_uprefs(&uprefs); + CodeBlobToOopClosure code_uprefs(&uprefs, CodeBlobToOopClosure::FixRelocations); + ShenandoahRootProcessor rp(_heap, 1); + rp.process_all_roots(&uprefs, + &cld_uprefs, + &code_uprefs); + */ + } + + { + MarkingCodeBlobClosure follow_code_closure(&MarkSweep::follow_root_closure, CodeBlobToOopClosure::FixRelocations); + // Need cleared claim bits for the roots processing + ClassLoaderDataGraph::clear_claimed_marks(); + ShenandoahRootProcessor rp(_heap, 1); + rp.process_strong_roots(&MarkSweep::follow_root_closure, + &MarkSweep::follow_cld_closure, + &follow_code_closure); + + // Also update (without marking) weak CLD refs, in case they're reachable. + UpdateRefsClosure uprefs; + CLDToOopClosure cld_uprefs(&uprefs); + ClassLoaderDataGraph::roots_cld_do(NULL, &cld_uprefs); + + // Same for weak JNI handles. + ShenandoahAlwaysTrueClosure always_true; + JNIHandles::weak_oops_do(&always_true, &uprefs); + } + + _heap->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::weakrefs); + bool clear_soft_refs = false; //fixme + rp->setup_policy(clear_soft_refs); + + const ReferenceProcessorStats& stats = + rp->process_discovered_references(&MarkSweep::is_alive, + &MarkSweep::keep_alive, + &MarkSweep::follow_stack_closure, + NULL, + NULL, + _heap->tracer()->gc_id()); + + // heap->tracer()->report_gc_reference_stats(stats); + + _heap->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::weakrefs); + + // Unload classes and purge the SystemDictionary. + bool purged_class = SystemDictionary::do_unloading(&MarkSweep::is_alive); + + // Unload nmethods. + CodeCache::do_unloading(&MarkSweep::is_alive, purged_class); + + // Prune dead klasses from subklass/sibling/implementor lists. + Klass::clean_weak_klass_links(&MarkSweep::is_alive); + + // Delete entries for dead interned string and clean up unreferenced symbols in symbol table. + _heap->unlink_string_and_symbol_table(&MarkSweep::is_alive); + + if (VerifyDuringGC) { + HandleMark hm; // handle scope + COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact); + // Universe::heap()->prepare_for_verify(); + _heap->prepare_for_verify(); + // Note: we can verify only the heap here. When an object is + // marked, the previous value of the mark word (including + // identity hash values, ages, etc) is preserved, and the mark + // word is set to markOop::marked_value - effectively removing + // any hash values from the mark word. These hash values are + // used when verifying the dictionaries and so removing them + // from the mark word can make verification of the dictionaries + // fail. At the end of the GC, the original mark word values + // (including hash values) are restored to the appropriate + // objects. + if (!VerifySilently) { + gclog_or_tty->print(" VerifyDuringGC:(full)[Verifying "); + } + // Universe::heap()->verify(VerifySilently, VerifyOption_G1UseMarkWord); + _heap->verify(VerifySilently, VerifyOption_G1UseMarkWord); + if (!VerifySilently) { + gclog_or_tty->print_cr("]"); + } + } +} + +class ShenandoahPrepareForCompaction : public ShenandoahHeapRegionClosure { + CompactPoint _cp; + ShenandoahHeap* _heap; + bool _dead_humongous; + +public: + ShenandoahPrepareForCompaction() : + _heap(ShenandoahHeap::heap()), + _dead_humongous(false) { + } + + bool doHeapRegion(ShenandoahHeapRegion* r) { + // We need to save the contents + if (!r->is_humongous()) { + if (_cp.space == NULL) { + _cp.space = r; + _cp.threshold = r->initialize_threshold(); + } + _dead_humongous = false; + r->prepare_for_compaction(&_cp); + } else { + if (r->is_humongous_start()) { + oop obj = oop(r->bottom() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE); + if (obj->is_gc_marked()) { + obj->forward_to(obj); + _dead_humongous = false; + } else { + if (_cp.space == NULL) { + _cp.space = r; + _cp.threshold = r->initialize_threshold(); + } + _dead_humongous = true; + guarantee(r->region_number() >= ((ShenandoahHeapRegion*)_cp.space)->region_number(), + "only reset regions that are not yet used for compaction"); + r->reset(); + r->prepare_for_compaction(&_cp); + } + } else { + assert(r->is_humongous_continuation(), "expect humongous continuation"); + if (_dead_humongous) { + guarantee(r->region_number() > ((ShenandoahHeapRegion*)_cp.space)->region_number(), + "only reset regions that are not yet used for compaction"); + r->reset(); + r->prepare_for_compaction(&_cp); + } + } + } + return false; + } +}; + +void ShenandoahMarkCompact::phase2_calculate_target_addresses() { + ShenandoahPrepareForCompaction prepare; + ShenandoahHeap::heap()->heap_region_iterate(&prepare); +} + + +class ShenandoahMarkCompactAdjustPointersClosure : public ShenandoahHeapRegionClosure { + bool doHeapRegion(ShenandoahHeapRegion* r) { + if (r->is_humongous()) { + if (r->is_humongous_start()) { + // We must adjust the pointers on the single H object. + oop obj = oop(r->bottom() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE); + assert(obj->is_gc_marked(), "should be marked"); + // point all the oops to the new location + MarkSweep::adjust_pointers(obj); + } + } else { + r->adjust_pointers(); + } + return false; + } +}; + +void ShenandoahMarkCompact::phase3_update_references() { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + + // Need cleared claim bits for the roots processing + ClassLoaderDataGraph::clear_claimed_marks(); + + CodeBlobToOopClosure adjust_code_closure(&MarkSweep::adjust_pointer_closure, + CodeBlobToOopClosure::FixRelocations); + + { + ShenandoahRootProcessor rp(heap, 1); + rp.process_all_roots(&MarkSweep::adjust_pointer_closure, + &MarkSweep::adjust_cld_closure, + &adjust_code_closure); + } + + assert(MarkSweep::ref_processor() == heap->ref_processor(), "Sanity"); + + // Now adjust pointers in remaining weak roots. (All of which should + // have been cleared if they pointed to non-surviving objects.) + heap->weak_roots_iterate(&MarkSweep::adjust_pointer_closure); + + // if (G1StringDedup::is_enabled()) { + // G1StringDedup::oops_do(&MarkSweep::adjust_pointer_closure); + // } + + MarkSweep::adjust_marks(); + + ShenandoahMarkCompactAdjustPointersClosure apc; + heap->heap_region_iterate(&apc); +} + +class ShenandoahCleanupObjectClosure : public ObjectClosure { + void do_object(oop p) { + ShenandoahHeap::heap()->initialize_brooks_ptr(p); + } +}; + +class CompactObjectsClosure : public ShenandoahHeapRegionClosure { + +public: + + CompactObjectsClosure() { + } + + bool doHeapRegion(ShenandoahHeapRegion* r) { + if (r->is_humongous()) { + if (r->is_humongous_start()) { + oop obj = oop(r->bottom() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE); + assert(obj->is_gc_marked(), "expect marked humongous object"); + obj->init_mark(); + } + } else { + r->compact(); + } + + return false; + } + +}; + +class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure { + size_t _live; + ShenandoahHeap* _heap; +public: + + ShenandoahPostCompactClosure() : _live(0), _heap(ShenandoahHeap::heap()) { + _heap->clear_free_regions(); + } + + bool doHeapRegion(ShenandoahHeapRegion* r) { + if (r->is_humongous()) { + _live += ShenandoahHeapRegion::RegionSizeBytes; + + } else { + size_t live = r->used(); + if (live == 0) { + r->recycle(); + _heap->add_free_region(r); + } + r->setLiveData(live); + _live += live; + } + + return false; + } + + size_t getLive() { return _live;} + +}; + +void ShenandoahMarkCompact::phase4_compact_objects() { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + CompactObjectsClosure coc; + heap->heap_region_iterate(&coc); + + ShenandoahCleanupObjectClosure cleanup; + heap->object_iterate(&cleanup); + + ShenandoahPostCompactClosure post_compact; + heap->heap_region_iterate(&post_compact); + + heap->set_used(post_compact.getLive()); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/shenandoahMarkCompact.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,45 @@ +/* +Copyright 2014 Red Hat, Inc. and/or its affiliates. + */ +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHMARKCOMPACT_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHMARKCOMPACT_HPP + +#include "gc/serial/genMarkSweep.hpp" +#include "gc/shared/taskqueue.hpp" +#include "gc/shared/workgroup.hpp" +#include "gc/shenandoah/shenandoahHeap.hpp" + +class HeapWord; +class ShenandoahMarkCompactBarrierSet; + +/** + * This implements full-GC (e.g. when invoking System.gc() ) using a + * mark-compact algorithm. It's implemented in four phases: + * + * 1. Mark all live objects of the heap by traversing objects starting at GC roots. + * 2. Calculate the new location of each live object. This is done by sequentially scanning + * the heap, keeping track of a next-location-pointer, which is then written to each + * object's brooks ptr field. + * 3. Update all references. This is implemented by another scan of the heap, and updates + * all references in live objects by what's stored in the target object's brooks ptr. + * 3. Compact the heap by copying all live objects to their new location. + */ + +class ShenandoahMarkCompact: AllStatic { + +public: + + static void do_mark_compact(); + +private: + + static void phase1_mark_heap(); + static void phase2_calculate_target_addresses(); + static void phase3_update_references(); + static void phase4_compact_objects(); + static void finish_compaction(HeapWord* last_addr); + + static void allocate_stacks(); +}; + +#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHMARKCOMPACT_HPP
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/shenandoahRootProcessor.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,118 @@ +#include "precompiled.hpp" + +#include "classfile/stringTable.hpp" +#include "classfile/systemDictionary.hpp" +#include "code/codeCache.hpp" +#include "gc/shenandoah/shenandoahRootProcessor.hpp" +#include "memory/allocation.inline.hpp" +#include "runtime/fprofiler.hpp" +#include "runtime/mutex.hpp" +#include "services/management.hpp" + +ShenandoahRootProcessor::ShenandoahRootProcessor(ShenandoahHeap* heap, uint n_workers) : + _process_strong_tasks(new SubTasksDone(SHENANDOAH_RP_PS_NumElements)), + _srs(n_workers) +{ +} + +void ShenandoahRootProcessor::process_roots(OopClosure* strong_oops, + OopClosure* weak_oops, + CLDClosure* strong_clds, + CLDClosure* weak_clds, + CLDClosure* thread_stack_clds, + CodeBlobClosure* strong_code) { + process_java_roots(strong_oops, thread_stack_clds, strong_clds, weak_clds, strong_code, 0); + process_vm_roots(strong_oops, weak_oops, 0); + + if (!_process_strong_tasks->is_task_claimed(SHENANDOAH_RP_PS_CodeCache_oops_do)) { + CodeCache::blobs_do(strong_code); + } + + if (!_process_strong_tasks->is_task_claimed(SHENANDOAH_RP_PS_JNIHandles_weak_oops_do)) { + ShenandoahAlwaysTrueClosure always_true; + JNIHandles::weak_oops_do(&always_true, weak_oops); + } + + _process_strong_tasks->all_tasks_completed(n_workers()); +} + +void ShenandoahRootProcessor::process_strong_roots(OopClosure* oops, + CLDClosure* clds, + CodeBlobClosure* blobs) { + + process_java_roots(oops, clds, clds, NULL, blobs, 0); + process_vm_roots(oops, NULL, 0); + + _process_strong_tasks->all_tasks_completed(n_workers()); +} + +void ShenandoahRootProcessor::process_all_roots(OopClosure* oops, + CLDClosure* clds, + CodeBlobClosure* blobs) { + + process_java_roots(oops, NULL, clds, clds, NULL, 0); + process_vm_roots(oops, oops, 0); + + if (!_process_strong_tasks->is_task_claimed(SHENANDOAH_RP_PS_CodeCache_oops_do)) { + CodeCache::blobs_do(blobs); + } + + _process_strong_tasks->all_tasks_completed(n_workers()); +} + +void ShenandoahRootProcessor::process_java_roots(OopClosure* strong_roots, + CLDClosure* thread_stack_clds, + CLDClosure* strong_clds, + CLDClosure* weak_clds, + CodeBlobClosure* strong_code, + uint worker_i) +{ + //assert(thread_stack_clds == NULL || weak_clds == NULL, "There is overlap between those, only one may be set"); + // Iterating over the CLDG and the Threads are done early to allow us to + // first process the strong CLDs and nmethods and then, after a barrier, + // let the thread process the weak CLDs and nmethods. + if (!_process_strong_tasks->is_task_claimed(SHENANDOAH_RP_PS_ClassLoaderDataGraph_oops_do)) { + ClassLoaderDataGraph::roots_cld_do(strong_clds, weak_clds); + } + + bool is_par = n_workers() > 1; + ResourceMark rm; + Threads::possibly_parallel_oops_do(is_par, strong_roots, thread_stack_clds, strong_code); +} + +void ShenandoahRootProcessor::process_vm_roots(OopClosure* strong_roots, + OopClosure* weak_roots, + uint worker_i) +{ + if (!_process_strong_tasks->is_task_claimed(SHENANDOAH_RP_PS_Universe_oops_do)) { + Universe::oops_do(strong_roots); + } + + if (!_process_strong_tasks->is_task_claimed(SHENANDOAH_RP_PS_JNIHandles_oops_do)) { + JNIHandles::oops_do(strong_roots); + } + if (!_process_strong_tasks-> is_task_claimed(SHENANDOAH_RP_PS_ObjectSynchronizer_oops_do)) { + ObjectSynchronizer::oops_do(strong_roots); + } + if (!_process_strong_tasks->is_task_claimed(SHENANDOAH_RP_PS_FlatProfiler_oops_do)) { + FlatProfiler::oops_do(strong_roots); + } + if (!_process_strong_tasks->is_task_claimed(SHENANDOAH_RP_PS_Management_oops_do)) { + Management::oops_do(strong_roots); + } + if (!_process_strong_tasks->is_task_claimed(SHENANDOAH_RP_PS_jvmti_oops_do)) { + JvmtiExport::oops_do(strong_roots); + } + if (!_process_strong_tasks->is_task_claimed(SHENANDOAH_RP_PS_SystemDictionary_oops_do)) { + SystemDictionary::roots_oops_do(strong_roots, weak_roots); + } + // All threads execute the following. A specific chunk of buckets + // from the StringTable are the individual tasks. + if (weak_roots != NULL) { + StringTable::possibly_parallel_oops_do(weak_roots); + } +} + +uint ShenandoahRootProcessor::n_workers() const { + return _srs.n_threads(); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/shenandoahRootProcessor.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,74 @@ + +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHROOTPROCESSOR_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHROOTPROCESSOR_HPP + +#include "gc/shared/strongRootsScope.hpp" +#include "memory/allocation.hpp" +#include "runtime/mutex.hpp" + +class CLDClosure; +class CodeBlobClosure; +class G1CollectedHeap; +class G1GCPhaseTimes; +class G1ParPushHeapRSClosure; +class Monitor; +class OopClosure; +class SubTasksDone; + +class ShenandoahRootProcessor : public StackObj { + SubTasksDone* _process_strong_tasks; + StrongRootsScope _srs; + + enum Shenandoah_process_roots_tasks { + SHENANDOAH_RP_PS_Universe_oops_do, + SHENANDOAH_RP_PS_JNIHandles_oops_do, + SHENANDOAH_RP_PS_JNIHandles_weak_oops_do, + SHENANDOAH_RP_PS_ObjectSynchronizer_oops_do, + SHENANDOAH_RP_PS_FlatProfiler_oops_do, + SHENANDOAH_RP_PS_Management_oops_do, + SHENANDOAH_RP_PS_SystemDictionary_oops_do, + SHENANDOAH_RP_PS_ClassLoaderDataGraph_oops_do, + SHENANDOAH_RP_PS_jvmti_oops_do, + SHENANDOAH_RP_PS_CodeCache_oops_do, + SHENANDOAH_RP_PS_filter_satb_buffers, + SHENANDOAH_RP_PS_refProcessor_oops_do, + // Leave this one last. + SHENANDOAH_RP_PS_NumElements + }; + + void process_java_roots(OopClosure* scan_non_heap_roots, + CLDClosure* thread_stack_clds, + CLDClosure* scan_strong_clds, + CLDClosure* scan_weak_clds, + CodeBlobClosure* scan_strong_code, + uint worker_i); + + void process_vm_roots(OopClosure* scan_non_heap_roots, + OopClosure* scan_non_heap_weak_roots, + uint worker_i); + +public: + ShenandoahRootProcessor(ShenandoahHeap* heap, uint n_workers); + + void process_roots(OopClosure* strong_oops, + OopClosure* weak_oops, + CLDClosure* strong_clds, + CLDClosure* weak_clds, + CLDClosure* thread_stack_clds, + CodeBlobClosure* strong_code); + + // Apply oops, clds and blobs to all strongly reachable roots in the system + void process_strong_roots(OopClosure* oops, + CLDClosure* clds, + CodeBlobClosure* blobs); + + // Apply oops, clds and blobs to strongly and weakly reachable roots in the system + void process_all_roots(OopClosure* oops, + CLDClosure* clds, + CodeBlobClosure* blobs); + + // Number of worker threads used by the root processor. + uint n_workers() const; +}; + +#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHROOTPROCESSOR_HPP
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/shenandoahRuntime.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,19 @@ +/* +Copyright 2014 Red Hat, Inc. and/or its affiliates. + */ + +#include "gc/shenandoah/shenandoahRuntime.hpp" +#include "runtime/interfaceSupport.hpp" +#include "oops/oop.inline.hpp" + +JRT_LEAF(bool, ShenandoahRuntime::compare_and_swap_object(HeapWord* addr, oopDesc* newval, oopDesc* old)) + bool success; + oop expected; + do { + expected = old; + old = oopDesc::atomic_compare_exchange_oop(newval, addr, expected, true); + success = (old == expected); + } while ((! success) && oopDesc::bs()->resolve_oop(old) == oopDesc::bs()->resolve_oop(expected)); + + return success; +JRT_END
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/shenandoahRuntime.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,13 @@ +/* +Copyright 2014 Red Hat, Inc. and/or its affiliates. + */ +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHRUNTIME_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHRUNTIME_HPP + +#include "oops/oop.hpp" + +class ShenandoahRuntime : AllStatic { +public: + static bool compare_and_swap_object(HeapWord* adr, oopDesc* newval, oopDesc* expected); +}; +#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHRUNTIME_HPP
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/vm_operations_shenandoah.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,243 @@ +/* + Copyright 2014 Red Hat, Inc. and/or its affiliates. +*/ + +#include "gc/shenandoah/shenandoahMarkCompact.hpp" +#include "gc/shenandoah/vm_operations_shenandoah.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" + +VM_Operation::VMOp_Type VM_ShenandoahInitMark::type() const { + return VMOp_ShenandoahInitMark; +} + +const char* VM_ShenandoahInitMark::name() const { + return "Shenandoah Initial Marking"; +} + +void VM_ShenandoahInitMark::doit() { + ShenandoahHeap *sh = (ShenandoahHeap*) Universe::heap(); + sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::init_mark); + + assert(sh->is_bitmap_clear(), "need clear marking bitmap"); + + if (ShenandoahGCVerbose) + tty->print("vm_ShenandoahInitMark\n"); + sh->start_concurrent_marking(); + if (UseTLAB) { + sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::resize_tlabs); + sh->resize_all_tlabs(); + sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::resize_tlabs); + } + + sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::init_mark); + + if (! ShenandoahConcurrentMarking) { + sh->concurrentMark()->mark_from_roots(); + VM_ShenandoahStartEvacuation finishMark; + finishMark.doit(); + } +} + +VM_Operation::VMOp_Type VM_ShenandoahFullGC::type() const { + return VMOp_ShenandoahFullGC; +} + +void VM_ShenandoahFullGC::doit() { + + ShenandoahMarkCompact::do_mark_compact(); + ShenandoahHeap *sh = ShenandoahHeap::heap(); + if (UseTLAB) { + sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::resize_tlabs); + sh->resize_all_tlabs(); + sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::resize_tlabs); + } +} + +const char* VM_ShenandoahFullGC::name() const { + return "Shenandoah Full GC"; +} + + +bool VM_ShenandoahReferenceOperation::doit_prologue() { + ShenandoahHeap *sh = (ShenandoahHeap*) Universe::heap(); + sh->acquire_pending_refs_lock(); + return true; +} + +void VM_ShenandoahReferenceOperation::doit_epilogue() { + ShenandoahHeap *sh = ShenandoahHeap::heap(); + sh->release_pending_refs_lock(); +} + +void VM_ShenandoahStartEvacuation::doit() { + + // We need to do the finish mark here, so that a JNI critical region + // can't divide it from evacuation start. It is critical that we + // evacuate roots right after finishing marking, so that we don't + // get unmarked objects in the roots. + ShenandoahHeap *sh = ShenandoahHeap::heap(); + if (!sh->cancelled_concgc()) { + if (ShenandoahGCVerbose) + tty->print("vm_ShenandoahFinalMark\n"); + + sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::final_mark); + sh->concurrentMark()->finish_mark_from_roots(); + sh->stop_concurrent_marking(); + sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::final_mark); + + sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::prepare_evac); + sh->prepare_for_concurrent_evacuation(); + sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::prepare_evac); + + if (!sh->cancelled_concgc()){ + sh->set_evacuation_in_progress(true); + + // From here on, we need to update references. + sh->set_need_update_refs(true); + + if (! ShenandoahConcurrentEvacuation) { + VM_ShenandoahEvacuation evacuation; + evacuation.doit(); + } else { + if (!sh->cancelled_concgc()) { + sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::init_evac); + sh->evacuate_and_update_roots(); + sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::init_evac); + } + } + } else { + sh->free_regions()->set_concurrent_iteration_safe_limits(); + // sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::prepare_evac); + // sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::final_mark); + } + } else { + sh->concurrentMark()->cancel(); + sh->stop_concurrent_marking(); + } +} + +VM_Operation::VMOp_Type VM_ShenandoahStartEvacuation::type() const { + return VMOp_ShenandoahStartEvacuation; +} + +const char* VM_ShenandoahStartEvacuation::name() const { + return "Start shenandoah evacuation"; +} + +VM_Operation::VMOp_Type VM_ShenandoahVerifyHeapAfterEvacuation::type() const { + return VMOp_ShenandoahVerifyHeapAfterEvacuation; +} + +const char* VM_ShenandoahVerifyHeapAfterEvacuation::name() const { + return "Shenandoah verify heap after evacuation"; +} + +void VM_ShenandoahVerifyHeapAfterEvacuation::doit() { + + ShenandoahHeap *sh = ShenandoahHeap::heap(); + sh->verify_heap_after_evacuation(); + +} + +VM_Operation::VMOp_Type VM_ShenandoahEvacuation::type() const { + return VMOp_ShenandoahEvacuation; +} + +const char* VM_ShenandoahEvacuation::name() const { + return "Shenandoah evacuation"; +} + +void VM_ShenandoahEvacuation::doit() { + if (ShenandoahGCVerbose) + tty->print("vm_ShenandoahEvacuation\n"); + + ShenandoahHeap *sh = ShenandoahHeap::heap(); + sh->do_evacuation(); + + if (! ShenandoahConcurrentUpdateRefs) { + assert(! ShenandoahConcurrentEvacuation, "turn off concurrent evacuation"); + sh->prepare_for_update_references(); + sh->update_references(); + } +} +/* + VM_Operation::VMOp_Type VM_ShenandoahVerifyHeapAfterUpdateRefs::type() const { + return VMOp_ShenandoahVerifyHeapAfterUpdateRefs; + } + + const char* VM_ShenandoahVerifyHeapAfterUpdateRefs::name() const { + return "Shenandoah verify heap after updating references"; + } + + void VM_ShenandoahVerifyHeapAfterUpdateRefs::doit() { + + ShenandoahHeap *sh = ShenandoahHeap::heap(); + sh->verify_heap_after_update_refs(); + + } +*/ +VM_Operation::VMOp_Type VM_ShenandoahUpdateRootRefs::type() const { + return VMOp_ShenandoahUpdateRootRefs; +} + +const char* VM_ShenandoahUpdateRootRefs::name() const { + return "Shenandoah update root references"; +} + +void VM_ShenandoahUpdateRootRefs::doit() { + ShenandoahHeap *sh = ShenandoahHeap::heap(); + if (! sh->cancelled_concgc()) { + + if (ShenandoahGCVerbose) + tty->print("vm_ShenandoahUpdateRootRefs\n"); + + + sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::final_uprefs); + + sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::update_roots); + + sh->update_roots(); + + sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::update_roots); + + sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::final_uprefs); + } + + sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::recycle_regions); + sh->recycle_dirty_regions(); + sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::recycle_regions); + + if (ShenandoahVerify && ! sh->cancelled_concgc()) { + sh->verify_heap_after_update_refs(); + sh->verify_regions_after_update_refs(); + } +#ifdef ASSERT + if (! ShenandoahVerify) { + assert(sh->is_bitmap_clear(), "need cleared bitmap here"); + } +#endif + +} + +VM_Operation::VMOp_Type VM_ShenandoahUpdateRefs::type() const { + return VMOp_ShenandoahUpdateRefs; +} + +const char* VM_ShenandoahUpdateRefs::name() const { + return "Shenandoah update references"; +} + +void VM_ShenandoahUpdateRefs::doit() { + ShenandoahHeap *sh = ShenandoahHeap::heap(); + if (!sh->cancelled_concgc()) { + + if (ShenandoahGCVerbose) + tty->print("vm_ShenandoahUpdateRefs\n"); + + sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::final_evac); + sh->set_evacuation_in_progress(false); + sh->prepare_for_update_references(); + assert(ShenandoahConcurrentUpdateRefs, "only do this when concurrent update references is turned on"); + sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::final_evac); + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc/shenandoah/vm_operations_shenandoah.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,97 @@ +/* +Copyright 2014 Red Hat, Inc. and/or its affiliates. + */ +#ifndef SHARE_VM_GC_SHENANDOAH_VM_OPERATIONS_SHENANDOAH_HPP +#define SHARE_VM_GC_SHENANDOAH_VM_OPERATIONS_SHENANDOAH_HPP + +#include "gc/shenandoah/shenandoahConcurrentMark.hpp" +#include "gc/shared/vmGCOperations.hpp" + +// VM_operations for the Shenandoah Collector. +// For now we are just doing two pauses. The initial marking pause, and the final finish up marking and perform evacuation pause. +// VM_ShenandoahInitMark +// VM_ShenandoahFinishMark + +class VM_ShenandoahInitMark: public VM_Operation { + +public: + virtual VMOp_Type type() const; + virtual void doit(); + + virtual const char* name() const; +}; + +class VM_ShenandoahReferenceOperation : public VM_Operation { + bool doit_prologue(); + void doit_epilogue(); + +}; + +class VM_ShenandoahStartEvacuation: public VM_ShenandoahReferenceOperation { + + public: + VMOp_Type type() const; + void doit(); + const char* name() const; + +}; + +class VM_ShenandoahFullGC : public VM_ShenandoahReferenceOperation { + public: + VMOp_Type type() const; + void doit(); + const char* name() const; +}; + +class VM_ShenandoahVerifyHeapAfterEvacuation: public VM_Operation { + + public: + virtual VMOp_Type type() const; + virtual void doit(); + + virtual const char* name() const; + +}; + +class VM_ShenandoahEvacuation: public VM_Operation { + + public: + virtual VMOp_Type type() const; + virtual void doit(); + + virtual const char* name() const; + +}; + +/* +class VM_ShenandoahVerifyHeapAfterUpdateRefs: public VM_Operation { + + public: + virtual VMOp_Type type() const; + virtual void doit(); + + virtual const char* name() const; + +}; +*/ +class VM_ShenandoahUpdateRootRefs: public VM_Operation { + + public: + virtual VMOp_Type type() const; + virtual void doit(); + + virtual const char* name() const; + +}; + +class VM_ShenandoahUpdateRefs: public VM_Operation { + + public: + virtual VMOp_Type type() const; + virtual void doit(); + + virtual const char* name() const; + +}; + +#endif //SHARE_VM_GC_SHENANDOAH_VM_OPERATIONS_SHENANDOAH_HPP
--- a/src/share/vm/interpreter/bytecodeInterpreter.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/interpreter/bytecodeInterpreter.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -24,6 +24,7 @@ // no precompiled headers #include "classfile/vmSymbols.hpp" +#include "gc/shared/barrierSet.hpp" #include "gc/shared/collectedHeap.hpp" #include "interpreter/bytecodeHistogram.hpp" #include "interpreter/bytecodeInterpreter.hpp" @@ -1797,7 +1798,7 @@ BasicObjectLock* entry = NULL; while (most_recent != limit ) { if (most_recent->obj() == NULL) entry = most_recent; - else if (most_recent->obj() == lockee) break; + else if (oopDesc::bs()->resolve_oop(most_recent->obj()) == lockee) break; most_recent++; } if (entry != NULL) { @@ -1900,7 +1901,7 @@ BasicObjectLock* limit = istate->monitor_base(); BasicObjectLock* most_recent = (BasicObjectLock*) istate->stack_base(); while (most_recent != limit ) { - if ((most_recent)->obj() == lockee) { + if (oopDesc::bs()->resolve_oop(most_recent->obj()) == lockee) { BasicLock* lock = most_recent->lock(); markOop header = lock->displaced_header(); most_recent->set_obj(NULL); @@ -1972,7 +1973,7 @@ oop obj; if ((Bytecodes::Code)opcode == Bytecodes::_getstatic) { Klass* k = cache->f1_as_klass(); - obj = k->java_mirror(); + obj = Universe::heap()->barrier_set()->resolve_oop(k->java_mirror()); MORE_STACK(1); // Assume single slot push } else { obj = (oop) STACK_OBJECT(-1); @@ -2087,7 +2088,7 @@ } if ((Bytecodes::Code)opcode == Bytecodes::_putstatic) { Klass* k = cache->f1_as_klass(); - obj = k->java_mirror(); + obj = Universe::heap()->barrier_set()->resolve_oop(k->java_mirror()); } else { --count; obj = (oop) STACK_OBJECT(count); @@ -2152,7 +2153,10 @@ Klass* k_entry = (Klass*) entry; assert(k_entry->oop_is_instance(), "Should be InstanceKlass"); InstanceKlass* ik = (InstanceKlass*) k_entry; - if ( ik->is_initialized() && ik->can_be_fastpath_allocated() ) { + // TODO: How can we do fastpath allocation with a clean GC interface? This + // code assumes a bunch of things about the GC, and the setup code is + // sensitive to changes in setup code in CollectedHeap. + if (false && ik->is_initialized() && ik->can_be_fastpath_allocated() ) { size_t obj_size = ik->size_helper(); oop result = NULL; // If the TLAB isn't pre-zeroed then we'll have to do it
--- a/src/share/vm/memory/universe.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/memory/universe.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -75,6 +75,8 @@ #include "utilities/macros.hpp" #include "utilities/preserveException.hpp" #if INCLUDE_ALL_GCS +#include "gc/shenandoah/shenandoahHeap.hpp" +#include "gc/shenandoah/shenandoahCollectorPolicy.hpp" #include "gc/cms/cmsCollectorPolicy.hpp" #include "gc/g1/g1CollectedHeap.inline.hpp" #include "gc/g1/g1CollectorPolicy.hpp" @@ -590,6 +592,9 @@ // a filled in stack trace. // - if there are no preallocated errors with backtrace available then return // an error without backtrace. + + default_err = resolve_oop(default_err); + int next; if (_preallocated_out_of_memory_error_avail_count > 0) { next = (int)Atomic::add(-1, &_preallocated_out_of_memory_error_avail_count); @@ -703,6 +708,9 @@ fatal("UseG1GC not supported in this VM."); } else if (UseConcMarkSweepGC) { fatal("UseConcMarkSweepGC not supported in this VM."); + } else if (UseShenandoahGC) { + fatal("UseShenandoahGC not supported in this VM."); + } #else if (UseParallelGC) { return Universe::create_heap_with_policy<ParallelScavengeHeap, GenerationSizer>(); @@ -710,6 +718,8 @@ return Universe::create_heap_with_policy<G1CollectedHeap, G1CollectorPolicy>(); } else if (UseConcMarkSweepGC) { return Universe::create_heap_with_policy<GenCollectedHeap, ConcurrentMarkSweepPolicy>(); + } else if (UseShenandoahGC) { + return Universe::create_heap_with_policy<ShenandoahHeap, ShenandoahCollectorPolicy>(); #endif } else if (UseSerialGC) { return Universe::create_heap_with_policy<GenCollectedHeap, MarkSweepPolicy>();
--- a/src/share/vm/memory/universe.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/memory/universe.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -25,6 +25,7 @@ #ifndef SHARE_VM_MEMORY_UNIVERSE_HPP #define SHARE_VM_MEMORY_UNIVERSE_HPP +#include "gc/shared/barrierSet.hpp" #include "runtime/handles.hpp" #include "utilities/array.hpp" #include "utilities/growableArray.hpp" @@ -203,8 +204,12 @@ static bool _bootstrapping; // true during genesis static bool _fully_initialized; // true after universe_init and initialize_vtables called + static oop resolve_oop(oop o) { + return (oop) oopDesc::bs()->resolve_oop(o); + } + // the array of preallocated errors with backtraces - static objArrayOop preallocated_out_of_memory_errors() { return _preallocated_out_of_memory_error_array; } + static objArrayOop preallocated_out_of_memory_errors() { return (objArrayOop) resolve_oop((oop) _preallocated_out_of_memory_error_array); } // generate an out of memory error; if possible using an error with preallocated backtrace; // otherwise return the given default error. @@ -230,7 +235,7 @@ // Mirrors for primitive classes (created eagerly) static oop check_mirror(oop m) { assert(m != NULL, "mirror not initialized"); - return m; + return resolve_oop(m); } static void set_narrow_oop_base(address base) { @@ -290,18 +295,19 @@ static oop java_mirror(BasicType t) { assert((uint)t < T_VOID+1, "range check"); - return check_mirror(_mirrors[t]); + oop mirror = check_mirror(_mirrors[t]); + return mirror; } - static oop main_thread_group() { return _main_thread_group; } + static oop main_thread_group() { return resolve_oop(_main_thread_group); } static void set_main_thread_group(oop group) { _main_thread_group = group;} - static oop system_thread_group() { return _system_thread_group; } + static oop system_thread_group() { return resolve_oop(_system_thread_group); } static void set_system_thread_group(oop group) { _system_thread_group = group;} - static objArrayOop the_empty_class_klass_array () { return _the_empty_class_klass_array; } + static objArrayOop the_empty_class_klass_array () { return (objArrayOop) resolve_oop((oop) _the_empty_class_klass_array); } static Array<Klass*>* the_array_interfaces_array() { return _the_array_interfaces_array; } - static oop the_null_string() { return _the_null_string; } - static oop the_min_jint_string() { return _the_min_jint_string; } + static oop the_null_string() { return resolve_oop(_the_null_string); } + static oop the_min_jint_string() { return resolve_oop(_the_min_jint_string); } static Method* finalizer_register_method() { return _finalizer_register_cache->get_method(); } static Method* loader_addClass_method() { return _loader_addClass_cache->get_method(); } @@ -309,10 +315,10 @@ static Method* protection_domain_implies_method() { return _pd_implies_cache->get_method(); } static Method* throw_illegal_access_error() { return _throw_illegal_access_error_cache->get_method(); } - static oop null_ptr_exception_instance() { return _null_ptr_exception_instance; } - static oop arithmetic_exception_instance() { return _arithmetic_exception_instance; } - static oop virtual_machine_error_instance() { return _virtual_machine_error_instance; } - static oop vm_exception() { return _vm_exception; } + static oop null_ptr_exception_instance() { return resolve_oop(_null_ptr_exception_instance); } + static oop arithmetic_exception_instance() { return resolve_oop(_arithmetic_exception_instance); } + static oop virtual_machine_error_instance() { return resolve_oop(_virtual_machine_error_instance); } + static oop vm_exception() { return resolve_oop(_vm_exception); } static inline oop allocation_context_notification_obj(); static inline void set_allocation_context_notification_obj(oop obj); @@ -345,6 +351,7 @@ // The particular choice of collected heap. static CollectedHeap* heap() { return _collectedHeap; } + static CollectedHeap** heap_addr() { return &_collectedHeap; } // For UseCompressedOops // Narrow Oop encoding mode:
--- a/src/share/vm/memory/universe.inline.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/memory/universe.inline.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -42,7 +42,7 @@ } inline oop Universe::allocation_context_notification_obj() { - return _allocation_context_notification_obj; + return resolve_oop(_allocation_context_notification_obj); } inline void Universe::set_allocation_context_notification_obj(oop obj) {
--- a/src/share/vm/oops/instanceKlass.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/oops/instanceKlass.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -1527,6 +1527,7 @@ Method* InstanceKlass::uncached_lookup_method(Symbol* name, Symbol* signature, OverpassLookupMode overpass_mode) const { OverpassLookupMode overpass_local_mode = overpass_mode; Klass* klass = const_cast<InstanceKlass*>(this); + bool dont_ignore_overpasses = true; // For the class being searched, find its overpasses. while (klass != NULL) { Method* method = InstanceKlass::cast(klass)->find_method_impl(name, signature, overpass_local_mode, find_static, find_private); if (method != NULL) {
--- a/src/share/vm/oops/instanceRefKlass.inline.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/oops/instanceRefKlass.inline.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -46,7 +46,7 @@ ReferenceProcessor* rp = closure->_ref_processor; if (!oopDesc::is_null(heap_oop)) { oop referent = oopDesc::decode_heap_oop_not_null(heap_oop); - if (!referent->is_gc_marked() && (rp != NULL) && + if ((UseShenandoahGC || !referent->is_gc_marked()) && (rp != NULL) && rp->discover_reference(obj, reference_type())) { return; } else if (contains(referent_addr)) {
--- a/src/share/vm/oops/klass.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/oops/klass.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -556,7 +556,7 @@ return NULL; } -oop Klass::class_loader() const { return class_loader_data()->class_loader(); } +oop Klass::class_loader() const { return oopDesc::bs()->resolve_oop(class_loader_data()->class_loader()); } const char* Klass::external_name() const { if (oop_is_instance()) {
--- a/src/share/vm/oops/klass.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/oops/klass.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -25,6 +25,7 @@ #ifndef SHARE_VM_OOPS_KLASS_HPP #define SHARE_VM_OOPS_KLASS_HPP +#include "gc/shared/barrierSet.hpp" #include "gc/shared/specialized_oop_closures.hpp" #include "memory/iterator.hpp" #include "memory/memRegion.hpp" @@ -223,7 +224,7 @@ void klass_oop_store(volatile oop* p, oop v); // java mirror - oop java_mirror() const { return _java_mirror; } + oop java_mirror() const { return oopDesc::bs()->resolve_oop(_java_mirror); } void set_java_mirror(oop m) { klass_oop_store(&_java_mirror, m); } // modifier flags
--- a/src/share/vm/oops/objArrayKlass.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/oops/objArrayKlass.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -200,6 +200,7 @@ for (int index = 0; index < length; index++) { ArrayKlass* ak = ArrayKlass::cast(h_lower_dimension()); oop sub_array = ak->multi_allocate(rank-1, &sizes[1], CHECK_NULL); + h_array = objArrayHandle(THREAD, (objArrayOop)oopDesc::bs()->resolve_and_maybe_copy_oop(h_array())); h_array->obj_at_put(index, sub_array); } } else {
--- a/src/share/vm/oops/objArrayOop.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/oops/objArrayOop.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -82,6 +82,11 @@ oop obj_at(int index) const; void obj_at_put(int index, oop value) { + objArrayOop forwarded_copy = + (objArrayOop) oopDesc::bs()->resolve_and_maybe_copy_oop(this); + if ((oopDesc*) forwarded_copy != this) + return forwarded_copy->obj_at_put(index, value); + if (UseCompressedOops) { oop_store(obj_at_addr<narrowOop>(index), value); } else {
--- a/src/share/vm/oops/oop.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/oops/oop.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -174,6 +174,10 @@ static oop load_decode_heap_oop(narrowOop* p); static oop load_decode_heap_oop(oop* p); +#ifdef ASSERT_DISABLED + static void shenandoah_check_store_value(oop v); +#endif + // Store an oop into the heap. static void store_heap_oop(narrowOop* p, narrowOop v); static void store_heap_oop(oop* p, oop v);
--- a/src/share/vm/oops/oop.inline.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/oops/oop.inline.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -134,7 +134,7 @@ inline bool oopDesc::is_objArray() const { return klass()->oop_is_objArray(); } inline bool oopDesc::is_typeArray() const { return klass()->oop_is_typeArray(); } -inline void* oopDesc::field_base(int offset) const { return (void*)&((char*)this)[offset]; } +inline void* oopDesc::field_base(int offset) const { return (void*)&((char*)(oopDesc*)bs()->resolve_oop((oop) this))[offset]; } template <class T> inline T* oopDesc::obj_field_addr(int offset) const { return (T*)field_base(offset); } inline Metadata** oopDesc::metadata_field_addr(int offset) const { return (Metadata**)field_base(offset); } @@ -218,20 +218,42 @@ } // Store already encoded heap oop into the heap. -inline void oopDesc::store_heap_oop(oop* p, oop v) { *p = v; } +inline void oopDesc::store_heap_oop(oop* p, oop v) { +#ifdef ASSERT_DISABLED + shenandoah_check_store_value(v); +#endif + *p = v; +} inline void oopDesc::store_heap_oop(narrowOop* p, narrowOop v) { *p = v; } // Encode and store a heap oop. inline void oopDesc::encode_store_heap_oop_not_null(narrowOop* p, oop v) { +#ifdef ASSERT_DISABLED + shenandoah_check_store_value(v); +#endif + *p = encode_heap_oop_not_null(v); } -inline void oopDesc::encode_store_heap_oop_not_null(oop* p, oop v) { *p = v; } +inline void oopDesc::encode_store_heap_oop_not_null(oop* p, oop v) { +#ifdef ASSERT_DISABLED + shenandoah_check_store_value(v); +#endif + *p = v; +} // Encode and store a heap oop allowing for null. inline void oopDesc::encode_store_heap_oop(narrowOop* p, oop v) { +#ifdef ASSERT_DISABLED + shenandoah_check_store_value(v); +#endif *p = encode_heap_oop(v); } -inline void oopDesc::encode_store_heap_oop(oop* p, oop v) { *p = v; } +inline void oopDesc::encode_store_heap_oop(oop* p, oop v) { +#ifdef ASSERT_DISABLED + shenandoah_check_store_value(v); +#endif + *p = v; +} // Store heap oop as is for volatile fields. inline void oopDesc::release_store_heap_oop(volatile oop* p, oop v) { @@ -286,6 +308,12 @@ } inline void oopDesc::obj_field_put(int offset, oop value) { + oopDesc* forwarded_copy = oopDesc::bs()->resolve_and_maybe_copy_oop(this); + if (forwarded_copy != this) + return forwarded_copy->obj_field_put(offset, value); + + value = oopDesc::bs()->resolve_oop(value); + UseCompressedOops ? oop_store(obj_field_addr<narrowOop>(offset), value) : oop_store(obj_field_addr<oop>(offset), value); } @@ -295,10 +323,19 @@ } inline void oopDesc::metadata_field_put(int offset, Metadata* value) { + oopDesc* forwarded_copy = oopDesc::bs()->resolve_and_maybe_copy_oop(this); + if (forwarded_copy != this) { + return forwarded_copy->metadata_field_put(offset, value); + } *metadata_field_addr(offset) = value; } inline void oopDesc::obj_field_put_raw(int offset, oop value) { + oopDesc* forwarded_copy = oopDesc::bs()->resolve_and_maybe_copy_oop(this); + if (forwarded_copy != this) { + return forwarded_copy->obj_field_put_raw(offset, value); + } + value = oopDesc::bs()->resolve_oop(value); UseCompressedOops ? encode_store_heap_oop(obj_field_addr<narrowOop>(offset), value) : encode_store_heap_oop(obj_field_addr<oop>(offset), value); @@ -310,31 +347,85 @@ } inline jbyte oopDesc::byte_field(int offset) const { return (jbyte) *byte_field_addr(offset); } -inline void oopDesc::byte_field_put(int offset, jbyte contents) { *byte_field_addr(offset) = (jint) contents; } +inline void oopDesc::byte_field_put(int offset, jbyte contents) { + oopDesc* forwarded_copy = oopDesc::bs()->resolve_and_maybe_copy_oop(this); + if (forwarded_copy != this) { + return forwarded_copy->byte_field_put(offset, contents); + } + *byte_field_addr(offset) = (jint) contents; +} inline jboolean oopDesc::bool_field(int offset) const { return (jboolean) *bool_field_addr(offset); } -inline void oopDesc::bool_field_put(int offset, jboolean contents) { *bool_field_addr(offset) = (jint) contents; } +inline void oopDesc::bool_field_put(int offset, jboolean contents) { + oopDesc* forwarded_copy = oopDesc::bs()->resolve_and_maybe_copy_oop(this); + if (forwarded_copy != this) { + return forwarded_copy->bool_field_put(offset, contents); + } + *bool_field_addr(offset) = (jint) contents; +} inline jchar oopDesc::char_field(int offset) const { return (jchar) *char_field_addr(offset); } -inline void oopDesc::char_field_put(int offset, jchar contents) { *char_field_addr(offset) = (jint) contents; } +inline void oopDesc::char_field_put(int offset, jchar contents) { + oopDesc* forwarded_copy = oopDesc::bs()->resolve_and_maybe_copy_oop(this); + if (forwarded_copy != this) { + return forwarded_copy->char_field_put(offset, contents); + } + *char_field_addr(offset) = (jint) contents; +} inline jint oopDesc::int_field(int offset) const { return *int_field_addr(offset); } -inline void oopDesc::int_field_put(int offset, jint contents) { *int_field_addr(offset) = contents; } +inline void oopDesc::int_field_put(int offset, jint contents) { + oopDesc* forwarded_copy = oopDesc::bs()->resolve_and_maybe_copy_oop(this); + if (forwarded_copy != this) { + return forwarded_copy->int_field_put(offset, contents); + } + *int_field_addr(offset) = contents; +} inline jshort oopDesc::short_field(int offset) const { return (jshort) *short_field_addr(offset); } -inline void oopDesc::short_field_put(int offset, jshort contents) { *short_field_addr(offset) = (jint) contents;} +inline void oopDesc::short_field_put(int offset, jshort contents) { + oopDesc* forwarded_copy = oopDesc::bs()->resolve_and_maybe_copy_oop(this); + if (forwarded_copy != this) { + return forwarded_copy->short_field_put(offset, contents); + } + *short_field_addr(offset) = (jint) contents; +} inline jlong oopDesc::long_field(int offset) const { return *long_field_addr(offset); } -inline void oopDesc::long_field_put(int offset, jlong contents) { *long_field_addr(offset) = contents; } +inline void oopDesc::long_field_put(int offset, jlong contents) { + oopDesc* forwarded_copy = oopDesc::bs()->resolve_and_maybe_copy_oop(this); + if (forwarded_copy != this) { + return forwarded_copy->long_field_put(offset, contents); + } + *long_field_addr(offset) = contents; +} inline jfloat oopDesc::float_field(int offset) const { return *float_field_addr(offset); } -inline void oopDesc::float_field_put(int offset, jfloat contents) { *float_field_addr(offset) = contents; } +inline void oopDesc::float_field_put(int offset, jfloat contents) { + oopDesc* forwarded_copy = oopDesc::bs()->resolve_and_maybe_copy_oop(this); + if (forwarded_copy != this) { + return forwarded_copy->float_field_put(offset, contents); + } + *float_field_addr(offset) = contents; +} inline jdouble oopDesc::double_field(int offset) const { return *double_field_addr(offset); } -inline void oopDesc::double_field_put(int offset, jdouble contents) { *double_field_addr(offset) = contents; } +inline void oopDesc::double_field_put(int offset, jdouble contents) { + oopDesc* forwarded_copy = oopDesc::bs()->resolve_and_maybe_copy_oop(this); + if (forwarded_copy != this) { + return forwarded_copy->double_field_put(offset, contents); + } + *double_field_addr(offset) = contents; +} inline address oopDesc::address_field(int offset) const { return *address_field_addr(offset); } -inline void oopDesc::address_field_put(int offset, address contents) { *address_field_addr(offset) = contents; } +inline void oopDesc::address_field_put(int offset, address contents) { + oopDesc* forwarded_copy = oopDesc::bs()->resolve_and_maybe_copy_oop(this); + if (forwarded_copy != this) { + return forwarded_copy->address_field_put(offset, contents); + } + *address_field_addr(offset) = contents; +} inline oop oopDesc::obj_field_acquire(int offset) const { return UseCompressedOops ? @@ -344,37 +435,99 @@ OrderAccess::load_ptr_acquire(obj_field_addr<oop>(offset))); } inline void oopDesc::release_obj_field_put(int offset, oop value) { + oopDesc* forwarded_copy = + (oopDesc*) oopDesc::bs()->resolve_and_maybe_copy_oop(this); + + value = oopDesc::bs()->resolve_oop(value); + + if (forwarded_copy != this) + return forwarded_copy->release_obj_field_put(offset, value); + UseCompressedOops ? oop_store((volatile narrowOop*)obj_field_addr<narrowOop>(offset), value) : oop_store((volatile oop*) obj_field_addr<oop>(offset), value); } inline jbyte oopDesc::byte_field_acquire(int offset) const { return OrderAccess::load_acquire(byte_field_addr(offset)); } -inline void oopDesc::release_byte_field_put(int offset, jbyte contents) { OrderAccess::release_store(byte_field_addr(offset), contents); } +inline void oopDesc::release_byte_field_put(int offset, jbyte contents) { + oopDesc* forwarded_copy = oopDesc::bs()->resolve_and_maybe_copy_oop(this); + if (forwarded_copy != this) { + return forwarded_copy->release_bool_field_put(offset, contents); + } + OrderAccess::release_store(byte_field_addr(offset), contents); +} inline jboolean oopDesc::bool_field_acquire(int offset) const { return OrderAccess::load_acquire(bool_field_addr(offset)); } -inline void oopDesc::release_bool_field_put(int offset, jboolean contents) { OrderAccess::release_store(bool_field_addr(offset), contents); } +inline void oopDesc::release_bool_field_put(int offset, jboolean contents) { + oopDesc* forwarded_copy = oopDesc::bs()->resolve_and_maybe_copy_oop(this); + if (forwarded_copy != this) { + return forwarded_copy->release_bool_field_put(offset, contents); + } + OrderAccess::release_store(bool_field_addr(offset), contents); +} inline jchar oopDesc::char_field_acquire(int offset) const { return OrderAccess::load_acquire(char_field_addr(offset)); } -inline void oopDesc::release_char_field_put(int offset, jchar contents) { OrderAccess::release_store(char_field_addr(offset), contents); } +inline void oopDesc::release_char_field_put(int offset, jchar contents) { + oopDesc* forwarded_copy = oopDesc::bs()->resolve_and_maybe_copy_oop(this); + if (forwarded_copy != this) { + return forwarded_copy->release_char_field_put(offset, contents); + } + OrderAccess::release_store(char_field_addr(offset), contents); +} inline jint oopDesc::int_field_acquire(int offset) const { return OrderAccess::load_acquire(int_field_addr(offset)); } -inline void oopDesc::release_int_field_put(int offset, jint contents) { OrderAccess::release_store(int_field_addr(offset), contents); } +inline void oopDesc::release_int_field_put(int offset, jint contents) { + oopDesc* forwarded_copy = oopDesc::bs()->resolve_and_maybe_copy_oop(this); + if (forwarded_copy != this) { + return forwarded_copy->release_int_field_put(offset, contents); + } + OrderAccess::release_store(int_field_addr(offset), contents); +} inline jshort oopDesc::short_field_acquire(int offset) const { return (jshort)OrderAccess::load_acquire(short_field_addr(offset)); } -inline void oopDesc::release_short_field_put(int offset, jshort contents) { OrderAccess::release_store(short_field_addr(offset), contents); } +inline void oopDesc::release_short_field_put(int offset, jshort contents) { + oopDesc* forwarded_copy = oopDesc::bs()->resolve_and_maybe_copy_oop(this); + if (forwarded_copy != this) { + return forwarded_copy->release_short_field_put(offset, contents); + } + OrderAccess::release_store(short_field_addr(offset), contents); +} inline jlong oopDesc::long_field_acquire(int offset) const { return OrderAccess::load_acquire(long_field_addr(offset)); } -inline void oopDesc::release_long_field_put(int offset, jlong contents) { OrderAccess::release_store(long_field_addr(offset), contents); } +inline void oopDesc::release_long_field_put(int offset, jlong contents) { + oopDesc* forwarded_copy = oopDesc::bs()->resolve_and_maybe_copy_oop(this); + if (forwarded_copy != this) { + return forwarded_copy->release_long_field_put(offset, contents); + } + OrderAccess::release_store(long_field_addr(offset), contents); +} inline jfloat oopDesc::float_field_acquire(int offset) const { return OrderAccess::load_acquire(float_field_addr(offset)); } -inline void oopDesc::release_float_field_put(int offset, jfloat contents) { OrderAccess::release_store(float_field_addr(offset), contents); } +inline void oopDesc::release_float_field_put(int offset, jfloat contents) { + oopDesc* forwarded_copy = oopDesc::bs()->resolve_and_maybe_copy_oop(this); + if (forwarded_copy != this) { + return forwarded_copy->release_float_field_put(offset, contents); + } + OrderAccess::release_store(float_field_addr(offset), contents); +} inline jdouble oopDesc::double_field_acquire(int offset) const { return OrderAccess::load_acquire(double_field_addr(offset)); } -inline void oopDesc::release_double_field_put(int offset, jdouble contents) { OrderAccess::release_store(double_field_addr(offset), contents); } +inline void oopDesc::release_double_field_put(int offset, jdouble contents) { + oopDesc* forwarded_copy = oopDesc::bs()->resolve_and_maybe_copy_oop(this); + if (forwarded_copy != this) { + return forwarded_copy->release_double_field_put(offset, contents); + } + OrderAccess::release_store(double_field_addr(offset), contents); +} inline address oopDesc::address_field_acquire(int offset) const { return (address) OrderAccess::load_ptr_acquire(address_field_addr(offset)); } -inline void oopDesc::release_address_field_put(int offset, address contents) { OrderAccess::release_store_ptr(address_field_addr(offset), contents); } +inline void oopDesc::release_address_field_put(int offset, address contents) { + oopDesc* forwarded_copy = oopDesc::bs()->resolve_and_maybe_copy_oop(this); + if (forwarded_copy != this) { + return forwarded_copy->release_address_field_put(offset, contents); + } + OrderAccess::release_store_ptr(address_field_addr(offset), contents); +} inline int oopDesc::size_given_klass(Klass* klass) { int lh = klass->layout_helper(); @@ -529,6 +682,7 @@ if (prebarrier) { update_barrier_set_pre((oop*)dest, exchange_value); } + return (oop)Atomic::cmpxchg_ptr(exchange_value, (oop*)dest, compare_value); } } @@ -558,6 +712,10 @@ if (!Universe::heap()->is_in_reserved(obj)) return false; // obj is aligned and accessible in heap if (Universe::heap()->is_in_reserved(obj->klass_or_null())) return false; + Klass* klass = obj->klass(); + if (! Metaspace::contains(klass)) { + return false; + } // Header verification: the mark is typically non-NULL. If we're // at a safepoint, it must not be null.
--- a/src/share/vm/opto/arraycopynode.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/arraycopynode.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -254,6 +254,12 @@ return false; } + if (UseShenandoahGC && dest_elem == T_OBJECT) { + // TODO: Disabled. We need to resolve the loaded values before storing them into + // the copy object. Or call out to the clone barrier that takes care of it. + return false; + } + value_type = ary_src->elem(); base_src = src;
--- a/src/share/vm/opto/c2_globals.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/c2_globals.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -480,7 +480,7 @@ notproduct(bool, PrintEliminateLocks, false, \ "Print out when locks are eliminated") \ \ - product(bool, EliminateAutoBox, true, \ + product(bool, EliminateAutoBox, false, \ "Control optimizations for autobox elimination") \ \ diagnostic(bool, UseImplicitStableValues, true, \
--- a/src/share/vm/opto/callGenerator.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/callGenerator.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -985,7 +985,7 @@ assert(old_jvms == kit.jvms(), "generate_predicate should not change jvm state"); SafePointNode* new_map = kit.map(); assert(old_io == new_map->i_o(), "generate_predicate should not change i_o"); - assert(old_mem == new_map->memory(), "generate_predicate should not change memory"); + assert(old_mem == new_map->memory() || UseShenandoahGC, "generate_predicate should not change memory"); assert(old_exc == new_map->next_exception(), "generate_predicate should not add exceptions"); #endif if (!kit.stopped()) {
--- a/src/share/vm/opto/chaitin.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/chaitin.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -280,7 +280,25 @@ Node* proj = orig->raw_out(i); if (proj->is_MachProj()) { assert(proj->outcnt() == 0, "only kill projections are expected here"); - assert(_cfg.get_block_for_node(proj) == borig, "incorrect block for kill projections"); + +#ifdef ASSERT + if (UseShenandoahGC && _cfg.get_block_for_node(proj) != borig) { + tty->print_cr("WARNING: block of original node doesn't match block of kill projection (NULL) in Shenandoah. Consider fixing this in chaitin.cpp PhaseChaitin::clone_projs()."); + /* + tty->print_cr("orig:"); + orig->dump(3); + + orig->raw_out(i+1)->dump(3); + + tty->print_cr("\nproj:"); + proj->dump(3); + tty->print_cr("\nblock(orig):"); + borig->dump(); + tty->print_cr(""); + */ + } +#endif + assert(_cfg.get_block_for_node(proj) == borig || UseShenandoahGC, "incorrect block for kill projections"); found_projs++; // Copy kill projections after the cloned node Node* kills = proj->clone();
--- a/src/share/vm/opto/classes.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/classes.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -45,6 +45,7 @@ #include "opto/node.hpp" #include "opto/opaquenode.hpp" #include "opto/rootnode.hpp" +#include "opto/shenandoahSupport.hpp" #include "opto/subnode.hpp" #include "opto/vectornode.hpp"
--- a/src/share/vm/opto/classes.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/classes.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -228,6 +228,9 @@ macro(RoundFloat) macro(SafePoint) macro(SafePointScalarObject) +macro(ShenandoahReadBarrier) +macro(ShenandoahWriteBarrier) +macro(ShenandoahWBMemProj) macro(SCMemProj) macro(SinD) macro(SqrtD)
--- a/src/share/vm/opto/compile.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/compile.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -61,6 +61,7 @@ #include "opto/phaseX.hpp" #include "opto/rootnode.hpp" #include "opto/runtime.hpp" +#include "opto/shenandoahSupport.hpp" #include "opto/stringopts.hpp" #include "opto/type.hpp" #include "opto/vectornode.hpp" @@ -757,7 +758,8 @@ StartNode* s = new StartNode(root(), tf()->domain()); initial_gvn()->set_type_bottom(s); init_start(s); - if (method()->intrinsic_id() == vmIntrinsics::_Reference_get && UseG1GC) { + if (method()->intrinsic_id() == vmIntrinsics::_Reference_get + && (UseG1GC || UseShenandoahGC)) { // With java.lang.ref.reference.get() we must go through the // intrinsic when G1 is enabled - even when get() is the root // method of the compile - so that, if necessary, the value in @@ -1434,6 +1436,9 @@ tj = TypeInstPtr::MARK; ta = TypeAryPtr::RANGE; // generic ignored junk ptr = TypePtr::BotPTR; + } else if (offset == -8) { + // Need to distinguish brooks ptr as is. + tj = ta = TypeAryPtr::make(ptr,ta->ary(),ta->klass(),false,offset); } else { // Random constant offset into array body offset = Type::OffsetBot; // Flatten constant access into array body tj = ta = TypeAryPtr::make(ptr,ta->ary(),ta->klass(),false,offset); @@ -1498,7 +1503,7 @@ if (!is_known_inst) { // Do it only for non-instance types tj = to = TypeInstPtr::make(TypePtr::BotPTR, env()->Object_klass(), false, NULL, offset); } - } else if (offset < 0 || offset >= k->size_helper() * wordSize) { + } else if ((offset != -8) && (offset < 0 || offset >= k->size_helper() * wordSize)) { // Static fields are in the space above the normal instance // fields in the java.lang.Class instance. if (to->klass() != ciEnv::current()->Class_klass()) { @@ -1596,7 +1601,8 @@ (offset == Type::OffsetBot && tj == TypePtr::BOTTOM) || (offset == oopDesc::mark_offset_in_bytes() && tj->base() == Type::AryPtr) || (offset == oopDesc::klass_offset_in_bytes() && tj->base() == Type::AryPtr) || - (offset == arrayOopDesc::length_offset_in_bytes() && tj->base() == Type::AryPtr) , + (offset == arrayOopDesc::length_offset_in_bytes() && tj->base() == Type::AryPtr) || + (offset == -8 && tj->base() == Type::AryPtr && UseShenandoahGC), "For oops, klasses, raw offset must be constant; for arrays the offset is never known" ); assert( tj->ptr() != TypePtr::TopPTR && tj->ptr() != TypePtr::AnyNull && @@ -2850,7 +2856,7 @@ Node *m = wq.at(next); for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) { Node* use = m->fast_out(i); - if (use->is_Mem() || use->is_EncodeNarrowPtr()) { + if (use->is_Mem() || use->is_EncodeNarrowPtr() || use->is_ShenandoahBarrier()) { use->ensure_control_or_add_prec(n->in(0)); } else if (use->in(0) == NULL) { switch(use->Opcode()) { @@ -3174,6 +3180,11 @@ n->set_req(MemBarNode::Precedent, top()); } break; + case Op_ShenandoahReadBarrier: + break; + case Op_ShenandoahWriteBarrier: + n->set_req(ShenandoahBarrierNode::Memory, immutable_memory()); + break; default: assert( !n->is_Call(), "" ); assert( !n->is_Mem(), "" ); @@ -3540,7 +3551,7 @@ // Currently supported: // - G1 pre-barriers (see GraphKit::g1_write_barrier_pre()) void Compile::verify_barriers() { - if (UseG1GC) { + if (UseG1GC || UseShenandoahGC) { // Verify G1 pre-barriers const int marking_offset = in_bytes(JavaThread::satb_mark_queue_offset() + PtrQueue::byte_offset_of_active());
--- a/src/share/vm/opto/escape.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/escape.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -36,6 +36,7 @@ #include "opto/phaseX.hpp" #include "opto/movenode.hpp" #include "opto/rootnode.hpp" +#include "opto/shenandoahSupport.hpp" ConnectionGraph::ConnectionGraph(Compile * C, PhaseIterGVN *igvn) : _nodes(C->comp_arena(), C->unique(), C->unique(), NULL), @@ -530,7 +531,7 @@ // Pointer stores in G1 barriers looks like unsafe access. // Ignore such stores to be able scalar replace non-escaping // allocations. - if (UseG1GC && adr->is_AddP()) { + if ((UseG1GC || UseShenandoahGC) && adr->is_AddP()) { Node* base = get_addp_base(adr); if (base->Opcode() == Op_LoadP && base->in(MemNode::Address)->is_AddP()) { @@ -572,6 +573,12 @@ add_java_object(n, PointsToNode::ArgEscape); break; } + case Op_ShenandoahReadBarrier: + case Op_ShenandoahWriteBarrier: + // Barriers 'pass through' its arguments. I.e. what goes in, comes out. + // It doesn't escape. + add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahBarrierNode::ValueIn), delayed_worklist); + break; default: ; // Do nothing for nodes not related to EA. } @@ -766,6 +773,12 @@ } break; } + case Op_ShenandoahReadBarrier: + case Op_ShenandoahWriteBarrier: + // Barriers 'pass through' its arguments. I.e. what goes in, comes out. + // It doesn't escape. + add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahBarrierNode::ValueIn), NULL); + break; default: { // This method should be called only for EA specific nodes which may // miss some edges when they were created. @@ -885,7 +898,7 @@ } else { // An other type of call, assume the worst case: // returned value is unknown and globally escapes. - assert(call->Opcode() == Op_CallDynamicJava, "add failed case check"); + assert(call->Opcode() == Op_CallDynamicJava || call->Opcode() == Op_CallLeaf, "add failed case check"); map_ideal_node(call, phantom_obj); } } @@ -961,6 +974,9 @@ (call->as_CallLeaf()->_name != NULL && (strcmp(call->as_CallLeaf()->_name, "g1_wb_pre") == 0 || strcmp(call->as_CallLeaf()->_name, "g1_wb_post") == 0 || + strcmp(call->as_CallLeaf()->_name, "shenandoah_clone_barrier") == 0 || + strcmp(call->as_CallLeaf()->_name, "shenandoah_read_barrier") == 0 || + strcmp(call->as_CallLeaf()->_name, "shenandoah_cas_obj") == 0 || strcmp(call->as_CallLeaf()->_name, "updateBytesCRC32") == 0 || strcmp(call->as_CallLeaf()->_name, "updateBytesCRC32C") == 0 || strcmp(call->as_CallLeaf()->_name, "updateBytesAdler32") == 0 || @@ -2054,6 +2070,9 @@ } else if (adr_type->isa_aryptr()) { if (offset == arrayOopDesc::length_offset_in_bytes()) { // Ignore array length load. + } else if (UseShenandoahGC && offset == -8) { + // Shenandoah read barrier. + bt = T_ARRAY; } else if (find_second_addp(n, n->in(AddPNode::Base)) != NULL) { // Ignore first AddP. } else { @@ -2642,6 +2661,7 @@ prev = result; if (result == start_mem) break; // hit one of our sentinels + assert(result->Opcode() != Op_ShenandoahWBMemProj, "unexpected memory slice"); if (result->is_Mem()) { const Type *at = igvn->type(result->in(MemNode::Address)); if (at == Type::TOP) @@ -3005,6 +3025,7 @@ n->is_CheckCastPP() || n->is_EncodeP() || n->is_DecodeN() || + n->is_ShenandoahBarrier() || (n->is_ConstraintCast() && n->Opcode() == Op_CastPP)) { if (visited.test_set(n->_idx)) { assert(n->is_Phi(), "loops only through Phi's"); @@ -3075,6 +3096,7 @@ use->is_CheckCastPP() || use->is_EncodeNarrowPtr() || use->is_DecodeNarrowPtr() || + use->is_ShenandoahBarrier() || (use->is_ConstraintCast() && use->Opcode() == Op_CastPP)) { alloc_worklist.append_if_missing(use); #ifdef ASSERT @@ -3099,7 +3121,8 @@ if (!(op == Op_CmpP || op == Op_Conv2B || op == Op_CastP2X || op == Op_StoreCM || op == Op_FastLock || op == Op_AryEq || op == Op_StrComp || - op == Op_StrEquals || op == Op_StrIndexOf)) { + op == Op_StrEquals || op == Op_StrIndexOf || + op == Op_ShenandoahWBMemProj)) { n->dump(); use->dump(); assert(false, "EA: missing allocation reference path");
--- a/src/share/vm/opto/graphKit.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/graphKit.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -24,6 +24,8 @@ #include "precompiled.hpp" #include "compiler/compileLog.hpp" +#include "gc/shenandoah/shenandoahBarrierSet.hpp" +#include "gc/shenandoah/shenandoahHeap.hpp" #include "gc/g1/g1SATBCardTableModRefBS.hpp" #include "gc/g1/heapRegion.hpp" #include "gc/shared/barrierSet.hpp" @@ -41,6 +43,7 @@ #include "opto/parse.hpp" #include "opto/rootnode.hpp" #include "opto/runtime.hpp" +#include "opto/shenandoahSupport.hpp" #include "runtime/deoptimization.hpp" #include "runtime/sharedRuntime.hpp" @@ -815,7 +818,7 @@ // Helper function for enforcing certain bytecodes to reexecute if // deoptimization happens static bool should_reexecute_implied_by_bytecode(JVMState *jvms, bool is_anewarray) { - ciMethod* cur_method = jvms->method(); + ciMethod* cur_method = jvms->has_method() ? jvms->method() : NULL; int cur_bci = jvms->bci(); if (cur_method != NULL && cur_bci != InvocationEntryBci) { Bytecodes::Code code = cur_method->java_code_at_bci(cur_bci); @@ -1153,6 +1156,9 @@ // Special-case a fresh allocation to avoid building nodes: Node* akls = AllocateNode::Ideal_klass(obj, &_gvn); if (akls != NULL) return akls; + if (ShenandoahVerifyReadsToFromSpace) { + obj = shenandoah_read_barrier(obj); + } Node* k_adr = basic_plus_adr(obj, oopDesc::klass_offset_in_bytes()); return _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), k_adr, TypeInstPtr::KLASS)); } @@ -1163,6 +1169,10 @@ AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(array, &_gvn); Node *alen; if (alloc == NULL) { + if (ShenandoahVerifyReadsToFromSpace) { + array = shenandoah_read_barrier(array); + } + Node *r_adr = basic_plus_adr(array, arrayOopDesc::length_offset_in_bytes()); alen = _gvn.transform( new LoadRangeNode(0, immutable_memory(), r_adr, TypeInt::POS)); } else { @@ -1519,6 +1529,7 @@ set_control(ctl); switch (bs->kind()) { case BarrierSet::G1SATBCTLogging: + case BarrierSet::ShenandoahBarrierSet: g1_write_barrier_pre(do_load, obj, adr, adr_idx, val, val_type, pre_val, bt); break; @@ -1537,6 +1548,7 @@ BarrierSet* bs = Universe::heap()->barrier_set(); switch (bs->kind()) { case BarrierSet::G1SATBCTLogging: + case BarrierSet::ShenandoahBarrierSet: return true; // Can move it if no safepoint case BarrierSet::CardTableForRS: @@ -1571,6 +1583,7 @@ break; case BarrierSet::ModRef: + case BarrierSet::ShenandoahBarrierSet: break; default : @@ -1678,6 +1691,9 @@ uint nargs = call->method()->arg_size(); for (uint i = 0; i < nargs; i++) { Node* arg = argument(i); + if (ShenandoahVerifyReadsToFromSpace && call->is_CallDynamicJava() && i == 0) { + arg = shenandoah_read_barrier(arg); + } call->init_req(i + TypeFunc::Parms, arg); } } @@ -2919,6 +2935,10 @@ } } + if (ShenandoahVerifyReadsToFromSpace) { + not_null_obj = shenandoah_read_barrier(not_null_obj); + } + // Load the object's klass Node* obj_klass = load_object_klass(not_null_obj); @@ -3000,6 +3020,10 @@ Node* null_ctl = top(); Node* not_null_obj = null_check_oop(obj, &null_ctl, never_see_null, safe_for_replace, speculative_not_null); + if (ShenandoahVerifyReadsToFromSpace) { + not_null_obj = shenandoah_read_barrier(not_null_obj); + } + // If not_null_obj is dead, only null-path is taken if (stopped()) { // Doing instance-of on a NULL? set_control(null_ctl); @@ -3149,6 +3173,8 @@ assert(dead_locals_are_killed(), "should kill locals before sync. point"); + obj = shenandoah_write_barrier(obj); + // Box the stack location Node* box = _gvn.transform(new BoxLockNode(next_monitor())); Node* mem = reset_memory(); @@ -3217,6 +3243,8 @@ return; } + // obj = shenandoah_write_barrier(obj); + // Memory barrier to avoid floating things down past the locked region insert_mem_bar(Op_MemBarReleaseLock); @@ -3622,6 +3650,10 @@ if (ptr == NULL) { // reduce dumb test in callers return NULL; } + + // Attempt to see through Shenandoah barriers. + ptr = ShenandoahBarrierNode::skip_through_barrier(ptr); + if (ptr->is_CheckCastPP()) { // strip only one raw-to-oop cast ptr = ptr->in(1); if (ptr == NULL) return NULL; @@ -4274,6 +4306,16 @@ false, NULL, 0); const TypePtr* offset_field_type = string_type->add_offset(offset_offset); int offset_field_idx = C->get_alias_index(offset_field_type); + + // TODO: This is only a workaround and is probably not needed, because + // the value never changes (strings are immutable). However, if we leave + // that out, it tries to 'see' the stored value (from the initializer) and + // fails because no stores have been captured. I don't know yet, why. So we + // leave this here as workaround for now. The other option would be + // to leave this barrier here in any case, and let C2 optimize it away + // if it can prove that the object is immutable. + str = shenandoah_read_barrier(str); + return make_load(ctrl, basic_plus_adr(str, str, offset_offset), TypeInt::INT, T_INT, offset_field_idx, MemNode::unordered); @@ -4289,6 +4331,10 @@ false, NULL, 0); const TypePtr* count_field_type = string_type->add_offset(count_offset); int count_field_idx = C->get_alias_index(count_field_type); + + // TODO: See comment in load_String_offset(). + str = shenandoah_read_barrier(str); + return make_load(ctrl, basic_plus_adr(str, str, count_offset), TypeInt::INT, T_INT, count_field_idx, MemNode::unordered); @@ -4306,6 +4352,10 @@ TypeAry::make(TypeInt::CHAR,TypeInt::POS), ciTypeArrayKlass::make(T_CHAR), true, 0); int value_field_idx = C->get_alias_index(value_field_type); + + // TODO: See comment in load_String_offset(). + str = shenandoah_read_barrier(str); + Node* load = make_load(ctrl, basic_plus_adr(str, str, value_offset), value_type, T_OBJECT, value_field_idx, MemNode::unordered); // String.value field is known to be @Stable. @@ -4321,7 +4371,12 @@ false, NULL, 0); const TypePtr* offset_field_type = string_type->add_offset(offset_offset); int offset_field_idx = C->get_alias_index(offset_field_type); - store_to_memory(ctrl, basic_plus_adr(str, offset_offset), + + // TODO: See comment in load_String_offset(). + // TODO: Use incoming ctrl. + str = shenandoah_write_barrier(str); + + store_to_memory(UseShenandoahGC ? control() : ctrl, basic_plus_adr(str, offset_offset), value, T_INT, offset_field_idx, MemNode::unordered); } @@ -4331,7 +4386,12 @@ false, NULL, 0); const TypePtr* value_field_type = string_type->add_offset(value_offset); - store_oop_to_object(ctrl, str, basic_plus_adr(str, value_offset), value_field_type, + // TODO: See comment in load_String_offset(). + // TODO: Use incoming ctrl. + str = shenandoah_write_barrier(str); + value = shenandoah_read_barrier_nomem(value); + + store_oop_to_object(UseShenandoahGC ? control() : ctrl, str, basic_plus_adr(str, value_offset), value_field_type, value, TypeAryPtr::CHARS, T_OBJECT, MemNode::unordered); } @@ -4341,7 +4401,12 @@ false, NULL, 0); const TypePtr* count_field_type = string_type->add_offset(count_offset); int count_field_idx = C->get_alias_index(count_field_type); - store_to_memory(ctrl, basic_plus_adr(str, count_offset), + + // TODO: See comment in load_String_offset(). + // TODO: Use incoming ctrl. + str = shenandoah_write_barrier(str); + + store_to_memory(UseShenandoahGC ? control() : ctrl, basic_plus_adr(str, count_offset), value, T_INT, count_field_idx, MemNode::unordered); } @@ -4350,3 +4415,195 @@ // assumption of CCP analysis. return _gvn.transform(new CastPPNode(ary, ary_type->cast_to_stable(true))); } + +Node* GraphKit::shenandoah_read_barrier(Node* obj) { + return shenandoah_read_barrier_impl(obj, false, true); +} + +Node* GraphKit::shenandoah_read_barrier_nomem(Node* obj) { + return shenandoah_read_barrier_impl(obj, false, false); +} + +Node* GraphKit::shenandoah_read_barrier_impl(Node* obj, bool use_ctrl, bool use_mem) { + + if (UseShenandoahGC && ShenandoahReadBarrier) { + const Type* obj_type = obj->bottom_type(); + if (obj_type->higher_equal(TypePtr::NULL_PTR)) { + // tty->print_cr("killed barrier for NULL object"); + return obj; + } + const TypePtr* adr_type = obj_type->is_ptr()->add_offset(-8); + Node* mem = use_mem ? memory(adr_type) : immutable_memory(); + + if (! ShenandoahBarrierNode::needs_barrier(&_gvn, NULL, obj, mem)) { + // We know it is null, no barrier needed. + return obj; + } + + + if (obj_type->meet(TypePtr::NULL_PTR) == obj_type->remove_speculative()) { + + // We don't know if it's null or not. Need null-check. + enum { _not_null_path = 1, _null_path, PATH_LIMIT }; + RegionNode* region = new RegionNode(PATH_LIMIT); + Node* phi = new PhiNode(region, obj_type); + Node* null_ctrl = top(); + Node* not_null_obj = null_check_oop(obj, &null_ctrl); + + region->init_req(_null_path, null_ctrl); + phi ->init_req(_null_path, obj); + + Node* ctrl = use_ctrl ? control() : NULL; + ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, not_null_obj); + Node* n = _gvn.transform(rb); + + region->init_req(_not_null_path, control()); + phi ->init_req(_not_null_path, n); + + set_control(_gvn.transform(region)); + record_for_igvn(region); + return _gvn.transform(phi); + + } else { + // We know it is not null. Simple barrier is sufficient. + Node* ctrl = use_ctrl ? control() : NULL; + ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, obj); + Node* n = _gvn.transform(rb); + record_for_igvn(n); + return n; + } + + } else { + return obj; + } +} + +Node* GraphKit::shenandoah_write_barrier(Node* obj) { + + if (UseShenandoahGC && ShenandoahWriteBarrier) { + + if (! ShenandoahBarrierNode::needs_barrier(&_gvn, NULL, obj, NULL)) { + return obj; + } + const Type* obj_type = obj->bottom_type(); + const TypePtr* adr_type = obj_type->is_ptr()->add_offset(-8); + // tty->print_cr("memory at:"); + // adr_type->dump(); + // tty->print_cr("\n"); + // memory(adr_type)->dump(); + if (obj_type->meet(TypePtr::NULL_PTR) == obj_type->remove_speculative()) { + // We don't know if it's null or not. Need null-check. + enum { _not_null_path = 1, _null_path, PATH_LIMIT }; + RegionNode* region = new RegionNode(PATH_LIMIT); + Node* phi = new PhiNode(region, obj_type); + Node* memphi = PhiNode::make(region, memory(adr_type), Type::MEMORY, C->alias_type(adr_type)->adr_type()); + + Node* prev_mem = memory(adr_type); + Node* null_ctrl = top(); + Node* not_null_obj = null_check_oop(obj, &null_ctrl); + + region->init_req(_null_path, null_ctrl); + phi ->init_req(_null_path, null()); + memphi->init_req(_null_path, prev_mem); + + ShenandoahWriteBarrierNode* wb = new ShenandoahWriteBarrierNode(NULL, memory(adr_type), not_null_obj); + Node* n = _gvn.transform(wb); + if (n == wb) { // New barrier needs memory projection. + Node* proj = _gvn.transform(new ShenandoahWBMemProjNode(n)); + set_memory(proj, adr_type); + } + + region->init_req(_not_null_path, control()); + phi ->init_req(_not_null_path, n); + memphi->init_req(_not_null_path, memory(adr_type)); + + set_control(_gvn.transform(region)); + record_for_igvn(region); + set_memory(_gvn.transform(memphi), adr_type); + + Node* res_val = _gvn.transform(phi); + // replace_in_map(obj, res_val); + return res_val; + } else { + // We know it is not null. Simple barrier is sufficient. + ShenandoahWriteBarrierNode* wb = new ShenandoahWriteBarrierNode(NULL, memory(adr_type), obj); + Node* n = _gvn.transform(wb); + if (n == wb) { + Node* proj = _gvn.transform(new ShenandoahWBMemProjNode(wb)); + set_memory(proj, adr_type); + } + // replace_in_map(obj, n); + record_for_igvn(n); + return n; + } + + } else { + return obj; + } +} + +void GraphKit::shenandoah_acmp_barrier(Node*& a, Node*& b) { + if (UseShenandoahGC) { + const Type* a_type = a->bottom_type(); + const Type* b_type = b->bottom_type(); + if (a_type->higher_equal(TypePtr::NULL_PTR) || b_type->higher_equal(TypePtr::NULL_PTR)) { + // We know one arg is gonna be null. No need for barriers. + // tty->print_cr("eliminate acmp barrier on null"); + return; + } + /* + if ((!a_type->isa_oopptr()) || (!b_type->isa_oopptr())) { + a_type->dump(); + b_type->dump(); + } + */ + if (a_type->is_oopptr()->const_oop() != NULL && b_type->is_oopptr()->const_oop() != NULL ) { + // We know one arg is inlined constant. No need for barriers. + // tty->print_cr("eliminate acmp barrier on constant"); + return; + } + if (a->Opcode() == Op_ShenandoahWriteBarrier && b->Opcode() == Op_ShenandoahWriteBarrier) { + // We know one arg is already write-barrier'd. No need for barriers. + // tty->print_cr("eliminate acmp barrier on write barrier"); + return; + } + if (AllocateNode::Ideal_allocation(a, &_gvn) != NULL || AllocateNode::Ideal_allocation(b, &_gvn) != NULL) { + // We know one arg is already in to-space. No need for barriers. + // tty->print_cr("eliminate acmp barrier on new obj"); + return; + } + + enum { _equal = 1, _not_equal, PATH_LIMIT }; + RegionNode* region = new RegionNode(PATH_LIMIT); + PhiNode* phiA = PhiNode::make(region, a); + PhiNode* phiB = PhiNode::make(region, b); + + Node* cmp = _gvn.transform(new CmpPNode(b, a)); + Node* tst = _gvn.transform(new BoolNode(cmp, BoolTest::eq)); + + // TODO: Use profiling data. + IfNode* iff = create_and_map_if(control(), tst, PROB_FAIR, COUNT_UNKNOWN); + Node* iftrue = _gvn.transform(new IfTrueNode(iff)); + Node* iffalse = _gvn.transform(new IfFalseNode(iff)); + + // Equal path: Use original values. + region->init_req(_equal, iftrue); + phiA->init_req(_equal, a); + phiB->init_req(_equal, b); + + // Unequal path: retry after read barriers. + set_control(iffalse); + a = shenandoah_read_barrier_impl(a, true, true); + b = shenandoah_read_barrier_impl(b, true, true); + + region->init_req(_not_equal, control()); + phiA->init_req(_not_equal, a); + phiB->init_req(_not_equal, b); + + set_control(_gvn.transform(region)); + record_for_igvn(region); + + a = _gvn.transform(phiA); + b = _gvn.transform(phiB); + } +}
--- a/src/share/vm/opto/graphKit.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/graphKit.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -894,6 +894,13 @@ // Produce new array node of stable type Node* cast_array_to_stable(Node* ary, const TypeAryPtr* ary_type); + + Node* shenandoah_read_barrier(Node* obj); + Node* shenandoah_read_barrier_nomem(Node* obj); + Node* shenandoah_write_barrier(Node* obj); + void shenandoah_acmp_barrier(Node*& a, Node*& b); +private: + Node* shenandoah_read_barrier_impl(Node* obj, bool use_ctrl, bool use_mem); }; // Helper class to support building of control flow branches. Upon
--- a/src/share/vm/opto/ifg.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/ifg.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -535,7 +535,7 @@ // The method add_input_to_liveout() keeps such nodes alive (put them on liveout list) // when it sees SCMemProj node in a block. Unfortunately SCMemProj node could be placed // in block in such order that KILL MachProj nodes are processed first. - if (def->has_out_with(Op_SCMemProj)) { + if (def->has_out_with(Op_SCMemProj) || def->has_out_with(Op_ShenandoahWBMemProj)) { return false; } } @@ -683,7 +683,7 @@ JVMState* jvms = n->jvms(); uint debug_start = jvms ? jvms->debug_start() : 999999; - for (uint k = ((n->Opcode() == Op_SCMemProj) ? 0:1); k < n->req(); k++) { + for (uint k = ((n->Opcode() == Op_SCMemProj || n->Opcode() == Op_ShenandoahWBMemProj) ? 0:1); k < n->req(); k++) { Node* def = n->in(k); uint lid = _lrg_map.live_range_id(def); if (!lid) {
--- a/src/share/vm/opto/lcm.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/lcm.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -174,6 +174,8 @@ case Op_LoadRange: case Op_LoadD_unaligned: case Op_LoadL_unaligned: + case Op_ShenandoahReadBarrier: + case Op_ShenandoahWriteBarrier: assert(mach->in(2) == val, "should be address"); break; case Op_StoreB: @@ -380,7 +382,7 @@ // Should be DU safe because no edge updates. for (DUIterator_Fast jmax, j = best->fast_outs(jmax); j < jmax; j++) { Node* n = best->fast_out(j); - if( n->is_MachProj() ) { + if( n->is_MachProj() || n->Opcode() == Op_ShenandoahWBMemProj) { get_block_for_node(n)->find_remove(n); block->add_inst(n); map_node_to_block(n, block);
--- a/src/share/vm/opto/library_call.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/library_call.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -28,6 +28,8 @@ #include "classfile/vmSymbols.hpp" #include "compiler/compileBroker.hpp" #include "compiler/compileLog.hpp" +#include "gc/shenandoah/shenandoahHeap.hpp" +#include "gc/shenandoah/shenandoahRuntime.hpp" #include "oops/objArrayKlass.hpp" #include "opto/addnode.hpp" #include "opto/arraycopynode.hpp" @@ -46,6 +48,7 @@ #include "opto/opaquenode.hpp" #include "opto/parse.hpp" #include "opto/runtime.hpp" +#include "opto/shenandoahSupport.hpp" #include "opto/subnode.hpp" #include "prims/nativeLookup.hpp" #include "runtime/sharedRuntime.hpp" @@ -972,9 +975,19 @@ //------------------------------inline_string_equals------------------------ bool LibraryCallKit::inline_string_equals() { Node* receiver = null_check_receiver(); + + if (ShenandoahVerifyReadsToFromSpace) { + receiver = shenandoah_read_barrier(receiver); + } + // NOTE: Do not null check argument for String.equals() because spec // allows to specify NULL as argument. Node* argument = this->argument(1); + + if (ShenandoahVerifyReadsToFromSpace) { + argument = shenandoah_read_barrier(argument); + } + if (stopped()) { return true; } @@ -1023,6 +1036,11 @@ // Get start addr of receiver Node* receiver_val = load_String_value(no_ctrl, receiver); + + if (ShenandoahVerifyReadsToFromSpace) { + receiver_val = shenandoah_read_barrier(receiver_val); + } + Node* receiver_offset = load_String_offset(no_ctrl, receiver); Node* receiver_start = array_element_address(receiver_val, receiver_offset, T_CHAR); @@ -1031,6 +1049,11 @@ // Get start addr of argument Node* argument_val = load_String_value(no_ctrl, argument); + + if (ShenandoahVerifyReadsToFromSpace) { + argument_val = shenandoah_read_barrier(argument_val); + } + Node* argument_offset = load_String_offset(no_ctrl, argument); Node* argument_start = array_element_address(argument_val, argument_offset, T_CHAR); @@ -1067,6 +1090,10 @@ bool LibraryCallKit::inline_array_equals() { Node* arg1 = argument(0); Node* arg2 = argument(1); + + arg1 = shenandoah_read_barrier(arg1); + arg2 = shenandoah_read_barrier(arg2); + set_result(_gvn.transform(new AryEqNode(control(), memory(TypeAryPtr::CHARS), arg1, arg2))); return true; } @@ -2152,7 +2179,7 @@ // runtime filters that guard the pre-barrier code. // Also add memory barrier for non volatile load from the referent field // to prevent commoning of loads across safepoint. - if (!UseG1GC && !need_mem_bar) + if (!(UseG1GC || UseShenandoahGC) && !need_mem_bar) return; // Some compile time checks. @@ -2341,6 +2368,18 @@ if (!is_native_ptr) { // The base is either a Java object or a value produced by Unsafe.staticFieldBase Node* base = argument(1); // type: oop + if (UseShenandoahGC) { + // Note: if we don't null-check here, we generate a read barrier with a built-in + // null-check. This will later be attempted to be split on the phi, which + // results in a load on a NULL-based address on the null-path, which blows up. + // It will go away when we do late-insertion of read barriers. + base = null_check(base); + } + if (is_store) { + base = shenandoah_write_barrier(base); + } else { + base = shenandoah_read_barrier(base); + } // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset offset = argument(2); // type: long // We currently rely on the cookies produced by Unsafe.xxxFieldOffset @@ -2492,6 +2531,7 @@ if (type != T_OBJECT ) { (void) store_to_memory(control(), adr, val, type, adr_type, mo, is_volatile); } else { + val = shenandoah_read_barrier_nomem(val); // Possibly an oop being stored to Java heap or native memory if (!TypePtr::NULL_PTR->higher_equal(_gvn.type(heap_base_oop))) { // oop to Java heap. @@ -2621,6 +2661,8 @@ return true; } + base = shenandoah_write_barrier(base); + // Build field offset expression. // We currently rely on the cookies produced by Unsafe.xxxFieldOffset // to be plain byte offsets, which are also the same as those accepted @@ -2662,6 +2704,7 @@ // For now, we handle only those cases that actually exist: ints, // longs, and Object. Adding others should be straightforward. Node* load_store; + Node* result; switch(type) { case T_INT: if (kind == LS_xadd) { @@ -2673,6 +2716,7 @@ } else { ShouldNotReachHere(); } + result = load_store; break; case T_LONG: if (kind == LS_xadd) { @@ -2684,6 +2728,7 @@ } else { ShouldNotReachHere(); } + result = load_store; break; case T_OBJECT: // Transformation of a value which could be NULL pointer (CastPP #NULL) @@ -2692,6 +2737,8 @@ if (_gvn.type(newval) == TypePtr::NULL_PTR) newval = _gvn.makecon(TypePtr::NULL_PTR); + newval = shenandoah_read_barrier_nomem(newval); + // Reference stores need a store barrier. if (kind == LS_xchg) { // If pre-barrier must execute before the oop store, old value will require do_load here. @@ -2706,6 +2753,7 @@ if (_gvn.type(oldval) == TypePtr::NULL_PTR) { oldval = _gvn.makecon(TypePtr::NULL_PTR); } + // The only known value which might get overwritten is oldval. pre_barrier(false /* do_load */, control(), NULL, NULL, max_juint, NULL, NULL, @@ -2727,6 +2775,7 @@ load_store = _gvn.transform(new CompareAndSwapNNode(control(), mem, adr, newval_enc, oldval_enc)); } + result = load_store; } else #endif { @@ -2735,6 +2784,67 @@ } else { assert(kind == LS_cmpxchg, "wrong LoadStore operation"); load_store = _gvn.transform(new CompareAndSwapPNode(control(), mem, adr, newval, oldval)); + result = load_store; + + if (UseShenandoahGC) { + // if (! success) + Node* cmp_true = _gvn.transform(new CmpINode(load_store, intcon(1))); + Node* tst_true = _gvn.transform(new BoolNode(cmp_true, BoolTest::eq)); + IfNode* iff = create_and_map_if(control(), tst_true, PROB_LIKELY_MAG(2), COUNT_UNKNOWN); + Node* iftrue = _gvn.transform(new IfTrueNode(iff)); + Node* iffalse = _gvn.transform(new IfFalseNode(iff)); + + enum { _success_path = 1, _fail_path, _shenandoah_path, PATH_LIMIT }; + RegionNode* region = new RegionNode(PATH_LIMIT); + Node* phi = new PhiNode(region, TypeInt::BOOL); + // success -> return result of CAS1. + region->init_req(_success_path, iftrue); + phi ->init_req(_success_path, load_store); + + // failure + set_control(iffalse); + + // if (read_barrier(expected) == read_barrier(old) + oldval = shenandoah_read_barrier(oldval); + + // Load old value from memory. We shuold really use what we get back from the CAS, + // if we can. + Node* current = make_load(control(), adr, TypeInstPtr::BOTTOM, type, MemNode::unordered); + // read_barrier(old) + Node* new_current = shenandoah_read_barrier(current); + + Node* chk = _gvn.transform(new CmpPNode(new_current, oldval)); + Node* test = _gvn.transform(new BoolNode(chk, BoolTest::eq)); + + IfNode* iff2 = create_and_map_if(control(), test, PROB_UNLIKELY_MAG(2), COUNT_UNKNOWN); + Node* iftrue2 = _gvn.transform(new IfTrueNode(iff2)); + Node* iffalse2 = _gvn.transform(new IfFalseNode(iff2)); + + // If they are not equal, it's a legitimate failure and we return the result of CAS1. + region->init_req(_fail_path, iffalse2); + phi ->init_req(_fail_path, load_store); + + // Otherwise we retry with old. + set_control(iftrue2); + + Node *call = make_runtime_call(RC_LEAF | RC_NO_IO, + OptoRuntime::shenandoah_cas_obj_Type(), + CAST_FROM_FN_PTR(address, ShenandoahRuntime::compare_and_swap_object), + "shenandoah_cas_obj", + NULL, + adr, newval, current); + + Node* retval = _gvn.transform(new ProjNode(call, TypeFunc::Parms + 0)); + + region->init_req(_shenandoah_path, control()); + phi ->init_req(_shenandoah_path, retval); + + set_control(_gvn.transform(region)); + record_for_igvn(region); + phi = _gvn.transform(phi); + result = phi; + } + } } if (kind == LS_cmpxchg) { @@ -2744,14 +2854,14 @@ // critical path, while CAS failure path can use the penalty for going through unlikely // path as backoff. Which is still better than doing a store barrier there. IdealKit ideal(this); - ideal.if_then(load_store, BoolTest::ne, ideal.ConI(0), PROB_STATIC_FREQUENT); { + ideal.if_then(result, BoolTest::ne, ideal.ConI(0), PROB_STATIC_FREQUENT); { sync_kit(ideal); - post_barrier(ideal.ctrl(), load_store, base, adr, alias_idx, newval, T_OBJECT, true); + post_barrier(ideal.ctrl(), result, base, adr, alias_idx, newval, T_OBJECT, true); ideal.sync_kit(this); } ideal.end_if(); final_sync(ideal); } else { - post_barrier(control(), load_store, base, adr, alias_idx, newval, T_OBJECT, true); + post_barrier(control(), result, base, adr, alias_idx, newval, T_OBJECT, true); } break; default: @@ -2768,7 +2878,7 @@ if (type == T_OBJECT && kind == LS_xchg) { #ifdef _LP64 if (adr->bottom_type()->is_ptr_to_narrowoop()) { - load_store = _gvn.transform(new DecodeNNode(load_store, load_store->get_ptr_type())); + result = _gvn.transform(new DecodeNNode(result, result->get_ptr_type())); } #endif if (can_move_pre_barrier()) { @@ -2777,7 +2887,7 @@ // gets inserted between them. pre_barrier(false /* do_load */, control(), NULL, NULL, max_juint, NULL, NULL, - load_store /* pre_val */, + result /* pre_val */, T_OBJECT); } } @@ -2786,8 +2896,8 @@ insert_mem_bar(Op_MemBarCPUOrder); insert_mem_bar(Op_MemBarAcquire); - assert(type2size[load_store->bottom_type()->basic_type()] == type2size[rtype], "result type should match"); - set_result(load_store); + assert(type2size[result->bottom_type()->basic_type()] == type2size[rtype], "result type should match"); + set_result(result); return true; } @@ -2831,6 +2941,8 @@ return true; } + base = shenandoah_write_barrier(base); + // Build field offset expression. assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled"); // 32-bit machines ignore the high half of long offsets @@ -2845,8 +2957,10 @@ // Ensure that the store is atomic for longs: const bool require_atomic_access = true; Node* store; - if (type == T_OBJECT) // reference stores need a store barrier. + if (type == T_OBJECT) { // reference stores need a store barrier. + val = shenandoah_read_barrier_nomem(val); store = store_oop_to_unknown(control(), base, adr, adr_type, val, type, MemNode::release); + } else { store = store_to_memory(control(), adr, val, type, adr_type, MemNode::release, require_atomic_access); } @@ -3109,7 +3223,8 @@ // Given a klass oop, load its java mirror (a java.lang.Class oop). Node* LibraryCallKit::load_mirror_from_klass(Node* klass) { Node* p = basic_plus_adr(klass, in_bytes(Klass::java_mirror_offset())); - return make_load(NULL, p, TypeInstPtr::MIRROR, T_OBJECT, MemNode::unordered); + Node* ld = make_load(NULL, p, TypeInstPtr::MIRROR, T_OBJECT, MemNode::unordered); + return ld; } //-----------------------load_klass_from_mirror_common------------------------- @@ -3168,6 +3283,11 @@ enum { _normal_path = 1, _prim_path = 2, PATH_LIMIT }; Node* mirror = argument(0); + + if (ShenandoahVerifyReadsToFromSpace) { + mirror = shenandoah_read_barrier(mirror); + } + Node* obj = top(); switch (id) { @@ -3175,6 +3295,9 @@ // nothing is an instance of a primitive type prim_return_value = intcon(0); obj = argument(1); + if (ShenandoahVerifyReadsToFromSpace) { + obj = shenandoah_read_barrier(obj); + } break; case vmIntrinsics::_getModifiers: prim_return_value = intcon(JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC); @@ -3420,8 +3543,12 @@ bool LibraryCallKit::inline_native_subtype_check() { // Pull both arguments off the stack. Node* args[2]; // two java.lang.Class mirrors: superc, subc - args[0] = argument(0); - args[1] = argument(1); + // We need write barriers here, because for primitive types we later compare + // the two Class objects using ==, and those would give false negatives + // if one obj is in from-space, and one in to-space. + args[0] = shenandoah_write_barrier(argument(0)); + args[1] = shenandoah_write_barrier(argument(1)); + Node* klasses[2]; // corresponding Klasses: superk, subk klasses[0] = klasses[1] = top(); @@ -3728,6 +3855,8 @@ Node* orig_tail = _gvn.transform(new SubINode(orig_length, start)); Node* moved = generate_min_max(vmIntrinsics::_min, orig_tail, length); + original = shenandoah_read_barrier(original); + // Generate a direct call to the right arraycopy function(s). // We know the copy is disjoint but we might not know if the // oop stores need checking. @@ -3909,6 +4038,10 @@ result_val->init_req(_null_path, _gvn.intcon(0)); } + if (ShenandoahVerifyReadsToFromSpace) { + obj = shenandoah_read_barrier(obj); + } + // Unconditionally null? Then return right away. if (stopped()) { set_control( result_reg->in(_null_path)); @@ -4224,6 +4357,9 @@ assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled"); + src_ptr = shenandoah_read_barrier(src_ptr); + dst_ptr = shenandoah_write_barrier(dst_ptr); + Node* src = make_unsafe_address(src_ptr, src_off); Node* dst = make_unsafe_address(dst_ptr, dst_off); @@ -4252,6 +4388,8 @@ Node* raw_obj = alloc_obj->in(1); assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), ""); + obj = shenandoah_read_barrier(obj); + AllocateNode* alloc = NULL; if (ReduceBulkZeroing) { // We will be completely responsible for initializing this object - @@ -4309,6 +4447,15 @@ set_all_memory(n); } + if (UseShenandoahGC) { + // Make sure that references in the cloned object are updated for Shenandoah. + make_runtime_call(RC_LEAF|RC_NO_FP, + OptoRuntime::shenandoah_clone_barrier_Type(), + CAST_FROM_FN_PTR(address, SharedRuntime::shenandoah_clone_barrier), + "shenandoah_clone_barrier", TypePtr::BOTTOM, + alloc_obj); + } + // If necessary, emit some card marks afterwards. (Non-arrays only.) if (card_mark) { assert(!is_array, ""); @@ -4435,6 +4582,9 @@ if (is_obja != NULL) { PreserveJVMState pjvms2(this); set_control(is_obja); + + obj = shenandoah_read_barrier(obj); + // Generate a direct call to the right arraycopy function(s). Node* alloc = tightly_coupled_allocation(alloc_obj, NULL); ArrayCopyNode* ac = ArrayCopyNode::make(this, true, obj, intcon(0), alloc_obj, intcon(0), obj_length, alloc != NULL); @@ -4683,6 +4833,8 @@ Node* dest_offset = argument(3); // type: int Node* length = argument(4); // type: int + src = shenandoah_read_barrier(src); + dest = shenandoah_write_barrier(dest); // Check for allocation before we add nodes that would confuse // tightly_coupled_allocation() @@ -4922,6 +5074,8 @@ if (stopped()) return NULL; // no fast path if (C->AliasLevel() == 0) return NULL; // no MergeMems around + ptr = ShenandoahBarrierNode::skip_through_barrier(ptr); + AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(ptr, &_gvn); if (alloc == NULL) return NULL; @@ -5001,6 +5155,9 @@ Node *dst_offset = argument(3); Node *length = argument(4); + src = shenandoah_read_barrier(src); + dst = shenandoah_write_barrier(dst); + const Type* src_type = src->Value(&_gvn); const Type* dst_type = dst->Value(&_gvn); const TypeAryPtr* top_src = src_type->isa_aryptr(); @@ -5050,6 +5207,10 @@ Node* ylen = argument(3); Node* z = argument(4); + x = shenandoah_read_barrier(x); + y = shenandoah_read_barrier(y); + z = shenandoah_write_barrier(z); + const Type* x_type = x->Value(&_gvn); const Type* y_type = y->Value(&_gvn); const TypeAryPtr* top_x = x_type->isa_aryptr(); @@ -5150,6 +5311,9 @@ Node* z = argument(2); Node* zlen = argument(3); + x = shenandoah_read_barrier(x); + z = shenandoah_write_barrier(z); + const Type* x_type = x->Value(&_gvn); const Type* z_type = z->Value(&_gvn); const TypeAryPtr* top_x = x_type->isa_aryptr(); @@ -5197,6 +5361,9 @@ Node* len = argument(3); Node* k = argument(4); + in = shenandoah_read_barrier(in); + out = shenandoah_write_barrier(out); + const Type* out_type = out->Value(&_gvn); const Type* in_type = in->Value(&_gvn); const TypeAryPtr* top_out = out_type->isa_aryptr(); @@ -5246,6 +5413,11 @@ Node* inv = argument(4); Node* m = argument(6); + a = shenandoah_read_barrier(a); + b = shenandoah_read_barrier(b); + n = shenandoah_read_barrier(n); + m = shenandoah_write_barrier(m); + const Type* a_type = a->Value(&_gvn); const TypeAryPtr* top_a = a_type->isa_aryptr(); const Type* b_type = b->Value(&_gvn); @@ -5305,6 +5477,10 @@ Node* inv = argument(3); Node* m = argument(5); + a = shenandoah_read_barrier(a); + n = shenandoah_read_barrier(n); + m = shenandoah_write_barrier(m); + const Type* a_type = a->Value(&_gvn); const TypeAryPtr* top_a = a_type->isa_aryptr(); const Type* n_type = a->Value(&_gvn); @@ -5391,6 +5567,8 @@ Node* offset = argument(2); // type: int Node* length = argument(3); // type: int + src = shenandoah_read_barrier(src); + const Type* src_type = src->Value(&_gvn); const TypeAryPtr* top_src = src_type->isa_aryptr(); if (top_src == NULL || top_src->klass() == NULL) { @@ -5645,6 +5823,10 @@ Node* reference_obj = null_check_receiver(); if (stopped()) return true; + if (ShenandoahVerifyReadsToFromSpace) { + reference_obj = shenandoah_read_barrier(reference_obj); + } + Node* adr = basic_plus_adr(reference_obj, reference_obj, referent_offset); ciInstanceKlass* klass = env()->Object_klass(); @@ -5693,6 +5875,8 @@ fromObj = makecon(tip); } + fromObj = shenandoah_read_barrier(fromObj); + // Next code copied from Parse::do_get_xxx(): // Compute address and memory type. @@ -5753,6 +5937,10 @@ Node* dest = argument(3); Node* dest_offset = argument(4); + // Resolve src and dest arrays for ShenandoahGC. + src = shenandoah_read_barrier(src); + dest = shenandoah_write_barrier(dest); + // (1) src and dest are arrays. const Type* src_type = src->Value(&_gvn); const Type* dest_type = dest->Value(&_gvn); @@ -5821,6 +6009,10 @@ Node* dest = argument(4); Node* dest_offset = argument(5); + // Resolve src and dest arrays for ShenandoahGC. + src = shenandoah_read_barrier(src); + dest = shenandoah_write_barrier(dest); + // (1) src and dest are arrays. const Type* src_type = src->Value(&_gvn); const Type* dest_type = dest->Value(&_gvn); @@ -5865,6 +6057,9 @@ // similarly, get the start address of the r vector Node* objRvec = load_field_from_object(cipherBlockChaining_object, "r", "[B", /*is_exact*/ false); + + objRvec = shenandoah_write_barrier(objRvec); + if (objRvec == NULL) return false; Node* r_start = array_element_address(objRvec, intcon(0), T_BYTE); @@ -5900,6 +6095,8 @@ assert (objAESCryptKey != NULL, "wrong version of com.sun.crypto.provider.AESCrypt"); if (objAESCryptKey == NULL) return (Node *) NULL; + objAESCryptKey = shenandoah_read_barrier(objAESCryptKey); + // now have the array, need to get the start address of the K array Node* k_start = array_element_address(objAESCryptKey, intcon(0), T_INT); return k_start;
--- a/src/share/vm/opto/locknode.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/locknode.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -184,6 +184,7 @@ // Null check; get casted pointer. Node* obj = null_check(peek()); + // Check for locking null object if (stopped()) return; @@ -203,5 +204,6 @@ // Because monitors are guaranteed paired (else we bail out), we know // the matching Lock for this Unlock. Hence we know there is no need // for a null check on Unlock. - shared_unlock(map()->peek_monitor_box(), map()->peek_monitor_obj()); + Node* obj = map()->peek_monitor_obj(); + shared_unlock(map()->peek_monitor_box(), obj); }
--- a/src/share/vm/opto/loopPredicate.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/loopPredicate.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -441,7 +441,7 @@ // loop, it was marked invariant but n is only invariant if // it depends only on that test. Otherwise, unless that test // is out of the loop, it's not invariant. - if (n->is_CFG() || n->depends_only_on_test() || n->in(0) == NULL || !_phase->is_member(_lpt, n->in(0))) { + if (n->Opcode() == Op_ShenandoahWBMemProj || n->is_CFG() || n->depends_only_on_test() || n->in(0) == NULL || !_phase->is_member(_lpt, n->in(0))) { _invariant.set(n->_idx); // I am a invariant too } }
--- a/src/share/vm/opto/loopTransform.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/loopTransform.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -927,6 +927,7 @@ // Check for 'n' being pinned in the backedge. if( n->in(0) && n->in(0) == back_ctrl ) { assert(clones.find(n->_idx) == NULL, "dead loop"); + assert(n->Opcode() != Op_ShenandoahWriteBarrier, "no wbs yet"); x = n->clone(); // Clone a copy of 'n' to preheader clones.push(x, n->_idx); x->set_req( 0, preheader_ctrl ); // Fix x's control input to preheader
--- a/src/share/vm/opto/loopUnswitch.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/loopUnswitch.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -180,6 +180,7 @@ ProjNode* invar_proj = invar_iff->proj_out(proj->_con)->as_Proj(); while (worklist.size() > 0) { Node* use = worklist.pop(); + assert(use->Opcode() != Op_ShenandoahWriteBarrier, "not with wbs yet"); Node* nuse = use->clone(); nuse->set_req(0, invar_proj); _igvn.replace_input_of(use, 1, nuse);
--- a/src/share/vm/opto/loopnode.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/loopnode.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -3089,7 +3089,7 @@ ++i; if (in == NULL) continue; if (in->pinned() && !in->is_CFG()) - set_ctrl(in, in->in(0)); + set_ctrl(in, in->Opcode() == Op_ShenandoahWBMemProj ? in->in(0)->in(0) : in->in(0)); int is_visited = visited.test_set( in->_idx ); if (!has_node(in)) { // No controlling input yet? assert( !in->is_CFG(), "CFG Node with no controlling input?" ); @@ -3274,7 +3274,7 @@ } } else { Node *sctrl = has_ctrl(s) ? get_ctrl(s) : s->in(0); - assert(sctrl != NULL || s->outcnt() == 0, "must have control"); + assert(sctrl != NULL || s->outcnt() == 0 || s->is_ShenandoahBarrier(), "must have control"); if (sctrl != NULL && !sctrl->is_top() && is_dominator(early, sctrl)) { LCA = dom_lca_for_get_late_ctrl(LCA, sctrl, n); } @@ -3495,6 +3495,8 @@ case Op_StrEquals: case Op_StrIndexOf: case Op_AryEq: + case Op_ShenandoahReadBarrier: + case Op_ShenandoahWriteBarrier: pinned = false; } if( pinned ) {
--- a/src/share/vm/opto/loopnode.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/loopnode.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -1062,6 +1062,8 @@ Node *split_thru_region( Node *n, Node *region ); // Split Node 'n' through merge point if there is enough win. Node *split_thru_phi( Node *n, Node *region, int policy ); + void split_mem_thru_phi(Node*, Node* r, Node* phi); + // Found an If getting its condition-code input from a Phi in the // same block. Split thru the Region. void do_split_if( Node *iff );
--- a/src/share/vm/opto/loopopts.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/loopopts.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -33,6 +33,7 @@ #include "opto/movenode.hpp" #include "opto/opaquenode.hpp" #include "opto/rootnode.hpp" +#include "opto/shenandoahSupport.hpp" #include "opto/subnode.hpp" //============================================================================= @@ -67,6 +68,9 @@ if (region->in(i) == C->top()) { x = C->top(); // Dead path? Use a dead data op } else { +#ifdef ASSERT + if (n->is_ShenandoahBarrier()) { n->as_ShenandoahBarrier()->check_invariants(); } +#endif x = n->clone(); // Else clone up the data op the_clone = x; // Remember for possible deletion. // Alter data node to use pre-phi inputs @@ -111,21 +115,26 @@ // otherwise it will be not updated during igvn->transform since // igvn->type(x) is set to x->Value() already. x->raise_bottom_type(t); - Node *y = x->Identity(&_igvn); - if (y != x) { - wins++; - x = y; + if (x->Opcode() != Op_ShenandoahWriteBarrier) { + Node *y = x->Identity(&_igvn); + if (y != x) { + wins++; + x = y; + } else { + y = _igvn.hash_find(x); + if (y) { + wins++; + x = y; + } else { + // Else x is a new node we are keeping + // We do not need register_new_node_with_optimizer + // because set_type has already been called. + _igvn._worklist.push(x); + } + } } else { - y = _igvn.hash_find(x); - if (y) { - wins++; - x = y; - } else { - // Else x is a new node we are keeping - // We do not need register_new_node_with_optimizer - // because set_type has already been called. - _igvn._worklist.push(x); - } + // wins++; + _igvn._worklist.push(x); } } if (x != the_clone && the_clone != NULL) @@ -194,6 +203,38 @@ return phi; } +void PhaseIdealLoop::split_mem_thru_phi(Node* n, Node* r, Node* phi) { + if (n->Opcode() == Op_ShenandoahWriteBarrier) { +#ifdef ASSERT + n->as_ShenandoahBarrier()->check_invariants(); +#endif + if (n->has_out_with(Op_ShenandoahWBMemProj)) { + Node* old_mem_phi = n->in(ShenandoahBarrierNode::Memory); + assert(r->is_Region(), "need region to control phi"); + assert(phi->is_Phi(), "expect phi"); + Node* memphi = PhiNode::make(r, old_mem_phi, Type::MEMORY, C->alias_type(n->adr_type())->adr_type()); + for (uint i = 1; i < r->req(); i++) { + Node* wb = phi->in(i); + if (wb->Opcode() == Op_ShenandoahWriteBarrier) { + assert(! wb->has_out_with(Op_ShenandoahWBMemProj), "new clone does not have mem proj"); + Node* new_proj = new ShenandoahWBMemProjNode(wb); + register_new_node(new_proj, r->in(i)); + memphi->set_req(i, new_proj); + } else { + if (old_mem_phi->is_Phi() && old_mem_phi->in(0) == r) { + memphi->set_req(i, old_mem_phi->in(i)); + } + } + } + register_new_node(memphi, r); + Node* old_mem_out = n->find_out_with(Op_ShenandoahWBMemProj); + assert(old_mem_out != NULL, "expect memory projection"); + _igvn.replace_node(old_mem_out, memphi); + } + assert(! n->has_out_with(Op_ShenandoahWBMemProj), "no more memory outs"); + } +} + //------------------------------dominated_by------------------------------------ // Replace the dominated test with an obvious true or false. Place it on the // IGVN worklist for later cleanup. Move control-dependent data Nodes on the @@ -934,12 +975,19 @@ // Found a Phi to split thru! // Replace 'n' with the new phi + split_mem_thru_phi(n, n_blk, phi); _igvn.replace_node( n, phi ); + // Moved a load around the loop, 'en-registering' something. if (n_blk->is_Loop() && n->is_Load() && !phi->in(LoopNode::LoopBackControl)->is_Load()) C->set_major_progress(); + // Moved a barrier around the loop, 'en-registering' something. + if (n_blk->is_Loop() && n->is_ShenandoahBarrier() && + !phi->in(LoopNode::LoopBackControl)->is_ShenandoahBarrier()) + C->set_major_progress(); + return phi; } @@ -1082,9 +1130,11 @@ // Found a Phi to split thru! // Replace 'n' with the new phi + assert(n->Opcode() != Op_ShenandoahWriteBarrier, "not with write barriers yet"); _igvn.replace_node( n, phi ); // Now split the bool up thru the phi + assert(bol->Opcode() != Op_ShenandoahWriteBarrier, "not with write barriers yet"); Node *bolphi = split_thru_phi( bol, n_ctrl, -1 ); guarantee(bolphi != NULL, "null boolean phi node"); @@ -1096,6 +1146,7 @@ // Conditional-move? Must split up now if( !iff->is_If() ) { + assert(iff->Opcode() != Op_ShenandoahWriteBarrier, "not with write barriers yet"); Node *cmovphi = split_thru_phi( iff, n_ctrl, -1 ); _igvn.replace_node( iff, cmovphi ); return; @@ -1229,7 +1280,7 @@ // to fold a StoreP and an AddP together (as part of an // address expression) and the AddP and StoreP have // different controls. - if (!x->is_Load() && !x->is_DecodeNarrowPtr()) _igvn._worklist.yank(x); + if (!x->is_Load() && !x->is_DecodeNarrowPtr() && !x->is_ShenandoahBarrier()) _igvn._worklist.yank(x); } _igvn.remove_dead_node(n); } @@ -1502,6 +1553,16 @@ uint i; for( i = 0; i < loop->_body.size(); i++ ) { Node *old = loop->_body.at(i); +#ifdef ASSERT + if (old->Opcode() == Op_ShenandoahWriteBarrier) { + Node* memproj = old->find_out_with(Op_ShenandoahWBMemProj); + assert(memproj == NULL || (loop->is_member(get_loop(has_ctrl(memproj) ? get_ctrl(memproj) : memproj))), "WB's mem-proj must be in loop too"); + } + if (old->Opcode() == Op_ShenandoahWBMemProj) { + Node* wb = old->in(0); + assert(loop->is_member(get_loop(has_ctrl(wb) ? get_ctrl(wb) : wb)), "WB must be in loop too"); + } +#endif Node *nnn = old->clone(); old_new.map( old->_idx, nnn ); if (C->do_vector_loop()) { @@ -1692,12 +1753,13 @@ // private Phi and those Phis need to be merged here. Node *phi; if( prev->is_Region() ) { - if( idx == 0 ) { // Updating control edge? + if( idx == 0 && use->Opcode() != Op_ShenandoahWBMemProj) { // Updating control edge? phi = prev; // Just use existing control } else { // Else need a new Phi phi = PhiNode::make( prev, old ); // Now recursively fix up the new uses of old! - for( uint i = 1; i < prev->req(); i++ ) { + uint first = use->Opcode() != Op_ShenandoahWBMemProj ? 1 : 0; + for( uint i = first; i < prev->req(); i++ ) { worklist.push(phi); // Onto worklist once for each 'old' input } } @@ -1705,7 +1767,7 @@ // Get new RegionNode merging old and new loop exits prev = old_new[prev->_idx]; assert( prev, "just made this in step 7" ); - if( idx == 0 ) { // Updating control edge? + if( idx == 0 && use->Opcode() != Op_ShenandoahWBMemProj) { // Updating control edge? phi = prev; // Just use existing control } else { // Else need a new Phi // Make a new Phi merging data values properly @@ -2188,6 +2250,7 @@ int PhaseIdealLoop::clone_for_use_outside_loop( IdealLoopTree *loop, Node* n, Node_List& worklist ) { int cloned = 0; assert(worklist.size() == 0, "should be empty"); + assert(n->Opcode() != Op_ShenandoahWriteBarrier, "not with write barriers yet"); for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) { Node* use = n->fast_out(j); if( !loop->is_member(get_loop(has_ctrl(use) ? get_ctrl(use) : use)) ) { @@ -2204,6 +2267,7 @@ assert(j < use->req(), "must be there"); // clone "n" and insert it between the inputs of "n" and the use outside the loop + assert(n->Opcode() != Op_ShenandoahWriteBarrier, "not with write barriers"); Node* n_clone = n->clone(); _igvn.replace_input_of(use, j, n_clone); cloned++; @@ -2250,6 +2314,7 @@ } if (worklist.size() > 0) { // clone "n" and insert it between inputs of "n" and the use + assert(n->Opcode() != Op_ShenandoahWriteBarrier, "not with write barriers"); Node* n_clone = n->clone(); loop->_body.push(n_clone); _igvn.register_new_node_with_optimizer(n_clone);
--- a/src/share/vm/opto/macro.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/macro.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -42,6 +42,7 @@ #include "opto/phaseX.hpp" #include "opto/rootnode.hpp" #include "opto/runtime.hpp" +#include "opto/shenandoahSupport.hpp" #include "opto/subnode.hpp" #include "opto/type.hpp" #include "runtime/sharedRuntime.hpp" @@ -933,6 +934,9 @@ field_val = transform_later(new DecodeNNode(field_val, field_val->get_ptr_type())); } } + if (field_val->isa_ShenandoahBarrier()) { + field_val = field_val->in(ShenandoahBarrierNode::ValueIn); + } sfpt->add_req(field_val); } JVMState *jvms = sfpt->jvms(); @@ -1411,6 +1415,13 @@ transform_later(old_eden_top); // Add to heap top to get a new heap top + + if (UseShenandoahGC) { + // Allocate one word more for the Shenandoah brooks pointer. + size_in_bytes = new AddLNode(size_in_bytes, _igvn.MakeConX(8)); + transform_later(size_in_bytes); + } + Node *new_eden_top = new AddPNode(top(), old_eden_top, size_in_bytes); transform_later(new_eden_top); // Check for needing a GC; compare against heap end @@ -1501,6 +1512,13 @@ 0, new_alloc_bytes, T_LONG); } + if (UseShenandoahGC) { + // Bump up object by one word. The preceding word is used for + // the Shenandoah brooks pointer. + fast_oop = new AddPNode(top(), fast_oop, _igvn.MakeConX(8)); + transform_later(fast_oop); + } + InitializeNode* init = alloc->initialization(); fast_oop_rawmem = initialize_object(alloc, fast_oop_ctrl, fast_oop_rawmem, fast_oop, @@ -1778,6 +1796,11 @@ header_size = Klass::layout_helper_header_size(k->layout_helper()); } + if (UseShenandoahGC) { + // Initialize Shenandoah brooks pointer to point to the object itself. + rawmem = make_store(control, rawmem, object, -8, object, T_OBJECT); + } + // Clear the object body, if necessary. if (init == NULL) { // The init has somehow disappeared; be cautious and clear everything. @@ -2692,7 +2715,7 @@ while (macro_idx >= 0) { Node * n = C->macro_node(macro_idx); assert(n->is_macro(), "only macro nodes expected here"); - if (_igvn.type(n) == Type::TOP || n->in(0)->is_top() ) { + if (_igvn.type(n) == Type::TOP || n->in(0)->is_top()) { // node is unreachable, so don't try to expand it C->remove_macro_node(n); } else if (n->is_ArrayCopy()){ @@ -2710,7 +2733,7 @@ int macro_count = C->macro_count(); Node * n = C->macro_node(macro_count-1); assert(n->is_macro(), "only macro nodes expected here"); - if (_igvn.type(n) == Type::TOP || n->in(0)->is_top() ) { + if (_igvn.type(n) == Type::TOP || n->in(0)->is_top()) { // node is unreachable, so don't try to expand it C->remove_macro_node(n); continue;
--- a/src/share/vm/opto/macro.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/macro.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -42,6 +42,8 @@ Node* intcon(jint con) const { return _igvn.intcon(con); } Node* longcon(jlong con) const { return _igvn.longcon(con); } Node* makecon(const Type *t) const { return _igvn.makecon(t); } + Node* zerocon(BasicType bt) const { return _igvn.zerocon(bt); } + Node* basic_plus_adr(Node* base, int offset) { return (offset == 0)? base: basic_plus_adr(base, MakeConX(offset)); }
--- a/src/share/vm/opto/macroArrayCopy.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/macroArrayCopy.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -547,7 +547,7 @@ // At this point we know we do not need type checks on oop stores. // Let's see if we need card marks: - if (alloc != NULL && GraphKit::use_ReduceInitialCardMarks()) { + if (alloc != NULL && GraphKit::use_ReduceInitialCardMarks() && ! UseShenandoahGC) { // If we do not need card marks, copy using the jint or jlong stub. copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT); assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type),
--- a/src/share/vm/opto/matcher.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/matcher.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -2156,6 +2156,11 @@ case Op_SafePoint: mem_op = true; break; + case Op_ShenandoahReadBarrier: + case Op_ShenandoahWriteBarrier: + mem_op = true; + set_shared(n); + break; default: if( n->is_Store() ) { // Do match stores, despite no ideal reg
--- a/src/share/vm/opto/memnode.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/memnode.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -41,6 +41,7 @@ #include "opto/narrowptrnode.hpp" #include "opto/phaseX.hpp" #include "opto/regmask.hpp" +#include "opto/shenandoahSupport.hpp" #include "utilities/copy.hpp" // Portions of code courtesy of Clifford Click @@ -966,6 +967,7 @@ // Now prove that we have a LoadQ matched to a StoreQ, for some Q. if (store_Opcode() != st->Opcode()) return NULL; + //tty->print_cr("can_see_stored_value 1"); return st->in(MemNode::ValueIn); } @@ -979,6 +981,7 @@ // (This is one of the few places where a generic PhaseTransform // can create new nodes. Think of it as lazily manifesting // virtually pre-existing constants.) + // tty->print_cr("can_see_stored_value 2"); return phase->zerocon(memory_type()); } @@ -1000,10 +1003,13 @@ (tp != NULL) && tp->is_ptr_to_boxed_value()) { intptr_t ignore = 0; Node* base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ignore); + base = ShenandoahBarrierNode::skip_through_barrier(base); if (base != NULL && base->is_Proj() && base->as_Proj()->_con == TypeFunc::Parms && base->in(0)->is_CallStaticJava() && base->in(0)->as_CallStaticJava()->is_boxing_method()) { + + //tty->print_cr("can_see_stored_value 3"); return base->in(0)->in(TypeFunc::Parms); } } @@ -1603,7 +1609,7 @@ // as to alignment, which will therefore produce the smallest // possible base offset. const int min_base_off = arrayOopDesc::base_offset_in_bytes(T_BYTE); - const bool off_beyond_header = ((uint)off >= (uint)min_base_off); + const bool off_beyond_header = (off != -8 || !UseShenandoahGC) && ((uint)off >= (uint)min_base_off); // Try to constant-fold a stable array element. if (FoldStableValues && ary->is_stable() && ary->const_oop() != NULL) {
--- a/src/share/vm/opto/multnode.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/multnode.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -31,6 +31,7 @@ #include "opto/opcodes.hpp" #include "opto/phaseX.hpp" #include "opto/regmask.hpp" +#include "opto/shenandoahSupport.hpp" #include "opto/type.hpp" //============================================================================= @@ -141,6 +142,7 @@ if (n->is_Mach()) return; // mach. projs. are not type-safe if (n->is_Start()) return; // alas, starts can have mach. projs. also if (_con == SCMemProjNode::SCMEMPROJCON ) return; + if (_con == ShenandoahWBMemProjNode::SWBMEMPROJCON ) return; const Type* t = n->bottom_type(); if (t == Type::TOP) return; // multi is dead assert(_con < t->is_tuple()->cnt(), "ProjNode::_con must be in range");
--- a/src/share/vm/opto/node.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/node.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -130,6 +130,7 @@ class RootNode; class SafePointNode; class SafePointScalarObjectNode; +class ShenandoahBarrierNode; class StartNode; class State; class StoreNode; @@ -621,6 +622,7 @@ DEFINE_CLASS_ID(EncodeNarrowPtr, Type, 6) DEFINE_CLASS_ID(EncodeP, EncodeNarrowPtr, 0) DEFINE_CLASS_ID(EncodePKlass, EncodeNarrowPtr, 1) + DEFINE_CLASS_ID(ShenandoahBarrier, Type, 7) DEFINE_CLASS_ID(Proj, Node, 3) DEFINE_CLASS_ID(CatchProj, Proj, 0) @@ -688,10 +690,6 @@ assert(c <= _max_classes, "invalid node class"); _class_id = c; // cast out const } - void init_flags(jushort fl) { - assert(fl <= _max_flags, "invalid node flag"); - _flags |= fl; - } void clear_flag(jushort fl) { assert(fl <= _max_flags, "invalid node flag"); _flags &= ~fl; @@ -701,6 +699,10 @@ const jushort class_id() const { return _class_id; } const jushort flags() const { return _flags; } + void init_flags(jushort fl) { + assert(fl <= _max_flags, "invalid node flag"); + _flags |= fl; + } void add_flag(jushort fl) { init_flags(fl); } @@ -800,6 +802,7 @@ DEFINE_CLASS_QUERY(Root) DEFINE_CLASS_QUERY(SafePoint) DEFINE_CLASS_QUERY(SafePointScalarObject) + DEFINE_CLASS_QUERY(ShenandoahBarrier) DEFINE_CLASS_QUERY(Start) DEFINE_CLASS_QUERY(Store) DEFINE_CLASS_QUERY(Sub)
--- a/src/share/vm/opto/output.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/output.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -907,8 +907,8 @@ !method->is_synchronized() || method->is_native() || num_mon > 0 || - !GenerateSynchronizationCode, - "monitors must always exist for synchronized methods"); + !GenerateSynchronizationCode || (UseShenandoahGC && jvms->bci() < 0), + err_msg("monitors must always exist for synchronized methods, bci: %d", jvms->bci())); // Build the growable array of ScopeValues for exp stack GrowableArray<MonitorValue*> *monarray = new GrowableArray<MonitorValue*>(num_mon);
--- a/src/share/vm/opto/parse.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/parse.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -483,7 +483,7 @@ // Helper function to generate array store void array_store(BasicType etype); // Helper function to compute array addressing - Node* array_addressing(BasicType type, int vals, const Type* *result2=NULL); + Node* array_addressing(BasicType type, int vals, bool is_store, const Type* *result2=NULL); void rtm_deopt();
--- a/src/share/vm/opto/parse1.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/parse1.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -132,7 +132,8 @@ } default: ShouldNotReachHere(); } - return _gvn.transform(l); + l = _gvn.transform(l); + return l; } // Helper routine to prevent the interpreter from handing
--- a/src/share/vm/opto/parse2.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/parse2.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -50,18 +50,19 @@ //---------------------------------array_load---------------------------------- void Parse::array_load(BasicType elem_type) { const Type* elem = Type::TOP; - Node* adr = array_addressing(elem_type, 0, &elem); + Node* adr = array_addressing(elem_type, 0, false, &elem); if (stopped()) return; // guaranteed null or range check dec_sp(2); // Pop array and index const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(elem_type); Node* ld = make_load(control(), adr, elem, elem_type, adr_type, MemNode::unordered); + assert(elem_type != T_ARRAY, "doesn't happen, right?"); push(ld); } //--------------------------------array_store---------------------------------- void Parse::array_store(BasicType elem_type) { - Node* adr = array_addressing(elem_type, 1); + Node* adr = array_addressing(elem_type, 1, true); if (stopped()) return; // guaranteed null or range check Node* val = pop(); dec_sp(2); // Pop array and index @@ -72,7 +73,7 @@ //------------------------------array_addressing------------------------------- // Pull array and index from the stack. Compute pointer-to-element. -Node* Parse::array_addressing(BasicType type, int vals, const Type* *result2) { +Node* Parse::array_addressing(BasicType type, int vals, bool is_store, const Type* *result2) { Node *idx = peek(0+vals); // Get from stack without popping Node *ary = peek(1+vals); // in case of exception @@ -158,6 +159,12 @@ // Check for always knowing you are throwing a range-check exception if (stopped()) return top(); + if (is_store) { + ary = shenandoah_write_barrier(ary); + } else { + ary = shenandoah_read_barrier(ary); + } + Node* ptr = array_element_address(ary, idx, type, sizetype); if (result2 != NULL) *result2 = elemtype; @@ -1696,14 +1703,14 @@ case Bytecodes::_faload: array_load(T_FLOAT); break; case Bytecodes::_aaload: array_load(T_OBJECT); break; case Bytecodes::_laload: { - a = array_addressing(T_LONG, 0); + a = array_addressing(T_LONG, 0, false); if (stopped()) return; // guaranteed null or range check dec_sp(2); // Pop array and index push_pair(make_load(control(), a, TypeLong::LONG, T_LONG, TypeAryPtr::LONGS, MemNode::unordered)); break; } case Bytecodes::_daload: { - a = array_addressing(T_DOUBLE, 0); + a = array_addressing(T_DOUBLE, 0, false); if (stopped()) return; // guaranteed null or range check dec_sp(2); // Pop array and index push_pair(make_load(control(), a, Type::DOUBLE, T_DOUBLE, TypeAryPtr::DOUBLES, MemNode::unordered)); @@ -1715,7 +1722,7 @@ case Bytecodes::_sastore: array_store(T_SHORT); break; case Bytecodes::_fastore: array_store(T_FLOAT); break; case Bytecodes::_aastore: { - d = array_addressing(T_OBJECT, 1); + d = array_addressing(T_OBJECT, 1, true); if (stopped()) return; // guaranteed null or range check array_store_check(); c = pop(); // Oop to store @@ -1723,12 +1730,17 @@ a = pop(); // the array itself const TypeOopPtr* elemtype = _gvn.type(a)->is_aryptr()->elem()->make_oopptr(); const TypeAryPtr* adr_type = TypeAryPtr::OOPS; + // Note: We don't need a write barrier for Shenandoah on a here, because + // a is not used except for an assert. The address d already has the + // write barrier. Adding a barrier on a only results in additional code + // being generated. + c = shenandoah_read_barrier_nomem(c); Node* store = store_oop_to_array(control(), a, d, adr_type, c, elemtype, T_OBJECT, StoreNode::release_if_reference(T_OBJECT)); break; } case Bytecodes::_lastore: { - a = array_addressing(T_LONG, 2); + a = array_addressing(T_LONG, 2, true); if (stopped()) return; // guaranteed null or range check c = pop_pair(); dec_sp(2); // Pop array and index @@ -1736,7 +1748,7 @@ break; } case Bytecodes::_dastore: { - a = array_addressing(T_DOUBLE, 2); + a = array_addressing(T_DOUBLE, 2, true); if (stopped()) return; // guaranteed null or range check c = pop_pair(); dec_sp(2); // Pop array and index @@ -2276,6 +2288,7 @@ maybe_add_safepoint(iter().get_dest()); a = pop(); b = pop(); + shenandoah_acmp_barrier(a, b); c = _gvn.transform( new CmpPNode(b, a) ); c = optimize_cmp_with_klass(c); do_if(btest, c);
--- a/src/share/vm/opto/parse3.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/parse3.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -162,6 +162,10 @@ // Compute address and memory type. int offset = field->offset_in_bytes(); const TypePtr* adr_type = C->alias_type(field)->adr_type(); + + // Insert read barrier for Shenandoah. + obj = shenandoah_read_barrier(obj); + Node *adr = basic_plus_adr(obj, obj, offset); BasicType bt = field->layout_type(); @@ -244,6 +248,9 @@ // another volatile read. if (is_vol) insert_mem_bar(Op_MemBarRelease); + // Insert write barrier for Shenandoah. + obj = shenandoah_write_barrier(obj); + // Compute address and memory type. int offset = field->offset_in_bytes(); const TypePtr* adr_type = C->alias_type(field)->adr_type(); @@ -273,6 +280,9 @@ } else { field_type = TypeOopPtr::make_from_klass(field->type()->as_klass()); } + + val = shenandoah_read_barrier_nomem(val); + store = store_oop_to_object(control(), obj, adr, adr_type, val, field_type, bt, mo); } else { bool needs_atomic_access = is_vol || AlwaysAtomicAccesses;
--- a/src/share/vm/opto/parseHelper.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/parseHelper.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -144,6 +144,11 @@ Node *idx = peek(1); Node *ary = peek(2); + if (ShenandoahVerifyReadsToFromSpace) { + obj = shenandoah_read_barrier(obj); + ary = shenandoah_read_barrier(ary); + } + if (_gvn.type(obj) == TypePtr::NULL_PTR) { // There's never a type check on null values. // This cutout lets us avoid the uncommon_trap(Reason_array_check)
--- a/src/share/vm/opto/phaseX.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/phaseX.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -34,6 +34,7 @@ #include "opto/phaseX.hpp" #include "opto/regalloc.hpp" #include "opto/rootnode.hpp" +#include "opto/shenandoahSupport.hpp" //============================================================================= #define NODE_HASH_MINIMUM_SIZE 255 @@ -565,6 +566,38 @@ #endif +bool PhaseTransform::eqv(const Node* n1, const Node* n2) const { + if (UseShenandoahGC) { + + if (n1 == n2) return true; + + if (n1 == NULL || n2 == NULL) return false; + + if (n1->is_AddP() && n2->is_AddP()) { + Node* addp1 = n1->as_AddP(); + Node* base1 = addp1->in(AddPNode::Base); + Node* addr1 = addp1->in(AddPNode::Address); + Node* offs1 = addp1->in(AddPNode::Offset); + + Node* addp2 = n2->as_AddP(); + Node* base2 = addp2->in(AddPNode::Base); + Node* addr2 = addp2->in(AddPNode::Address); + Node* offs2 = addp2->in(AddPNode::Offset); + + if (base1 == addr1 && base2 == addr2) { + + addr1 = ShenandoahBarrierNode::skip_through_barrier(addr1); + addr2 = ShenandoahBarrierNode::skip_through_barrier(addr2); + + if (addr1 == addr2 && offs1 == offs2) return true; + } + + } + return false; + } else { + return n1 == n2; + } +} //============================================================================= //------------------------------PhaseValues------------------------------------ @@ -1190,6 +1223,9 @@ NOT_PRODUCT(set_progress();) Node* con = makecon(t); // Make a constant add_users_to_worklist(k); + if (k->Opcode() == Op_ShenandoahWriteBarrier) { + assert(con->is_top(), "can only replace barrier with top"); + } subsume_node(k, con); // Everybody using k now uses con return con; }
--- a/src/share/vm/opto/phaseX.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/phaseX.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -190,6 +190,8 @@ // _nodes is used in varying ways by subclasses, which define local accessors public: + virtual PhaseIterGVN *is_IterGVN() { return 0; } + // Get a previously recorded type for the node n. // This type must already have been recorded. // If you want the type of a very new (untransformed) node, @@ -257,7 +259,7 @@ // Return whether two Nodes are equivalent. // Must not be recursive, since the recursive version is built from this. // For pessimistic optimizations this is simply pointer equivalence. - bool eqv(const Node* n1, const Node* n2) const { return n1 == n2; } + bool eqv(const Node* n1, const Node* n2) const; // For pessimistic passes, the return type must monotonically narrow. // For optimistic passes, the return type must monotonically widen.
--- a/src/share/vm/opto/runtime.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/runtime.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -34,6 +34,7 @@ #include "code/vtableStubs.hpp" #include "compiler/compileBroker.hpp" #include "compiler/oopMap.hpp" +#include "gc/shenandoah/shenandoahBarrierSet.hpp" #include "gc/g1/g1SATBCardTableModRefBS.hpp" #include "gc/g1/heapRegion.hpp" #include "gc/shared/barrierSet.hpp" @@ -91,6 +92,7 @@ address OptoRuntime::_multianewarray4_Java = NULL; address OptoRuntime::_multianewarray5_Java = NULL; address OptoRuntime::_multianewarrayN_Java = NULL; +address OptoRuntime::_shenandoah_write_barrier_Java = NULL; address OptoRuntime::_g1_wb_pre_Java = NULL; address OptoRuntime::_g1_wb_post_Java = NULL; address OptoRuntime::_vtable_must_compile_Java = NULL; @@ -142,6 +144,7 @@ gen(env, _multianewarray4_Java , multianewarray4_Type , multianewarray4_C , 0 , true , false, false); gen(env, _multianewarray5_Java , multianewarray5_Type , multianewarray5_C , 0 , true , false, false); gen(env, _multianewarrayN_Java , multianewarrayN_Type , multianewarrayN_C , 0 , true , false, false); + gen(env, _shenandoah_write_barrier_Java , shenandoah_write_barrier_Type, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_c2, 0, false, false, false); gen(env, _g1_wb_pre_Java , g1_wb_pre_Type , SharedRuntime::g1_wb_pre , 0 , false, false, false); gen(env, _g1_wb_post_Java , g1_wb_post_Type , SharedRuntime::g1_wb_post , 0 , false, false, false); gen(env, _complete_monitor_locking_Java , complete_monitor_enter_Type , SharedRuntime::complete_monitor_locking_C, 0, false, false, false); @@ -591,6 +594,33 @@ return TypeFunc::make(domain, range); } +const TypeFunc *OptoRuntime::shenandoah_clone_barrier_Type() { + const Type **fields = TypeTuple::fields(1); + fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value + const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields); + + // create result type (range) + fields = TypeTuple::fields(0); + const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields); + + return TypeFunc::make(domain, range); +} + +const TypeFunc *OptoRuntime::shenandoah_cas_obj_Type() { + const Type **fields = TypeTuple::fields(3); + fields[TypeFunc::Parms+0] = TypeRawPtr::NOTNULL; // Address + fields[TypeFunc::Parms+1] = TypeInstPtr::BOTTOM; // New value + fields[TypeFunc::Parms+2] = TypeInstPtr::BOTTOM; // Expected value + const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+3, fields); + + // create result type (range) + fields = TypeTuple::fields(1); + fields[TypeFunc::Parms+0] = TypeInt::BOOL; + const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields); + + return TypeFunc::make(domain, range); +} + const TypeFunc *OptoRuntime::uncommon_trap_Type() { // create input type (domain) const Type **fields = TypeTuple::fields(1); @@ -1520,6 +1550,37 @@ return TypeFunc::make(domain,range); } +/** + * oop resolve_and_maybe_copy_oop_static(oop obj) + * oop resolve_oop_static(oop obj) + */ +const TypeFunc* OptoRuntime::shenandoah_barrier_Type(const Type* type) { + // create input type (domain) + const Type** fields = TypeTuple::fields(1); + fields[TypeFunc::Parms+0] = type; + + const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+1, fields); + + // result type needed + fields = TypeTuple::fields(1); + fields[TypeFunc::Parms+0] = type; + const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields); + return TypeFunc::make(domain, range); +} + +const TypeFunc* OptoRuntime::shenandoah_write_barrier_Type() { + // create input type (domain) + const Type** fields = TypeTuple::fields(1); + fields[TypeFunc::Parms+0] = TypeOopPtr::NOTNULL; + + const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+1, fields); + + // result type needed + fields = TypeTuple::fields(1); + fields[TypeFunc::Parms+0] = TypeOopPtr::NOTNULL; + const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields); + return TypeFunc::make(domain, range); +} JRT_ENTRY_NO_ASYNC(void, OptoRuntime::register_finalizer(oopDesc* obj, JavaThread* thread)) assert(obj->is_oop(), "must be a valid oop");
--- a/src/share/vm/opto/runtime.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/runtime.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -141,6 +141,7 @@ static address _multianewarray4_Java; static address _multianewarray5_Java; static address _multianewarrayN_Java; + static address _shenandoah_write_barrier_Java; static address _g1_wb_pre_Java; static address _g1_wb_post_Java; static address _vtable_must_compile_Java; @@ -246,6 +247,7 @@ static address multianewarray4_Java() { return _multianewarray4_Java; } static address multianewarray5_Java() { return _multianewarray5_Java; } static address multianewarrayN_Java() { return _multianewarrayN_Java; } + static address shenandoah_write_barrier_Java() { return _shenandoah_write_barrier_Java; } static address g1_wb_pre_Java() { return _g1_wb_pre_Java; } static address g1_wb_post_Java() { return _g1_wb_post_Java; } static address vtable_must_compile_stub() { return _vtable_must_compile_Java; } @@ -290,6 +292,8 @@ static const TypeFunc* multianewarrayN_Type(); // multianewarray static const TypeFunc* g1_wb_pre_Type(); static const TypeFunc* g1_wb_post_Type(); + static const TypeFunc* shenandoah_clone_barrier_Type(); + static const TypeFunc* shenandoah_cas_obj_Type(); static const TypeFunc* complete_monitor_enter_Type(); static const TypeFunc* complete_monitor_exit_Type(); static const TypeFunc* monitor_notify_Type(); @@ -353,6 +357,10 @@ static const TypeFunc* dtrace_method_entry_exit_Type(); static const TypeFunc* dtrace_object_alloc_Type(); + // Shenandoah support + static const TypeFunc* shenandoah_barrier_Type(const Type* type); + static const TypeFunc* shenandoah_write_barrier_Type(); + # ifdef ENABLE_ZAP_DEAD_LOCALS static const TypeFunc* zap_dead_locals_Type(); # endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/opto/shenandoahSupport.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,520 @@ + +#include "opto/callnode.hpp" +#include "opto/movenode.hpp" +#include "opto/phaseX.hpp" +#include "opto/shenandoahSupport.hpp" + +Node* ShenandoahBarrierNode::skip_through_barrier(Node* n) { + if (n->is_ShenandoahBarrier()) { + return n->in(ValueIn); + } else { + return n; + } +} + +bool ShenandoahBarrierNode::needs_barrier(PhaseTransform* phase, Node* orig, Node* n, Node* rb_mem) { + Unique_Node_List visited; + return needs_barrier_impl(phase, orig, n, rb_mem, visited); +} + +bool ShenandoahBarrierNode::needs_barrier_impl(PhaseTransform* phase, Node* orig, Node* n, Node* rb_mem, Unique_Node_List &visited) { + + if (visited.member(n)) { + return false; // Been there. + } + visited.push(n); + + if (n->is_Allocate()) { + // tty->print_cr("killed barrier for newly allocated object"); + return false; + } + + if (n->is_CallJava()) { + return true; + } + + const Type* type = phase->type(n); + if (type->higher_equal(TypePtr::NULL_PTR)) { + // tty->print_cr("killed barrier for NULL object"); + return false; + } + if (type->isa_oopptr() && type->is_oopptr()->const_oop() != NULL) { + // tty->print_cr("killed barrier for constant object"); + return false; + } + + if (n->is_CheckCastPP() || n->is_ConstraintCast()) { + return needs_barrier_impl(phase, orig, n->in(1), rb_mem, visited); + } + if (n->is_Parm()) { + return true; + } + if (n->is_Proj()) { + return needs_barrier_impl(phase, orig, n->in(0), rb_mem, visited); + } + if (n->is_Phi()) { + bool need_barrier = false; + for (uint i = 1; i < n->req() && ! need_barrier; i++) { + Node* input = n->in(i); + if (input == NULL) { + need_barrier = true; // Phi not complete yet? + } else if (needs_barrier_impl(phase, orig, input, rb_mem, visited)) { + need_barrier = true; + } + } + return need_barrier; + } + if (n->is_CMove()) { + return needs_barrier_impl(phase, orig, n->in(CMoveNode::IfFalse), rb_mem, visited) || + needs_barrier_impl(phase, orig, n->in(CMoveNode::IfTrue ), rb_mem, visited); + } + if (n->Opcode() == Op_CreateEx) { + return true; + } + if (n->Opcode() == Op_ShenandoahWriteBarrier) { + // tty->print_cr("skipped barrier for chained write barrier object"); + return false; + } + if (n->Opcode() == Op_ShenandoahReadBarrier) { + if (rb_mem == n->in(Memory)) { + // tty->print_cr("Eliminated chained read barrier"); + return false; + } else { + return true; + } + } + + if (n->Opcode() == Op_LoadP) { + return true; + } +#ifdef ASSERT + tty->print("need barrier on?: "); n->dump(); +#endif + return true; +} + +bool ShenandoahReadBarrierNode::is_independent(const Type* in_type, const Type* this_type) const { + assert(in_type->isa_oopptr(), "expect oop ptr"); + assert(this_type->isa_oopptr(), "expect oop ptr"); + /* + if ((! in_type->isa_oopptr()) || (! this_type->isa_oopptr())) { +#ifdef ASSERT + tty->print_cr("not oopptr"); + tty->print("in: "); in_type->dump(); tty->print_cr(" "); + tty->print("this: "); this_type->dump(); tty->print_cr(" "); +#endif + return false; + } + */ + + ciKlass* in_kls = in_type->is_oopptr()->klass(); + ciKlass* this_kls = this_type->is_oopptr()->klass(); + if ((!in_kls->is_subclass_of(this_kls)) && + (!this_kls->is_subclass_of(in_kls))) { +#ifdef ASSERT + // tty->print_cr("independent: "); + // tty->print("in: "); in_kls->print(); tty->print_cr(" "); + // tty->print("this: "); this_kls->print(); tty->print_cr(" "); +#endif + return true; + } +#ifdef ASSERT + // tty->print_cr("possibly dependend?"); + // tty->print("in: "); in_type->dump(); tty->print_cr(" "); + // tty->print("this: "); this_type->dump(); tty->print_cr(" "); +#endif + return false; +} + +Node* ShenandoahReadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { + + if (! can_reshape) { + return NULL; + } + + // if (true) return NULL; + + if (in(Memory) == phase->C->immutable_memory()) return NULL; + + // If memory input is a MergeMem, take the appropriate slice out of it. + Node* mem_in = in(Memory); + if (mem_in->isa_MergeMem()) { + const TypePtr* adr_type = bottom_type()->is_ptr()->add_offset(-8); + uint alias_idx = phase->C->get_alias_index(adr_type); + mem_in = mem_in->as_MergeMem()->memory_at(alias_idx); + set_req(Memory, mem_in); + return this; + } + + Node* input = in(Memory); + if (input->Opcode() == Op_ShenandoahWBMemProj) { + input = input->in(0); + if (input->is_top()) return NULL; // Dead path. + assert(input->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier"); + const Type* in_type = phase->type(input); + const Type* this_type = phase->type(this); + if (is_independent(in_type, this_type)) { + phase->igvn_rehash_node_delayed(input); + set_req(Memory, input->in(Memory)); + return this; + } + } + return NULL; +} + +bool ShenandoahBarrierNode::has_barrier_users(Node* n, Unique_Node_List &visited) { + if (visited.member(n)) { + return false; + } + visited.push(n); + + bool has_users = false; + for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax && ! has_users; j++) { + Node* o = n->fast_out(j); + if (o->Opcode() == Op_ShenandoahReadBarrier || + o->Opcode() == Op_ShenandoahWriteBarrier) { + // tty->print("counting barrier"); o->dump(); + has_users = true; + } else if (o->isa_Phi()) { + has_users = has_barrier_users(o, visited); + } else if (o->Opcode() == Op_MergeMem) { + // Not a user. ? + } else { + // tty->print_cr("unknown user"); o->dump(); + ShouldNotReachHere(); + } + } + return has_users; +} + +Node* ShenandoahWriteBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { + + if (! can_reshape) return NULL; + + if (in(Memory) == phase->C->immutable_memory()) return NULL; + + Node* mem_in = in(Memory); + if (mem_in->isa_MergeMem()) { + const TypePtr* adr_type = bottom_type()->is_ptr()->add_offset(-8); + uint alias_idx = phase->C->get_alias_index(adr_type); + mem_in = mem_in->as_MergeMem()->memory_at(alias_idx); + set_req(Memory, mem_in); + return this; + } + + Node* mem_proj = find_out_with(Op_ShenandoahWBMemProj); + if (mem_proj == NULL) { + // tty->print_cr("no mem proj: kill input mem"); dump(); + set_req(Memory, phase->C->immutable_memory()); + return this; + } + + if (true) return NULL; + + Unique_Node_List visited; + if (! has_barrier_users(mem_proj, visited)) { + // tty->print_cr("no users of mem projection. kill input mem"); dump(); + phase->igvn_rehash_node_delayed(in(Memory)); + set_req(Memory, phase->C->immutable_memory()); + + // tty->print_cr("reshaped wb: "); + // dump(); + return this; + } + // tty->print_cr("leave mem proj alone"); + return NULL; +} +/* +bool ShenandoahBarrierNode::dominates_control_impl(PhaseTransform* phase, + Node* c1, + Node* c2, + Node* current, + Unique_Node_List & visited) { + if (current == NULL) { + return true; + } else if (visited.member(current)) { + return true; + } + visited.push(current); + + current->dump(); + ShouldNotReachHere(); +} +*/ + +bool ShenandoahBarrierNode::dominates_control(PhaseTransform* phase, + Node* c1, + Node* c2) { + if (c1 == c2) { + return true; + } + if (c1 == NULL) { + return true; + } +#ifdef ASSERT + tty->print("c1: "); c1->dump(2); + tty->print("c2: "); c2->dump(2); +#endif + ShouldNotReachHere(); + return false; +} + +bool ShenandoahBarrierNode::dominates_memory_impl(PhaseTransform* phase, + Node* b1, + Node* b2, + Node* current, + Unique_Node_List &visited) { + /* + tty->print_cr("phistack:"); + for (int x = 0; x < phistack->length(); x++) { + tty->print("-->"); + phistack->at(x)->dump(); + } + */ + if (current == NULL) { + return false; + } else if (visited.member(current)) { + // We have already seen it. + return true; + } + + visited.push(current); + + if (current == b1) { + // tty->print_cr("current == b1: "); current->dump(); + return true; + } else if (current == b2) { + // tty->print_cr("current == b2: "); current->dump(); + return false; + } else if (current == phase->C->immutable_memory()) { + // tty->print_cr("current == immutable_memory: "); current->dump(); + return false; + } else if (current->isa_Phi()) { + // tty->print_cr("current == phi: "); current->dump(); + bool dominates = true; + for (uint i = 1; i < current->req() && dominates == true; i++) { + Node* in = current->in(i); + dominates = dominates && dominates_memory_impl(phase, b1, b2, in, visited); + } + return dominates; + } else if (current->Opcode() == Op_ShenandoahWriteBarrier) { + // tty->print_cr("current == wb: "); current->dump(); + // Follow through memory input. + Node* in = current->in(Memory); + return dominates_memory_impl(phase, b1, b2, in, visited); + } else if (current->Opcode() == Op_ShenandoahWBMemProj) { + // tty->print_cr("current == wb-memproj: "); current->dump(); + // Follow through memory input. + Node* in = current->in(0); + return dominates_memory_impl(phase, b1, b2, in, visited); + } else if (current->is_top()) { + return false; // Dead path + } else if (current->is_Proj()) { + // tty->print_cr("current == proj: "); current->dump(); + return dominates_memory_impl(phase, b1, b2, current->in(0), visited); + } else if (current->is_Call()) { + // tty->print_cr("current == call: "); current->dump(); + return dominates_memory_impl(phase, b1, b2, current->in(TypeFunc::Memory), visited); + } else if (current->is_MemBar()) { + // tty->print_cr("current == membar: "); current->dump(); + return dominates_memory_impl(phase, b1, b2, current->in(TypeFunc::Memory), visited); + } else if (current->is_MergeMem()) { + // tty->print_cr("current == mergemem: "); current->dump(); + const TypePtr* adr_type = phase->type(b2)->is_ptr()->add_offset(-8); + uint alias_idx = phase->C->get_alias_index(adr_type); + Node* mem_in = current->as_MergeMem()->memory_at(alias_idx); + return dominates_memory_impl(phase, b1, b2, current->in(TypeFunc::Memory), visited); + } else { + // tty->print_cr("what else can we see here:"); +#ifdef ASSERT + current->dump(); +#endif + ShouldNotReachHere(); + return false; + } +} + +/** + * Determines if b1 dominates b2 through memory inputs. It returns true if: + * - b1 can be reached by following each branch in b2's memory input (through phis, etc) + * - or we get back to b2 (i.e. through a loop) without seeing b1 + * In all other cases, (in particular, if we reach immutable_memory without having seen b1) + * we return false. + */ +bool ShenandoahBarrierNode::dominates_memory(PhaseTransform* phase, Node* b1, Node* b2) { + Unique_Node_List visited; + return dominates_memory_impl(phase, b1->in(Memory), b2, b2->in(Memory), visited); +} + +Node* ShenandoahBarrierNode::Identity_impl(PhaseTransform* phase) { + Node* n = in(ValueIn); + + Node* rb_mem = Opcode() == Op_ShenandoahReadBarrier ? in(Memory) : NULL; + if (! needs_barrier(phase, this, n, rb_mem)) { + return n; + } + + // tty->print_cr("find sibling for: "); dump(2); + // Try to find a write barrier sibling with identical inputs that we can fold into. + for (DUIterator i = n->outs(); n->has_out(i); i++) { + Node* sibling = n->out(i); + if (sibling == this) { + continue; + } + /* + assert(sibling->Opcode() != Op_ShenandoahWriteBarrier || + Opcode() != Op_ShenandoahWriteBarrier || hash() == sibling->hash(), + "if this is a write barrier, then sibling can't be write barrier too"); + */ + if (sibling->Opcode() != Op_ShenandoahWriteBarrier) { + continue; + } + /* + if (sibling->outcnt() == 0) { + // Some dead node. + continue; + } + */ + assert(sibling->in(ValueIn) == in(ValueIn), "sanity"); + assert(sibling->Opcode() == Op_ShenandoahWriteBarrier, "sanity"); + // tty->print_cr("candidate: "); sibling->dump(); + + if (dominates_control(phase, sibling->in(Control), in(Control)) && + dominates_memory(phase, sibling, this)) { + + /* + tty->print_cr("matched barrier:"); + sibling->dump(); + tty->print_cr("for: "); + dump(); + */ + return sibling; + } + + /* + tty->print_cr("couldn't match candidate:"); + sibling->dump(2); + */ + } + /* + tty->print_cr("couldn't match barrier to any:"); + dump(); + */ + return this; +} + +Node* ShenandoahBarrierNode::Identity(PhaseTransform* phase) { + + Node* replacement = Identity_impl(phase); + if (replacement != this) { + // If we have a memory projection, we first need to make it go away. + Node* mem_proj = find_out_with(Op_ShenandoahWBMemProj); + if (mem_proj != NULL) { + phase->igvn_rehash_node_delayed(mem_proj); + return this; + } + } + return replacement; +} + +Node* ShenandoahReadBarrierNode::Identity(PhaseTransform* phase) { + + // if (true) return this; + + // tty->print("optimizing rb: "); dump(); + Node* id = ShenandoahBarrierNode::Identity(phase); + + if (true) return id; + + if (id == this) { + Node* n = in(ValueIn); + // No success in super call. Try to combine identical read barriers. + for (DUIterator i = n->outs(); n->has_out(i); i++) { + Node* sibling = n->out(i); + if (sibling == this) { + continue; + } + if (sibling->outcnt() == 0) { + continue; + } + if (sibling->Opcode() == Op_ShenandoahReadBarrier && + sibling->in(Control) == in(Control) && + sibling->in(Memory) == in(Memory) && + sibling->in(ValueIn) == in(ValueIn)) { + // assert(sibling->hash() == hash(), "ident node needs same hash"); + // assert(cmp(sibling) == 1, "ident node needs equals"); + //if (sibling->_hash_lock == 0) { + // tty->print_cr("skipped sibling not in hash: "); sibling->dump(); + //} else { + return sibling; + //} + } + } + } + return id; +} + +const Type* ShenandoahBarrierNode::Value(PhaseTransform* phase) const { + // Either input is TOP ==> the result is TOP + const Type *t1 = phase->type(in(Memory)); + if (t1 == Type::TOP) return Type::TOP; + const Type *t2 = phase->type(in(ValueIn)); + if( t2 == Type::TOP ) return Type::TOP; + + Node* input = in(ValueIn); + const Type* type = phase->type(input); + return type; +} + +#ifdef ASSERT +uint ShenandoahBarrierNode::num_mem_projs() { + uint num_mem_proj = 0; + for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) { + Node* use = fast_out(i); + if (use->Opcode() == Op_ShenandoahWBMemProj) { + num_mem_proj++; + } + } + return num_mem_proj; +} + +void ShenandoahBarrierNode::check_invariants() { + // if (Opcode() == Op_ShenandoahWriteBarrier) { + // uint mem_projs = num_mem_projs(); + // if (mem_projs > 1) { + // tty->print_cr("num_mem_proj: %u", mem_projs); + // dump(2); + // } + // assert(mem_projs <= 1, "exactly 1 memory projection"); + // } +} +#endif + +Node* ShenandoahWBMemProjNode::Identity(PhaseTransform* phase) { + + Node* wb = in(0); + if (wb->is_top()) return phase->C->top(); // Dead path. + + assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier"); + if (wb->as_ShenandoahBarrier()->Identity_impl(phase) != wb) { + // If the parent write barrier would go away, make this mem proj go away first. + // Poke parent to give it a chance to go away too. + phase->igvn_rehash_node_delayed(wb); + return wb->in(ShenandoahBarrierNode::Memory); + } + + // We can't do the below unless the graph is fully constructed. + if (! phase->is_IterGVN()) { + return this; + } + + // If the mem projection has no barrier users, it's not needed anymore. + Unique_Node_List visited; + if (! ShenandoahWriteBarrierNode::has_barrier_users(this, visited)) { + // tty->print_cr("mem proj has no users. kill"); dump(); + phase->igvn_rehash_node_delayed(wb); + return wb->in(ShenandoahBarrierNode::Memory); + } + + return this; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/opto/shenandoahSupport.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,134 @@ +#ifndef SHARE_VM_OPTO_SHENANDOAH_SUPPORT_HPP +#define SHARE_VM_OPTO_SHENANDOAH_SUPPORT_HPP + +#include "memory/allocation.hpp" +#include "opto/addnode.hpp" +#include "opto/memnode.hpp" +#include "opto/multnode.hpp" +#include "opto/node.hpp" + +class PhaseTransform; + + +class ShenandoahBarrierNode : public TypeNode { +public: + +public: + enum { Control, + Memory, + ValueIn + }; + + ShenandoahBarrierNode(Node* ctrl, Node* mem, Node* obj) + : TypeNode(obj->bottom_type(), 3) { + + init_req(Control, ctrl); + init_req(Memory, mem); + init_req(ValueIn, obj); + + init_class_id(Class_ShenandoahBarrier); + } + + static Node* skip_through_barrier(Node* n); + + virtual const class TypePtr* adr_type() const { + //const TypePtr* adr_type = in(MemNode::Address)->bottom_type()->is_ptr(); + const TypePtr* adr_type = bottom_type()->is_ptr()->add_offset(-8); + assert(adr_type->offset() == -8, "sane offset"); + assert(Compile::current()->alias_type(adr_type)->is_rewritable(), "brooks ptr must be rewritable"); + return adr_type; + } + + virtual uint ideal_reg() const { return Op_RegP; } + virtual uint match_edge(uint idx) const { + return idx >= ValueIn; + } + virtual uint size_of() const { return sizeof(*this); } + + virtual Node* Identity(PhaseTransform* phase); + Node* Identity_impl(PhaseTransform* phase); + + virtual const Type* Value(PhaseTransform* phase) const; + virtual bool depends_only_on_test() const { + return true; + }; +#ifdef ASSERT + void check_invariants(); + uint num_mem_projs(); +#endif + + static bool needs_barrier(PhaseTransform* phase, Node* orig, Node* n, Node* rb_mem); + + static bool has_barrier_users(Node* n, Unique_Node_List &visited); + +private: + static bool needs_barrier_impl(PhaseTransform* phase, Node* orig, Node* n, Node* rb_mem, Unique_Node_List &visited); + + + bool dominates_control(PhaseTransform* phase, Node* c1, Node* c2); + bool dominates_memory(PhaseTransform* phase, Node* b1, Node* b2); + bool dominates_memory_impl(PhaseTransform* phase, Node* b1, Node* b2, Node* current, Unique_Node_List &visisted); +}; + +class ShenandoahReadBarrierNode : public ShenandoahBarrierNode { +public: + ShenandoahReadBarrierNode(Node* ctrl, Node* mem, Node* obj) + : ShenandoahBarrierNode(ctrl, mem, obj) { + } + + virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); + virtual Node* Identity(PhaseTransform* phase); + virtual int Opcode() const; + +private: + bool is_independent(const Type* in_type, const Type* this_type) const; + +}; + +class ShenandoahWriteBarrierNode : public ShenandoahBarrierNode { +public: + ShenandoahWriteBarrierNode(Node* ctrl, Node* mem, Node* obj) + : ShenandoahBarrierNode(ctrl, mem, obj) { + //tty->print("new wb: "); dump(); + } + + virtual int Opcode() const; + virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); + + // virtual void set_req( uint i, Node *n ) { + // if (i == MemNode::Memory) { assert(n == Compiler::current()->immutable_memory(), "set only immutable mem on wb"); } + // Node::set_req(i, n); + // } +}; + +class ShenandoahWBMemProjNode : public ProjNode { +public: + enum {SWBMEMPROJCON = (uint)-3}; + ShenandoahWBMemProjNode(Node *src) : ProjNode( src, SWBMEMPROJCON) { + assert(src->Opcode() == Op_ShenandoahWriteBarrier || src->is_Mach(), "epxect wb"); +#ifdef ASSERT + in(0)->as_ShenandoahBarrier()->check_invariants(); +#endif + } + virtual Node* Identity(PhaseTransform* phase); + + virtual int Opcode() const; + virtual bool is_CFG() const { return false; } + virtual const Type *bottom_type() const {return Type::MEMORY;} + virtual const TypePtr *adr_type() const { + Node* ctrl = in(0); + if (ctrl == NULL) return NULL; // node is dead + assert(ctrl->Opcode() == Op_ShenandoahWriteBarrier || ctrl->is_Mach(), "expect wb"); + return ctrl->adr_type(); + } + + virtual uint ideal_reg() const { return 0;} // memory projections don't have a register + virtual const Type *Value( PhaseTransform *phase ) const { + return bottom_type(); + } +#ifndef PRODUCT + virtual void dump_spec(outputStream *st) const {}; +#endif +}; + +#endif // SHARE_VM_OPTO_SHENANDOAH_SUPPORT_HPP
--- a/src/share/vm/opto/split_if.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/split_if.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -32,6 +32,7 @@ //------------------------------split_thru_region------------------------------ // Split Node 'n' through merge point. Node *PhaseIdealLoop::split_thru_region( Node *n, Node *region ) { + assert(n->Opcode() != Op_ShenandoahWriteBarrier, "not with write barriers"); uint wins = 0; assert( n->is_CFG(), "" ); assert( region->is_Region(), "" ); @@ -73,7 +74,8 @@ if( n->is_Phi() ) return false; // Local PHIs are expected // Recursively split-up inputs - for (uint i = 1; i < n->req(); i++) { + uint first_input = n->Opcode() == Op_ShenandoahWBMemProj ? 0 : 1; + for (uint i = first_input; i < n->req(); i++) { if( split_up( n->in(i), blk1, blk2 ) ) { // Got split recursively and self went dead? if (n->outcnt() == 0) @@ -216,6 +218,7 @@ register_new_node(phi, blk1); // Remove cloned-up value from optimizer; use phi instead + split_mem_thru_phi(n, blk1, phi); _igvn.replace_node( n, phi ); // (There used to be a self-recursive call to split_up() here,
--- a/src/share/vm/opto/stringopts.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/stringopts.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -1603,6 +1603,7 @@ char_alloc->maybe_set_complete(_gvn); // Now copy the string representations into the final char[] + char_array = __ shenandoah_write_barrier(char_array); Node* start = __ intcon(0); for (int argi = 0; argi < sc->num_arguments(); argi++) { Node* arg = sc->argument(argi);
--- a/src/share/vm/opto/superword.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/superword.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -2074,10 +2074,11 @@ Node *n = lpt()->_body.at(i); set_bb_idx(n, i); // Create a temporary map if (in_bb(n)) { - if (n->is_LoadStore() || n->is_MergeMem() || + if (n->is_LoadStore() || n->is_MergeMem() || n->is_ShenandoahBarrier() || (n->is_Proj() && !n->as_Proj()->is_CFG())) { // Bailout if the loop has LoadStore, MergeMem or data Proj // nodes. Superword optimization does not work with them. + // Also disable with Shenandoah barriers for now. return false; } bb_ct++;
--- a/src/share/vm/opto/type.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/opto/type.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -2832,7 +2832,7 @@ if (klass() == ciEnv::current()->Class_klass() && (_offset == java_lang_Class::klass_offset_in_bytes() || - _offset == java_lang_Class::array_klass_offset_in_bytes())) { + _offset == java_lang_Class::array_klass_offset_in_bytes()) || UseShenandoahGC) { // Special hidden fields from the Class. assert(this->isa_instptr(), "must be an instance ptr."); _is_ptr_to_narrowoop = false;
--- a/src/share/vm/prims/jni.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/prims/jni.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -575,6 +575,10 @@ oop sub_mirror = JNIHandles::resolve_non_null(sub); oop super_mirror = JNIHandles::resolve_non_null(super); + + sub_mirror = oopDesc::bs()->resolve_and_maybe_copy_oop(sub_mirror); + super_mirror = oopDesc::bs()->resolve_and_maybe_copy_oop(super_mirror); + if (java_lang_Class::is_primitive(sub_mirror) || java_lang_Class::is_primitive(super_mirror)) { jboolean ret = (sub_mirror == super_mirror); @@ -817,6 +821,7 @@ HOTSPOT_JNI_ISSAMEOBJECT_ENTRY(env, r1, r2); oop a = JNIHandles::resolve(r1); + a = oopDesc::bs()->resolve_and_maybe_copy_oop(a); oop b = JNIHandles::resolve(r2); jboolean ret = (a == b) ? JNI_TRUE : JNI_FALSE; @@ -2062,7 +2067,7 @@ // If G1 is enabled and we are accessing the value of the referent // field in a reference object then we need to register a non-null // referent with the SATB barrier. - if (UseG1GC) { + if (UseG1GC || UseShenandoahGC) { bool needs_barrier = false; if (ret != NULL && @@ -2165,6 +2170,7 @@ JNIWrapper("SetObjectField"); HOTSPOT_JNI_SETOBJECTFIELD_ENTRY(env, obj, (uintptr_t) fieldID, value); oop o = JNIHandles::resolve_non_null(obj); + o = oopDesc::bs()->resolve_and_maybe_copy_oop(o); Klass* k = o->klass(); int offset = jfieldIDWorkaround::from_instance_jfieldID(k, fieldID); // Keep JVMTI addition small and only check enabled flag here. @@ -2189,6 +2195,7 @@ EntryProbe; \ \ oop o = JNIHandles::resolve_non_null(obj); \ + o = oopDesc::bs()->resolve_and_maybe_copy_oop(o); \ Klass* k = o->klass(); \ int offset = jfieldIDWorkaround::from_instance_jfieldID(k, fieldID); \ /* Keep JVMTI addition small and only check enabled flag here. */ \ @@ -2398,7 +2405,9 @@ field_value.unionType = value; \ JvmtiExport::jni_SetField_probe(thread, NULL, NULL, id->holder(), fieldID, true, SigType, (jvalue *)&field_value); \ } \ - id->holder()->java_mirror()-> Fieldname##_field_put (id->offset(), value); \ + oop o = id->holder()->java_mirror(); \ + o = oopDesc::bs()->resolve_and_maybe_copy_oop(o); \ + o-> Fieldname##_field_put (id->offset(), value); \ ReturnProbe;\ JNI_END @@ -2629,6 +2638,7 @@ DT_VOID_RETURN_MARK(SetObjectArrayElement); objArrayOop a = objArrayOop(JNIHandles::resolve_non_null(array)); + a = objArrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(a)); oop v = JNIHandles::resolve(value); if (a->is_within_bounds(index)) { if (v == NULL || v->is_a(ObjArrayKlass::cast(a->klass())->element_klass())) { @@ -2773,6 +2783,7 @@ JNIWrapper("Release" XSTR(Result) "ArrayElements"); \ EntryProbe; \ typeArrayOop a = typeArrayOop(JNIHandles::resolve_non_null(array)); \ + a = typeArrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(a)); \ int len = a->length(); \ if (len != 0) { /* Empty array: nothing to free or copy. */ \ if ((mode == 0) || (mode == JNI_COMMIT)) { \ @@ -2873,6 +2884,7 @@ EntryProbe; \ DT_VOID_RETURN_MARK(Set##Result##ArrayRegion); \ typeArrayOop dst = typeArrayOop(JNIHandles::resolve_non_null(array)); \ + dst = typeArrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(dst)); \ if (start < 0 || len < 0 || ((unsigned int)start + (unsigned int)len > (unsigned int)dst->length())) { \ THROW(vmSymbols::java_lang_ArrayIndexOutOfBoundsException()); \ } else { \ @@ -3156,6 +3168,7 @@ *isCopy = JNI_FALSE; } oop a = JNIHandles::resolve_non_null(array); + a = oopDesc::bs()->resolve_and_maybe_copy_oop(a); assert(a->is_array(), "just checking"); BasicType type; if (a->is_objArray()) {
--- a/src/share/vm/prims/jvm.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/prims/jvm.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -357,7 +357,10 @@ assert(s->is_oop(), "JVM_ArrayCopy: src not an oop"); assert(d->is_oop(), "JVM_ArrayCopy: dst not an oop"); // Do copy - s->klass()->copy_array(s, src_pos, d, dst_pos, length, thread); + s->klass()->copy_array(s, src_pos, + (arrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(d))), + dst_pos, + length, thread); JVM_END @@ -634,6 +637,9 @@ (size_t)align_object_size(size) / HeapWordsPerLong); // Clear the header new_obj_oop->init_mark(); + // TODO: Find a nicer way to hook up Shenandoah's special handling of + // age bits. + Universe::heap()->post_allocation_collector_specific_setup((HeapWord*) new_obj_oop); // Store check (mark entire object and let gc sort it out) BarrierSet* bs = Universe::heap()->barrier_set(); @@ -3318,6 +3324,7 @@ JVM_ENTRY(void, JVM_SetArrayElement(JNIEnv *env, jobject arr, jint index, jobject val)) JVMWrapper("JVM_SetArrayElement"); arrayOop a = check_array(env, arr, false, CHECK); + a = arrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(a)); oop box = JNIHandles::resolve(val); jvalue value; value.i = 0; // to initialize value before getting used in CHECK @@ -3335,6 +3342,7 @@ JVM_ENTRY(void, JVM_SetPrimitiveArrayElement(JNIEnv *env, jobject arr, jint index, jvalue v, unsigned char vCode)) JVMWrapper("JVM_SetPrimitiveArrayElement"); arrayOop a = check_array(env, arr, true, CHECK); + a = arrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(a)); assert(a->is_typeArray(), "just checking"); BasicType value_type = (BasicType) vCode; Reflection::array_set(&v, a, index, value_type, CHECK);
--- a/src/share/vm/prims/unsafe.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/prims/unsafe.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -74,7 +74,6 @@ #define UnsafeWrapper(arg) /*nothing, for the present*/ - inline void* addr_from_java(jlong addr) { // This assert fails in a variety of ways on 32-bit systems. // It is impossible to predict whether native code that converts @@ -121,8 +120,13 @@ if (p != NULL) { assert(byte_offset >= 0 && byte_offset <= (jlong)MAX_OBJECT_SIZE, "sane offset"); if (byte_offset == (jint)byte_offset) { + // We need to preemptively evacuate the object here to make the comparison + // in the assert below not give false negatives in case the object + // gets moved by concurrent threads while executing this code. + p = oopDesc::bs()->resolve_and_maybe_copy_oop(p); void* ptr_plus_disp = (address)p + byte_offset; - assert((void*)p->obj_field_addr<oop>((jint)byte_offset) == ptr_plus_disp, + void* obj_field_addr = (void*)p->obj_field_addr<oop>((jint)byte_offset); + assert(obj_field_addr == ptr_plus_disp, "raw [ptr+disp] must be consistent with oop::field_base"); } jlong p_size = HeapWordSize * (jlong)(p->size()); @@ -159,6 +163,7 @@ #define SET_FIELD(obj, offset, type_name, x) \ oop p = JNIHandles::resolve(obj); \ + p = oopDesc::bs()->resolve_and_maybe_copy_oop(p); \ *(type_name*)index_oop_from_field_offset_long(p, offset) = x #define GET_FIELD_VOLATILE(obj, offset, type_name, v) \ @@ -170,6 +175,7 @@ #define SET_FIELD_VOLATILE(obj, offset, type_name, x) \ oop p = JNIHandles::resolve(obj); \ + p = oopDesc::bs()->resolve_and_maybe_copy_oop(p); \ OrderAccess::release_store_fence((volatile type_name*)index_oop_from_field_offset_long(p, offset), x); @@ -193,7 +199,7 @@ // We could be accessing the referent field in a reference // object. If G1 is enabled then we need to register non-null // referent with the SATB barrier. - if (UseG1GC) { + if (UseG1GC || UseShenandoahGC) { bool needs_barrier = false; if (ret != NULL) { @@ -219,7 +225,7 @@ UNSAFE_ENTRY(void, Unsafe_SetObject(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jobject x_h)) UnsafeWrapper("Unsafe_SetObject"); oop x = JNIHandles::resolve(x_h); - oop p = JNIHandles::resolve(obj); + oop p = oopDesc::bs()->resolve_and_maybe_copy_oop(JNIHandles::resolve(obj)); if (UseCompressedOops) { oop_store((narrowOop*)index_oop_from_field_offset_long(p, offset), x); } else { @@ -246,6 +252,7 @@ UnsafeWrapper("Unsafe_SetObjectVolatile"); oop x = JNIHandles::resolve(x_h); oop p = JNIHandles::resolve(obj); + p = oopDesc::bs()->resolve_and_maybe_copy_oop(p); void* addr = index_oop_from_field_offset_long(p, offset); OrderAccess::release(); if (UseCompressedOops) { @@ -328,7 +335,7 @@ SET_FIELD_VOLATILE(obj, offset, jlong, x); } else { - Handle p (THREAD, JNIHandles::resolve(obj)); + Handle p (THREAD, oopDesc::bs()->resolve_and_maybe_copy_oop(JNIHandles::resolve(obj))); jlong* addr = (jlong*)(index_oop_from_field_offset_long(p(), offset)); MutexLockerEx mu(UnsafeJlong_lock, Mutex::_no_safepoint_check_flag); Atomic::store(x, addr); @@ -435,6 +442,7 @@ UnsafeWrapper("Unsafe_SetOrderedObject"); oop x = JNIHandles::resolve(x_h); oop p = JNIHandles::resolve(obj); + p = oopDesc::bs()->resolve_and_maybe_copy_oop(p); void* addr = index_oop_from_field_offset_long(p, offset); OrderAccess::release(); if (UseCompressedOops) { @@ -456,7 +464,7 @@ SET_FIELD_VOLATILE(obj, offset, jlong, x); } else { - Handle p (THREAD, JNIHandles::resolve(obj)); + Handle p (THREAD, oopDesc::bs()->resolve_and_maybe_copy_oop(JNIHandles::resolve(obj))); jlong* addr = (jlong*)(index_oop_from_field_offset_long(p(), offset)); MutexLockerEx mu(UnsafeJlong_lock, Mutex::_no_safepoint_check_flag); Atomic::store(x, addr); @@ -635,6 +643,7 @@ THROW(vmSymbols::java_lang_IllegalArgumentException()); } oop base = JNIHandles::resolve(obj); + base = oopDesc::bs()->resolve_and_maybe_copy_oop(base); void* p = index_oop_from_field_offset_long(base, offset); Copy::fill_to_memory_atomic(p, sz, value); UNSAFE_END @@ -650,6 +659,7 @@ } oop srcp = JNIHandles::resolve(srcObj); oop dstp = JNIHandles::resolve(dstObj); + dstp = oopDesc::bs()->resolve_and_maybe_copy_oop(dstp); if (dstp != NULL && !dstp->is_typeArray()) { // NYI: This works only for non-oop arrays at present. // Generalizing it would be reasonable, but requires card marking. @@ -1082,12 +1092,22 @@ UNSAFE_ENTRY(jboolean, Unsafe_CompareAndSwapObject(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jobject e_h, jobject x_h)) UnsafeWrapper("Unsafe_CompareAndSwapObject"); + // We are about to write to this entry so check to see if we need to copy it. + oop p = oopDesc::bs()->resolve_and_maybe_copy_oop(JNIHandles::resolve(obj)); + HeapWord* addr = (HeapWord *)index_oop_from_field_offset_long(p, offset); oop x = JNIHandles::resolve(x_h); - oop e = JNIHandles::resolve(e_h); - oop p = JNIHandles::resolve(obj); - HeapWord* addr = (HeapWord *)index_oop_from_field_offset_long(p, offset); - oop res = oopDesc::atomic_compare_exchange_oop(x, addr, e, true); - jboolean success = (res == e); + oop old = JNIHandles::resolve(e_h); + jboolean success; + if (UseShenandoahGC) { + oop expected; + do { + expected = old; + old = oopDesc::atomic_compare_exchange_oop(x, addr, expected, true); + success = (old == expected); + } while ((! success) && oopDesc::bs()->resolve_oop(old) == oopDesc::bs()->resolve_oop(expected)); + } else { + success = (old == oopDesc::atomic_compare_exchange_oop(x, addr, old, true)); + } if (success) update_barrier_set((void*)addr, x); return success; @@ -1095,14 +1115,15 @@ UNSAFE_ENTRY(jboolean, Unsafe_CompareAndSwapInt(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jint e, jint x)) UnsafeWrapper("Unsafe_CompareAndSwapInt"); - oop p = JNIHandles::resolve(obj); + // We are about to write to this entry so check to see if we need to copy it. + oop p = oopDesc::bs()->resolve_and_maybe_copy_oop(JNIHandles::resolve(obj)); jint* addr = (jint *) index_oop_from_field_offset_long(p, offset); return (jint)(Atomic::cmpxchg(x, addr, e)) == e; UNSAFE_END UNSAFE_ENTRY(jboolean, Unsafe_CompareAndSwapLong(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jlong e, jlong x)) UnsafeWrapper("Unsafe_CompareAndSwapLong"); - Handle p (THREAD, JNIHandles::resolve(obj)); + Handle p (THREAD, oopDesc::bs()->resolve_and_maybe_copy_oop(JNIHandles::resolve(obj))); jlong* addr = (jlong*)(index_oop_from_field_offset_long(p(), offset)); #ifdef SUPPORTS_NATIVE_CX8 return (jlong)(Atomic::cmpxchg(x, addr, e)) == e; @@ -1177,7 +1198,7 @@ double la[max_nelem]; jint ret; - typeArrayOop a = typeArrayOop(JNIHandles::resolve_non_null(loadavg)); + typeArrayOop a = typeArrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(JNIHandles::resolve_non_null(loadavg))); assert(a->is_typeArray(), "must be type array"); if (nelem < 0 || nelem > max_nelem || a->length() < nelem) {
--- a/src/share/vm/runtime/arguments.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/runtime/arguments.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -51,6 +51,7 @@ #include "utilities/macros.hpp" #include "utilities/stringUtils.hpp" #if INCLUDE_ALL_GCS +#include "gc/shenandoah/shenandoahHeap.hpp" #include "gc/cms/compactibleFreeListSpace.hpp" #include "gc/g1/g1CollectedHeap.inline.hpp" #include "gc/parallel/parallelScavengeHeap.hpp" @@ -1471,6 +1472,11 @@ // the only value that can override MaxHeapSize if we are // to use UseCompressedOops is InitialHeapSize. size_t max_heap_size = MAX2(MaxHeapSize, InitialHeapSize); + if (UseShenandoahGC && FLAG_IS_DEFAULT(UseCompressedOops)) { + warning("Compressed Oops not supported with ShenandoahGC"); + FLAG_SET_ERGO(bool, UseCompressedOops, false); + FLAG_SET_ERGO(bool, UseCompressedClassPointers, false); + } if (max_heap_size <= max_heap_for_compressed_oops()) { #if !defined(COMPILER1) || defined(TIERED) @@ -1529,6 +1535,10 @@ heap_alignment = ParallelScavengeHeap::conservative_max_heap_alignment(); } else if (UseG1GC) { heap_alignment = G1CollectedHeap::conservative_max_heap_alignment(); + } else if (UseShenandoahGC) { + // TODO: This sucks. Can't we have a clean interface to call the GC's collector + // policy for this? + heap_alignment = ShenandoahHeap::conservative_max_heap_alignment(); } #endif // INCLUDE_ALL_GCS _conservative_max_heap_alignment = MAX4(heap_alignment, @@ -1687,6 +1697,21 @@ } } +void Arguments::set_shenandoah_gc_flags() { + FLAG_SET_DEFAULT(UseDynamicNumberOfGCThreads, true); + FLAG_SET_DEFAULT(ParallelGCThreads, + Abstract_VM_Version::parallel_worker_threads()); + + if (FLAG_IS_DEFAULT(ConcGCThreads)) { + uint conc_threads = MAX2((uint) 1, ParallelGCThreads); + FLAG_SET_DEFAULT(ConcGCThreads, conc_threads); + } + + if (FLAG_IS_DEFAULT(ParallelRefProcEnabled)) { + FLAG_SET_DEFAULT(ParallelRefProcEnabled, true); + } +} + #if !INCLUDE_ALL_GCS #ifdef ASSERT static bool verify_serial_gc_flags() { @@ -1706,6 +1731,8 @@ set_cms_and_parnew_gc_flags(); } else if (UseG1GC) { set_g1_gc_flags(); + } else if (UseShenandoahGC) { + set_shenandoah_gc_flags(); } check_deprecated_gc_flags(); if (AssumeMP && !UseSerialGC) {
--- a/src/share/vm/runtime/arguments.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/runtime/arguments.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -334,6 +334,8 @@ static void set_parallel_gc_flags(); // Garbage-First (UseG1GC) static void set_g1_gc_flags(); + // Shenandoah GC (UseShenandoahGC) + static void set_shenandoah_gc_flags(); // GC ergonomics static void set_conservative_max_heap_alignment(); static void set_use_compressed_oops(); @@ -609,7 +611,7 @@ }; bool Arguments::gc_selected() { - return UseConcMarkSweepGC || UseG1GC || UseParallelGC || UseParallelOldGC || UseSerialGC; + return UseConcMarkSweepGC || UseG1GC || UseParallelGC || UseParallelOldGC || UseSerialGC || UseShenandoahGC; } // Disable options not supported in this release, with a warning if they
--- a/src/share/vm/runtime/basicLock.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/runtime/basicLock.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -62,7 +62,7 @@ public: // Manipulation - oop obj() const { return _obj; } + oop obj() const { return oopDesc::bs()->resolve_oop(_obj); } void set_obj(oop obj) { _obj = obj; } BasicLock* lock() { return &_lock; }
--- a/src/share/vm/runtime/biasedLocking.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/runtime/biasedLocking.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -531,11 +531,14 @@ BiasedLocking::Condition BiasedLocking::revoke_and_rebias(Handle obj, bool attempt_rebias, TRAPS) { assert(!SafepointSynchronize::is_at_safepoint(), "must not be called while at safepoint"); + Handle n_obj(THREAD, + oopDesc::bs()->resolve_and_maybe_copy_oop(obj())); + // We can revoke the biases of anonymously-biased objects // efficiently enough that we should not cause these revocations to // update the heuristics because doing so may cause unwanted bulk // revocations (which are expensive) to occur. - markOop mark = obj->mark(); + markOop mark = n_obj->mark(); if (mark->is_biased_anonymously() && !attempt_rebias) { // We are probably trying to revoke the bias of this object due to // an identity hash code computation. Try to revoke the bias @@ -545,12 +548,12 @@ // the bias of the object. markOop biased_value = mark; markOop unbiased_prototype = markOopDesc::prototype()->set_age(mark->age()); - markOop res_mark = (markOop) Atomic::cmpxchg_ptr(unbiased_prototype, obj->mark_addr(), mark); + markOop res_mark = (markOop) Atomic::cmpxchg_ptr(unbiased_prototype, n_obj->mark_addr(), mark); if (res_mark == biased_value) { return BIAS_REVOKED; } } else if (mark->has_bias_pattern()) { - Klass* k = obj->klass(); + Klass* k = n_obj->klass(); markOop prototype_header = k->prototype_header(); if (!prototype_header->has_bias_pattern()) { // This object has a stale bias from before the bulk revocation @@ -560,8 +563,8 @@ // by another thread so we simply return and let the caller deal // with it. markOop biased_value = mark; - markOop res_mark = (markOop) Atomic::cmpxchg_ptr(prototype_header, obj->mark_addr(), mark); - assert(!(*(obj->mark_addr()))->has_bias_pattern(), "even if we raced, should still be revoked"); + markOop res_mark = (markOop) Atomic::cmpxchg_ptr(prototype_header, n_obj->mark_addr(), mark); + assert(!(*(n_obj->mark_addr()))->has_bias_pattern(), "even if we raced, should still be revoked"); return BIAS_REVOKED; } else if (prototype_header->bias_epoch() != mark->bias_epoch()) { // The epoch of this biasing has expired indicating that the @@ -575,14 +578,14 @@ assert(THREAD->is_Java_thread(), ""); markOop biased_value = mark; markOop rebiased_prototype = markOopDesc::encode((JavaThread*) THREAD, mark->age(), prototype_header->bias_epoch()); - markOop res_mark = (markOop) Atomic::cmpxchg_ptr(rebiased_prototype, obj->mark_addr(), mark); + markOop res_mark = (markOop) Atomic::cmpxchg_ptr(rebiased_prototype, n_obj->mark_addr(), mark); if (res_mark == biased_value) { return BIAS_REVOKED_AND_REBIASED; } } else { markOop biased_value = mark; markOop unbiased_prototype = markOopDesc::prototype()->set_age(mark->age()); - markOop res_mark = (markOop) Atomic::cmpxchg_ptr(unbiased_prototype, obj->mark_addr(), mark); + markOop res_mark = (markOop) Atomic::cmpxchg_ptr(unbiased_prototype, n_obj->mark_addr(), mark); if (res_mark == biased_value) { return BIAS_REVOKED; } @@ -590,11 +593,11 @@ } } - HeuristicsResult heuristics = update_heuristics(obj(), attempt_rebias); + HeuristicsResult heuristics = update_heuristics(n_obj(), attempt_rebias); if (heuristics == HR_NOT_BIASED) { return NOT_BIASED; } else if (heuristics == HR_SINGLE_REVOKE) { - Klass *k = obj->klass(); + Klass *k = n_obj->klass(); markOop prototype_header = k->prototype_header(); if (mark->biased_locker() == THREAD && prototype_header->bias_epoch() == mark->bias_epoch()) { @@ -611,12 +614,12 @@ if (TraceBiasedLocking) { tty->print_cr("Revoking bias by walking my own stack:"); } - BiasedLocking::Condition cond = revoke_bias(obj(), false, false, (JavaThread*) THREAD); + BiasedLocking::Condition cond = revoke_bias(n_obj(), false, false, (JavaThread*) THREAD); ((JavaThread*) THREAD)->set_cached_monitor_info(NULL); assert(cond == BIAS_REVOKED, "why not?"); return cond; } else { - VM_RevokeBias revoke(&obj, (JavaThread*) THREAD); + VM_RevokeBias revoke(&n_obj, (JavaThread*) THREAD); VMThread::execute(&revoke); return revoke.status_code(); } @@ -624,7 +627,7 @@ assert((heuristics == HR_BULK_REVOKE) || (heuristics == HR_BULK_REBIAS), "?"); - VM_BulkRevokeBias bulk_revoke(&obj, (JavaThread*) THREAD, + VM_BulkRevokeBias bulk_revoke(&n_obj, (JavaThread*) THREAD, (heuristics == HR_BULK_REBIAS), attempt_rebias); VMThread::execute(&bulk_revoke);
--- a/src/share/vm/runtime/frame.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/runtime/frame.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -1021,7 +1021,7 @@ guarantee(oop_adr != NULL, "bad register save location"); return NULL; } - oop r = *oop_adr; + oop r = oopDesc::load_heap_oop(oop_adr); assert(Universe::heap()->is_in_or_null(r), err_msg("bad receiver: " INTPTR_FORMAT " (" INTX_FORMAT ")", (void *) r, (void *) r)); return r; }
--- a/src/share/vm/runtime/globals.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/runtime/globals.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -1518,6 +1518,125 @@ product(bool, UseG1GC, false, \ "Use the Garbage-First garbage collector") \ \ + product(bool, UseShenandoahGC, false, \ + "Use the Shenandoah garbage collector") \ + \ + product(uintx, ShenandoahHeapRegionSize, 0, \ + "Size of the Shenandoah regions.") \ + \ + develop(bool, ShenandoahDumpHeapBeforeConcurrentMark, false, \ + "Dump the ShenanodahHeap Before Each ConcurrentMark") \ + \ + develop(bool, ShenandoahDumpHeapAfterConcurrentMark, false, \ + "Dump the ShenanodahHeap After Each Concurrent Mark") \ + \ + product(bool, ShenandoahTraceFullGC, false, \ + "Trace Shenandoah full GC") \ + \ + product(bool, ShenandoahTracePhases, false, \ + "Trace Shenandoah GC phases") \ + \ + develop(bool, ShenandoahTraceJNICritical, false, \ + "Trace Shenandoah stalls for JNI critical regions") \ + \ + product(bool, ShenandoahTraceHumongous, false, \ + "Trace Shenandoah humongous objects") \ + \ + develop(bool, ShenandoahTraceAllocations, false, \ + "Trace Shenandoah Allocations") \ + \ + develop(bool, ShenandoahTraceBrooksPointers, false, \ + "Trace Brooks Pointer updates") \ + \ + develop(bool, ShenandoahTraceEvacuations, false, \ + "Trace Shenandoah Evacuations") \ + \ + develop(bool, ShenandoahVerifyWritesToFromSpace, false, \ + "Use Memory Protection to signal illegal writes to from space") \ + \ + develop(bool, ShenandoahVerifyReadsToFromSpace, false, \ + "Use Memory Protection to signal illegal reads to from space") \ + \ + develop(bool, ShenandoahTraceConcurrentMarking, false, \ + "Trace Concurrent Marking") \ + \ + develop(bool, ShenandoahTraceUpdates, false, \ + "Trace Shenandoah Updates") \ + \ + develop(bool, ShenandoahTraceTLabs, false, \ + "Trace TLabs in Shenandoah Heap") \ + \ + product(bool, ShenandoahProcessReferences, true, \ + "Enable processing of (soft/weak/..) references in Shenandoah") \ + \ + develop(bool, ShenandoahTraceWeakReferences, false, \ + "Trace Weak Reference Processing in Shenandoah Heap") \ + \ + product(bool, ShenandoahGCVerbose, false, \ + "Verbose information about the Shenandoah garbage collector") \ + \ + product(bool, ShenandoahLogConfig, false, \ + "Log information about Shenandoah's configuration settings") \ + \ + develop(bool, ShenandoahVerify, false, \ + "Verify the Shenandoah garbage collector") \ + \ + product(bool, ShenandoahParallelRootScan, true, \ + "Turn on/off parallel root scanning in Shenandoah") \ + \ + product(bool, ShenandoahConcurrentEvacuation, true, \ + "Turn on/off concurrent evacuation in Shenandoah") \ + \ + product(bool, ShenandoahConcurrentMarking, true, \ + "Turn on/off concurrent marking in Shenandoah") \ + \ + product(bool, ShenandoahUpdateRefsEarly,false, \ + "Turn on/off early updating of references in Shenandoah") \ + \ + product(bool, ShenandoahConcurrentUpdateRefs, true, \ + "Turn on/off concurrent reference updating in Shenandoah") \ + \ + product(bool, ShenandoahWriteBarrier, true, \ + "Turn on/off write barriers in Shenandoah") \ + \ + product(bool, ShenandoahReadBarrier, true, \ + "Turn on/off read barriers in Shenandoah") \ + \ + product(ccstr, ShenandoahGCHeuristics, "dynamic", \ + "The heuristics to use in Shenandoah GC; possible values: " \ + "statusquo, aggressive, halfway, lazy, dynamic") \ + \ + product(uintx, ShenandoahGarbageThreshold, 60, \ + "Sets the percentage of garbage a region need to contain before " \ + "it can be marked for collection. Applies to " \ + "Shenandoah GC dynamic Heuristic mode only (ignored otherwise)") \ + \ + product(uintx, ShenandoahFreeThreshold, 25, \ + "Set the percentage of heap free in relation to the total " \ + "capacity before a region can enter the concurrent marking " \ + "phase. Applies to Shenandoah GC dynamic Heuristic mode only " \ + "(ignored otherwise)") \ + \ + product(uintx, ShenandoahInitialFreeThreshold, 50, \ + "Set the percentage of heap free in relation to the total " \ + "capacity before a region can enter the concurrent marking " \ + "phase. Applies to Shenandoah GC dynamic Heuristic mode only " \ + "(ignored otherwise)") \ + \ + product(uintx, ShenandoahAllocationThreshold, 0, \ + "Set the number of bytes allocated since last GC cycle before" \ + "a region can enter the concurrent marking " \ + "phase. Applies to Shenandoah GC dynamic Heuristic mode only " \ + "(ignored otherwise)") \ + \ + product(uintx, ShenandoahTargetHeapOccupancy, 80, \ + "Sets the target maximum percentage occupance of the heap we" \ + "would like to maintain." \ + "Shenandoah GC newadaptive Heuristic mode only.") \ + \ + product(bool, ShenandoahPrintCollectionSet, false, \ + "Print the collection set before each GC phase") \ + \ product(bool, UseParallelGC, false, \ "Use the Parallel Scavenge garbage collector") \ \
--- a/src/share/vm/runtime/handles.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/runtime/handles.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -36,6 +36,9 @@ oop* HandleArea::allocate_handle(oop obj) { assert(_handle_mark_nesting > 1, "memory leak: allocating handle outside HandleMark"); assert(_no_handle_mark_nesting == 0, "allocating handle inside NoHandleMark"); + if (ShenandoahVerifyReadsToFromSpace) { + obj = oopDesc::bs()->resolve_oop(obj); + } assert(obj->is_oop(), err_msg("not an oop: " INTPTR_FORMAT, (intptr_t*) obj)); return real_allocate_handle(obj); }
--- a/src/share/vm/runtime/handles.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/runtime/handles.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -25,6 +25,7 @@ #ifndef SHARE_VM_RUNTIME_HANDLES_HPP #define SHARE_VM_RUNTIME_HANDLES_HPP +#include "gc/shared/barrierSet.hpp" #include "oops/oop.hpp" #include "oops/oopsHierarchy.hpp" @@ -70,8 +71,8 @@ oop* _handle; protected: - oop obj() const { return _handle == NULL ? (oop)NULL : *_handle; } - oop non_null_obj() const { assert(_handle != NULL, "resolving NULL handle"); return *_handle; } + oop obj() const { return _handle == NULL ? (oop)NULL : oopDesc::bs()->resolve_oop(*_handle); } + oop non_null_obj() const { assert(_handle != NULL, "resolving NULL handle"); return oopDesc::bs()->resolve_oop(*_handle);} public: // Constructors @@ -82,7 +83,7 @@ // General access oop operator () () const { return obj(); } oop operator -> () const { return non_null_obj(); } - bool operator == (oop o) const { return obj() == o; } + bool operator == (oop o) const { return obj() == oopDesc::bs()->resolve_oop(o); } bool operator == (const Handle& h) const { return obj() == h.obj(); } // Null checks
--- a/src/share/vm/runtime/jniHandles.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/runtime/jniHandles.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -27,6 +27,7 @@ #include "runtime/handles.hpp" #include "utilities/top.hpp" +#include "oops/oop.hpp" class JNIHandleBlock; @@ -65,7 +66,7 @@ // Sentinel marking deleted handles in block. Note that we cannot store NULL as // the sentinel, since clearing weak global JNI refs are done by storing NULL in // the handle. The handle may not be reused before destroy_weak_global is called. - static oop deleted_handle() { return _deleted_handle; } + static oop deleted_handle() { return oopDesc::bs()->resolve_oop(_deleted_handle); } // Initialization static void initialize(); @@ -174,6 +175,7 @@ inline oop JNIHandles::resolve(jobject handle) { oop result = (handle == NULL ? (oop)NULL : *(oop*)handle); + result = oopDesc::bs()->resolve_oop(result); assert(result != NULL || (handle == NULL || !CheckJNICalls || is_weak_global_handle(handle)), "Invalid value read from jni handle"); assert(result != badJNIHandle, "Pointing to zapped jni handle area"); return result; @@ -183,6 +185,7 @@ inline oop JNIHandles::resolve_external_guard(jobject handle) { if (handle == NULL) return NULL; oop result = *(oop*)handle; + result = oopDesc::bs()->resolve_oop(result); if (result == NULL || result == badJNIHandle) return NULL; return result; }; @@ -191,6 +194,7 @@ inline oop JNIHandles::resolve_non_null(jobject handle) { assert(handle != NULL, "JNI handle should not be null"); oop result = *(oop*)handle; + result = oopDesc::bs()->resolve_oop(result); assert(result != NULL, "Invalid value read from jni handle"); assert(result != badJNIHandle, "Pointing to zapped jni handle area"); // Don't let that private _deleted_handle object escape into the wild.
--- a/src/share/vm/runtime/mutexLocker.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/runtime/mutexLocker.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -53,6 +53,9 @@ Mutex* JvmtiThreadState_lock = NULL; Monitor* JvmtiPendingEvent_lock = NULL; Monitor* Heap_lock = NULL; +Monitor* ShenandoahHeap_lock = NULL; +Monitor* ShenandoahFullGC_lock = NULL; +Monitor* ShenandoahJNICritical_lock = NULL; Mutex* ExpandHeap_lock = NULL; Mutex* AdapterHandlerLibrary_lock = NULL; Mutex* SignatureHandlerLibrary_lock = NULL; @@ -100,6 +103,7 @@ Mutex* ExceptionCache_lock = NULL; Monitor* ObjAllocPost_lock = NULL; Mutex* OsrList_lock = NULL; +Monitor* ShenandoahMemProtect_lock = NULL; #ifndef PRODUCT Mutex* FullGCALot_lock = NULL; @@ -203,6 +207,14 @@ def(StringDedupQueue_lock , Monitor, leaf, true, Monitor::_safepoint_check_never); def(StringDedupTable_lock , Mutex , leaf, true, Monitor::_safepoint_check_never); } + if (UseShenandoahGC) { + def(SATB_Q_FL_lock , Mutex , special, true, Monitor::_safepoint_check_never); + def(SATB_Q_CBL_mon , Monitor, nonleaf, true, Monitor::_safepoint_check_never); + def(Shared_SATB_Q_lock , Mutex, nonleaf, true, Monitor::_safepoint_check_never); + def(ShenandoahFullGC_lock , Monitor, leaf, true, Monitor::_safepoint_check_always); + def(ShenandoahJNICritical_lock , Monitor, nonleaf+1, false, Monitor::_safepoint_check_never); + def(ShenandoahMemProtect_lock , Monitor, native, false, Monitor::_safepoint_check_never); + } def(ParGCRareEvent_lock , Mutex , leaf , true, Monitor::_safepoint_check_sometimes); def(DerivedPointerTableGC_lock , Mutex, leaf, true, Monitor::_safepoint_check_never); def(CodeCache_lock , Mutex , special, true, Monitor::_safepoint_check_never); @@ -255,8 +267,8 @@ if (UseConcMarkSweepGC) { def(SLT_lock , Monitor, nonleaf, false, Monitor::_safepoint_check_never); // used in CMS GC for locking PLL lock } - def(Heap_lock , Monitor, nonleaf+1, false, Monitor::_safepoint_check_sometimes); + def(ShenandoahHeap_lock , Monitor, special, false, Monitor::_safepoint_check_never); def(JfieldIdCreation_lock , Mutex , nonleaf+1, true, Monitor::_safepoint_check_always); // jfieldID, Used in VM_Operation def(MemberNameTable_lock , Mutex , nonleaf+1, false, Monitor::_safepoint_check_always); // Used to protect MemberNameTable
--- a/src/share/vm/runtime/mutexLocker.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/runtime/mutexLocker.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -45,6 +45,9 @@ extern Mutex* JvmtiThreadState_lock; // a lock on modification of JVMTI thread data extern Monitor* JvmtiPendingEvent_lock; // a lock on the JVMTI pending events list extern Monitor* Heap_lock; // a lock on the heap +extern Monitor* ShenandoahHeap_lock; // a lock on the heap, used by ShenandoahGC when evacuating at a safepoint +extern Monitor* ShenandoahFullGC_lock; // a monitor to wait/notify the Shenandoah background thread on full-GC requests +extern Monitor* ShenandoahJNICritical_lock; // a monitor to wait/notify the Shenandoah background thread on full-GC requests extern Mutex* ExpandHeap_lock; // a lock on expanding the heap extern Mutex* AdapterHandlerLibrary_lock; // a lock on the AdapterHandlerLibrary extern Mutex* SignatureHandlerLibrary_lock; // a lock on the SignatureHandlerLibrary @@ -102,6 +105,7 @@ extern Mutex* ProfilePrint_lock; // a lock used to serialize the printing of profiles extern Mutex* ExceptionCache_lock; // a lock used to synchronize exception cache updates extern Mutex* OsrList_lock; // a lock used to serialize access to OSR queues +extern Monitor* ShenandoahMemProtect_lock; // ShenandoahGC uses this for memory protection to verify operations on the heap. #ifndef PRODUCT extern Mutex* FullGCALot_lock; // a lock to make FullGCALot MT safe @@ -347,13 +351,17 @@ Monitor * _mutex; bool _reentrant; public: - VerifyMutexLocker(Monitor * mutex) { + VerifyMutexLocker(Monitor * mutex, bool no_safepoint_check = !Mutex::_no_safepoint_check_flag) { _mutex = mutex; _reentrant = mutex->owned_by_self(); if (!_reentrant) { // We temp. disable strict safepoint checking, while we require the lock FlagSetting fs(StrictSafepointChecks, false); - _mutex->lock(); + if (no_safepoint_check == Mutex::_no_safepoint_check_flag) { + _mutex->lock_without_safepoint_check(); + } else { + _mutex->lock(); + } } }
--- a/src/share/vm/runtime/objectMonitor.inline.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/runtime/objectMonitor.inline.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -66,7 +66,7 @@ inline void* ObjectMonitor::object() const { - return _object; + return oopDesc::bs()->maybe_resolve_oop(oop(_object)); } inline void* ObjectMonitor::object_addr() {
--- a/src/share/vm/runtime/os.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/runtime/os.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -436,6 +436,7 @@ vm_thread, cgc_thread, // Concurrent GC thread pgc_thread, // Parallel GC thread + shenandoah_thread, java_thread, compiler_thread, watcher_thread,
--- a/src/share/vm/runtime/reflection.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/runtime/reflection.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -405,7 +405,7 @@ assert(lower_dim->oop_is_array(), "just checking"); result2 = lower_dim->java_mirror(); } - assert(result == result2, "results must be consistent"); + assert(oopDesc::bs()->resolve_oop(result) == oopDesc::bs()->resolve_oop(result2), "results must be consistent"); #endif //ASSERT return result; }
--- a/src/share/vm/runtime/safepoint.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/runtime/safepoint.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -56,6 +56,7 @@ #include "utilities/events.hpp" #include "utilities/macros.hpp" #if INCLUDE_ALL_GCS +#include "gc/shenandoah/shenandoahConcurrentThread.hpp" #include "gc/cms/concurrentMarkSweepThread.hpp" #include "gc/g1/suspendibleThreadSet.hpp" #endif // INCLUDE_ALL_GCS @@ -95,7 +96,10 @@ ConcurrentMarkSweepThread::synchronize(false); } else if (UseG1GC) { SuspendibleThreadSet::synchronize(); + } else if (UseShenandoahGC) { + ShenandoahConcurrentThread::safepoint_synchronize(); } + #endif // INCLUDE_ALL_GCS // By getting the Threads_lock, we assure that no threads are about to start or @@ -469,6 +473,8 @@ ConcurrentMarkSweepThread::desynchronize(false); } else if (UseG1GC) { SuspendibleThreadSet::desynchronize(); + } else if (UseShenandoahGC) { + ShenandoahConcurrentThread::safepoint_desynchronize(); } #endif // INCLUDE_ALL_GCS // record this time so VMThread can keep track how much time has elapsed @@ -968,6 +974,9 @@ // the other registers. In order to preserve it over GCs we need // to keep it in a handle. oop result = caller_fr.saved_oop_result(&map); + if (ShenandoahVerifyReadsToFromSpace) { + result = oopDesc::bs()->resolve_oop(result); + } assert(result == NULL || result->is_oop(), "must be oop"); return_value = Handle(thread(), result); assert(Universe::heap()->is_in_or_null(result), "must be heap pointer");
--- a/src/share/vm/runtime/sharedRuntime.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/runtime/sharedRuntime.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -211,6 +211,11 @@ #endif // INCLUDE_ALL_GCS +// G1 write-barrier pre: executed before a pointer store. +JRT_LEAF(void, SharedRuntime::shenandoah_clone_barrier(oopDesc* obj)) + oopDesc::bs()->write_region(MemRegion((HeapWord*) obj, obj->size())); +JRT_END + JRT_LEAF(jlong, SharedRuntime::lmul(jlong y, jlong x)) return x * y; @@ -1750,6 +1755,10 @@ if (src == NULL || dest == NULL) { THROW(vmSymbols::java_lang_NullPointerException()); } + + src = oopDesc::bs()->resolve_oop(src); + dest = oopDesc::bs()->resolve_and_maybe_copy_oop(dest); + // Do the copy. The casts to arrayOop are necessary to the copy_array API, // even though the copy_array API also performs dynamic checks to ensure // that src and dest are truly arrays (and are conformable). @@ -1825,7 +1834,7 @@ // Handles the uncommon cases of monitor unlocking in compiled code JRT_LEAF(void, SharedRuntime::complete_monitor_unlocking_C(oopDesc* _obj, BasicLock* lock, JavaThread * THREAD)) oop obj(_obj); - assert(JavaThread::current() == THREAD, "invariant"); + obj = oopDesc::bs()->resolve_oop(obj); // I'm not convinced we need the code contained by MIGHT_HAVE_PENDING anymore // testing was unable to ever fire the assert that guarded it so I have removed it. assert(!HAS_PENDING_EXCEPTION, "Do we need code below anymore?");
--- a/src/share/vm/runtime/sharedRuntime.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/runtime/sharedRuntime.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -180,6 +180,7 @@ // G1 write barriers static void g1_wb_pre(oopDesc* orig, JavaThread *thread); static void g1_wb_post(void* card_addr, JavaThread* thread); + static void shenandoah_clone_barrier(oopDesc* obj); #endif // INCLUDE_ALL_GCS // exception handling and implicit exceptions
--- a/src/share/vm/runtime/synchronizer.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/runtime/synchronizer.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -275,6 +275,8 @@ void ObjectSynchronizer::fast_exit(oop object, BasicLock* lock, TRAPS) { assert(!object->mark()->has_bias_pattern(), "should not see bias pattern here"); // if displaced header is null, the previous enter is recursive enter, no-op + object = oopDesc::bs()->resolve_and_maybe_copy_oop(object); + markOop dhw = lock->displaced_header(); markOop mark; if (dhw == NULL) { @@ -314,14 +316,16 @@ // We don't need to use fast path here, because it must have been // failed in the interpreter/compiler code. void ObjectSynchronizer::slow_enter(Handle obj, BasicLock* lock, TRAPS) { - markOop mark = obj->mark(); + Handle n_obj(THREAD, + oopDesc::bs()->resolve_and_maybe_copy_oop(obj())); + markOop mark = n_obj->mark(); assert(!mark->has_bias_pattern(), "should not see bias pattern here"); if (mark->is_neutral()) { // Anticipate successful CAS -- the ST of the displaced mark must // be visible <= the ST performed by the CAS. lock->set_displaced_header(mark); - if (mark == (markOop) Atomic::cmpxchg_ptr(lock, obj()->mark_addr(), mark)) { + if (mark == (markOop) Atomic::cmpxchg_ptr(lock, n_obj()->mark_addr(), mark)) { TEVENT(slow_enter: release stacklock); return; } @@ -329,7 +333,7 @@ } else if (mark->has_locker() && THREAD->is_lock_owned((address)mark->locker())) { assert(lock != mark->locker(), "must not re-lock the same lock"); - assert(lock != (BasicLock*)obj->mark(), "don't relock with same BasicLock"); + assert(lock != (BasicLock*)n_obj->mark(), "don't relock with same BasicLock"); lock->set_displaced_header(NULL); return; } @@ -339,7 +343,7 @@ // must be non-zero to avoid looking like a re-entrant lock, // and must not look locked either. lock->set_displaced_header(markOopDesc::unused_mark()); - ObjectSynchronizer::inflate(THREAD, obj())->enter(THREAD); + ObjectSynchronizer::inflate(THREAD, n_obj())->enter(THREAD); } // This routine is used to handle interpreter/compiler slow case @@ -669,6 +673,7 @@ } intptr_t ObjectSynchronizer::FastHashCode(Thread * Self, oop obj) { + obj = oopDesc::bs()->resolve_and_maybe_copy_oop(obj); if (UseBiasedLocking) { // NOTE: many places throughout the JVM do not expect a safepoint // to be taken here, in particular most operations on perm gen @@ -686,6 +691,7 @@ "biases should not be seen by VM thread here"); BiasedLocking::revoke_and_rebias(hobj, false, JavaThread::current()); obj = hobj(); + obj = oopDesc::bs()->resolve_and_maybe_copy_oop(obj); assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now"); } } @@ -902,6 +908,7 @@ for (int i = _BLOCKSIZE - 1; i > 0; i--) { mid = (ObjectMonitor *)(block + i); oop object = (oop) mid->object(); + object = oopDesc::bs()->resolve_oop(object); if (object != NULL) { closure->do_monitor(mid); } @@ -1289,6 +1296,7 @@ oop object) { // Inflate mutates the heap ... // Relaxing assertion for bug 6320749. + object = oopDesc::bs()->resolve_and_maybe_copy_oop(object); assert(Universe::verify_in_progress() || !SafepointSynchronize::is_at_safepoint(), "invariant"); @@ -1307,7 +1315,7 @@ if (mark->has_monitor()) { ObjectMonitor * inf = mark->monitor(); assert(inf->header()->is_neutral(), "invariant"); - assert(inf->object() == object, "invariant"); + assert(oopDesc::bs()->resolve_oop((oop) inf->object()) == object, "invariant"); assert(ObjectSynchronizer::verify_objmon_isinpool(inf), "monitor is invalid"); return inf; } @@ -1632,6 +1640,7 @@ for (int i = 1; i < _BLOCKSIZE; i++) { ObjectMonitor* mid = (ObjectMonitor*)&block[i]; oop obj = (oop) mid->object(); + obj = oopDesc::bs()->resolve_oop(obj); if (obj == NULL) { // The monitor is not associated with an object. @@ -1798,6 +1807,8 @@ for (int i = 1; i < _BLOCKSIZE; i++) { mid = (ObjectMonitor *)(block + i); oop object = (oop) mid->object(); + object = oopDesc::bs()->resolve_oop(object); + if (object != NULL) { mid->verify(); }
--- a/src/share/vm/runtime/thread.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/runtime/thread.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -95,6 +95,7 @@ #include "utilities/macros.hpp" #include "utilities/preserveException.hpp" #if INCLUDE_ALL_GCS +#include "gc/shenandoah/shenandoahConcurrentThread.hpp" #include "gc/cms/concurrentMarkSweepThread.hpp" #include "gc/g1/concurrentMarkThread.inline.hpp" #include "gc/parallel/pcTasks.hpp" @@ -262,6 +263,8 @@ _MutexEvent = ParkEvent::Allocate(this); _MuxEvent = ParkEvent::Allocate(this); + _evacuating = false; + #ifdef CHECK_UNHANDLED_OOPS if (CheckUnhandledOops) { _unhandled_oops = new UnhandledOops(this); @@ -1265,6 +1268,7 @@ } void WatcherThread::run() { + assert(this == watcher_thread(), "just checking"); this->record_stack_base_and_size(); @@ -1471,13 +1475,15 @@ #if INCLUDE_ALL_GCS SATBMarkQueueSet JavaThread::_satb_mark_queue_set; DirtyCardQueueSet JavaThread::_dirty_card_queue_set; +bool JavaThread::_evacuation_in_progress_global = false; #endif // INCLUDE_ALL_GCS JavaThread::JavaThread(bool is_attaching_via_jni) : Thread() #if INCLUDE_ALL_GCS , _satb_mark_queue(&_satb_mark_queue_set), - _dirty_card_queue(&_dirty_card_queue_set) + _dirty_card_queue(&_dirty_card_queue_set), + _evacuation_in_progress(_evacuation_in_progress_global) #endif // INCLUDE_ALL_GCS { initialize(); @@ -1535,7 +1541,8 @@ Thread() #if INCLUDE_ALL_GCS , _satb_mark_queue(&_satb_mark_queue_set), - _dirty_card_queue(&_dirty_card_queue_set) + _dirty_card_queue(&_dirty_card_queue_set), + _evacuation_in_progress(_evacuation_in_progress_global) #endif // INCLUDE_ALL_GCS { initialize(); @@ -1864,9 +1871,12 @@ // from the list of active threads. We must do this after any deferred // card marks have been flushed (above) so that any entries that are // added to the thread's dirty card queue as a result are not lost. - if (UseG1GC) { + if (UseG1GC || UseShenandoahGC) { flush_barrier_queues(); } + if (UseShenandoahGC && UseTLAB) { + gclab().make_parsable(true); + } #endif // INCLUDE_ALL_GCS // Remove from list of active threads list, and notify VM thread if we are the last non-daemon thread @@ -1901,6 +1911,21 @@ // active field set to true. assert(dirty_queue.is_active(), "dirty card queue should be active"); } + +bool JavaThread::evacuation_in_progress() const { + return _evacuation_in_progress; +} + +void JavaThread::set_evacuation_in_progress(bool in_prog) { + _evacuation_in_progress = in_prog; +} + +void JavaThread::set_evacuation_in_progress_all_threads(bool in_prog) { + _evacuation_in_progress_global = in_prog; + for (JavaThread* t = Threads::first(); t; t = t->next()) { + t->set_evacuation_in_progress(in_prog); + } +} #endif // INCLUDE_ALL_GCS void JavaThread::cleanup_failed_attach_current_thread() { @@ -1930,9 +1955,12 @@ } #if INCLUDE_ALL_GCS - if (UseG1GC) { + if (UseG1GC || UseShenandoahGC) { flush_barrier_queues(); } + if (UseShenandoahGC && UseTLAB) { + gclab().make_parsable(true); + } #endif // INCLUDE_ALL_GCS Threads::remove(this); @@ -3512,9 +3540,11 @@ // Support for ConcurrentMarkSweep. This should be cleaned up // and better encapsulated. The ugly nested if test would go away // once things are properly refactored. XXX YSR - if (UseConcMarkSweepGC || UseG1GC) { + if (UseConcMarkSweepGC || UseG1GC || UseShenandoahGC) { if (UseConcMarkSweepGC) { ConcurrentMarkSweepThread::makeSurrogateLockerThread(CHECK_JNI_ERR); + } else if (UseShenandoahGC) { + ShenandoahConcurrentThread::makeSurrogateLockerThread(CHECK_JNI_ERR); } else { ConcurrentMarkThread::makeSurrogateLockerThread(CHECK_JNI_ERR); } @@ -3923,6 +3953,9 @@ thread->exit(true); + // Stop GC threads. + Universe::heap()->shutdown(); + // Stop VM thread. { // 4945125 The vm thread comes to a safepoint during exit.
--- a/src/share/vm/runtime/thread.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/runtime/thread.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -264,8 +264,11 @@ friend class GC_locker; ThreadLocalAllocBuffer _tlab; // Thread-local eden + ThreadLocalAllocBuffer _gclab; // Thread-local allocation buffer for GC (e.g. evacuation) jlong _allocated_bytes; // Cumulative number of bytes allocated on // the Java heap + jlong _allocated_bytes_gclab; // Cumulative number of bytes allocated on + // the Java heap, in GCLABs TRACE_DATA _trace_data; // Thread-local data for tracing @@ -281,6 +284,8 @@ // ObjectMonitor on which this thread called Object.wait() ObjectMonitor* _current_waiting_monitor; + bool _evacuating; + // Private thread-local objectmonitor list - a simple cache organized as a SLL. public: ObjectMonitor* omFreeList; @@ -424,15 +429,23 @@ ThreadLocalAllocBuffer& tlab() { return _tlab; } void initialize_tlab() { if (UseTLAB) { - tlab().initialize(); + tlab().initialize(false); + gclab().initialize(true); } } + // Thread-Local GC Allocation Buffer (GCLAB) support + ThreadLocalAllocBuffer& gclab() { return _gclab; } + jlong allocated_bytes() { return _allocated_bytes; } void set_allocated_bytes(jlong value) { _allocated_bytes = value; } void incr_allocated_bytes(jlong size) { _allocated_bytes += size; } inline jlong cooked_allocated_bytes(); + jlong allocated_bytes_gclab() { return _allocated_bytes_gclab; } + void set_allocated_bytes_gclab(jlong value) { _allocated_bytes_gclab = value; } + void incr_allocated_bytes_gclab(jlong size) { _allocated_bytes_gclab += size; } + TRACE_DATA* trace_data() { return &_trace_data; } const ThreadExt& ext() const { return _ext; } @@ -465,6 +478,14 @@ _current_waiting_monitor = monitor; } + bool is_evacuating() { + return _evacuating; + } + + void set_evacuating(bool evacuating) { + _evacuating = evacuating; + } + // GC support // Apply "f->do_oop" to all root oops in "this". // Apply "cld_f->do_cld" to CLDs that are otherwise not kept alive. @@ -614,6 +635,8 @@ #undef TLAB_FIELD_OFFSET + static ByteSize gclab_start_offset() { return byte_offset_of(Thread, _gclab) + ThreadLocalAllocBuffer::start_offset(); } + static ByteSize allocated_bytes_offset() { return byte_offset_of(Thread, _allocated_bytes); } public: @@ -956,6 +979,10 @@ static DirtyCardQueueSet _dirty_card_queue_set; void flush_barrier_queues(); + + bool _evacuation_in_progress; + static bool _evacuation_in_progress_global; + #endif // INCLUDE_ALL_GCS friend class VMThread; @@ -1008,7 +1035,7 @@ // Thread oop. threadObj() can be NULL for initial JavaThread // (or for threads attached via JNI) - oop threadObj() const { return _threadObj; } + oop threadObj() const { return oopDesc::bs()->resolve_oop(_threadObj); } void set_threadObj(oop p) { _threadObj = p; } ThreadPriority java_priority() const; // Read from threadObj() @@ -1250,7 +1277,7 @@ void set_callee_target (Method* x) { _callee_target = x; } // Oop results of vm runtime calls - oop vm_result() const { return _vm_result; } + oop vm_result() const { return oopDesc::bs()->resolve_oop(_vm_result); } void set_vm_result (oop x) { _vm_result = x; } Metadata* vm_result_2() const { return _vm_result_2; } @@ -1260,7 +1287,7 @@ void set_deferred_card_mark(MemRegion mr) { _deferred_card_mark = mr; } // Exception handling for compiled methods - oop exception_oop() const { return _exception_oop; } + oop exception_oop() const { return oopDesc::bs()->resolve_oop(_exception_oop); } address exception_pc() const { return _exception_pc; } address exception_handler_pc() const { return _exception_handler_pc; } bool is_method_handle_return() const { return _is_method_handle_return == 1; } @@ -1375,6 +1402,9 @@ #if INCLUDE_ALL_GCS static ByteSize satb_mark_queue_offset() { return byte_offset_of(JavaThread, _satb_mark_queue); } static ByteSize dirty_card_queue_offset() { return byte_offset_of(JavaThread, _dirty_card_queue); } + + static ByteSize evacuation_in_progress_offset() { return byte_offset_of(JavaThread, _evacuation_in_progress); } + #endif // INCLUDE_ALL_GCS // Returns the jni environment for this thread @@ -1671,6 +1701,12 @@ static DirtyCardQueueSet& dirty_card_queue_set() { return _dirty_card_queue_set; } + + bool evacuation_in_progress() const; + + void set_evacuation_in_progress(bool in_prog); + + static void set_evacuation_in_progress_all_threads(bool in_prog); #endif // INCLUDE_ALL_GCS // This method initializes the SATB and dirty card queues before a
--- a/src/share/vm/runtime/vframe.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/runtime/vframe.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -265,11 +265,11 @@ // Accessors oop owner() const { assert(!_owner_is_scalar_replaced, "should not be called for scalar replaced object"); - return _owner; + return oopDesc::bs()->resolve_oop(_owner); } oop owner_klass() const { assert(_owner_is_scalar_replaced, "should not be called for not scalar replaced object"); - return _owner_klass; + return oopDesc::bs()->resolve_oop(_owner_klass); } BasicLock* lock() const { return _lock; } bool eliminated() const { return _eliminated; }
--- a/src/share/vm/runtime/vmStructs.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/runtime/vmStructs.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -208,6 +208,7 @@ #include "opto/phaseX.hpp" #include "opto/regalloc.hpp" #include "opto/rootnode.hpp" +#include "opto/shenandoahSupport.hpp" #include "opto/subnode.hpp" #include "opto/vectornode.hpp" #endif // COMPILER2 @@ -2076,6 +2077,9 @@ declare_c2_type(OverflowAddLNode, OverflowLNode) \ declare_c2_type(OverflowSubLNode, OverflowLNode) \ declare_c2_type(OverflowMulLNode, OverflowLNode) \ + declare_c2_type(ShenandoahBarrierNode, TypeNode) \ + declare_c2_type(ShenandoahReadBarrierNode, ShenandoahBarrierNode) \ + declare_c2_type(ShenandoahWriteBarrierNode, ShenandoahBarrierNode) \ \ /*********************/ \ /* Adapter Blob Entries */ \ @@ -2257,6 +2261,7 @@ declare_constant(BarrierSet::CardTableExtension) \ declare_constant(BarrierSet::G1SATBCT) \ declare_constant(BarrierSet::G1SATBCTLogging) \ + declare_constant(BarrierSet::ShenandoahBarrierSet) \ \ declare_constant(BlockOffsetSharedArray::LogN) \ declare_constant(BlockOffsetSharedArray::LogN_words) \
--- a/src/share/vm/runtime/vm_operations.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/runtime/vm_operations.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -95,6 +95,13 @@ template(HeapIterateOperation) \ template(ReportJavaOutOfMemory) \ template(JFRCheckpoint) \ + template(ShenandoahFullGC) \ + template(ShenandoahInitMark) \ + template(ShenandoahStartEvacuation) \ + template(ShenandoahVerifyHeapAfterEvacuation) \ + template(ShenandoahEvacuation) \ + template(ShenandoahUpdateRootRefs) \ + template(ShenandoahUpdateRefs) \ template(Exit) \ template(LinuxDllLoad) \ template(RotateGCLog) \
--- a/src/share/vm/services/lowMemoryDetector.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/services/lowMemoryDetector.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -297,7 +297,7 @@ if (_sensor_obj != NULL) { Klass* k = Management::sun_management_Sensor_klass(CHECK); instanceKlassHandle sensorKlass (THREAD, k); - Handle sensor_h(THREAD, _sensor_obj); + Handle sensor_h(THREAD, oopDesc::bs()->resolve_oop(_sensor_obj)); Handle usage_h = MemoryService::create_MemoryUsage_obj(_usage, CHECK); JavaValue result(T_VOID); @@ -326,7 +326,7 @@ if (_sensor_obj != NULL) { Klass* k = Management::sun_management_Sensor_klass(CHECK); instanceKlassHandle sensorKlass (THREAD, k); - Handle sensor(THREAD, _sensor_obj); + Handle sensor(THREAD, oopDesc::bs()->resolve_oop(_sensor_obj)); JavaValue result(T_VOID); JavaCallArguments args(sensor);
--- a/src/share/vm/services/memoryManager.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/services/memoryManager.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -91,6 +91,10 @@ return (GCMemoryManager*) new G1OldGenMemoryManager(); } +GCMemoryManager* MemoryManager::get_shenandoah_memory_manager() { + return (GCMemoryManager*) new ShenandoahMemoryManager(); +} + instanceOop MemoryManager::get_memory_manager_instance(TRAPS) { // Must do an acquire so as to force ordering of subsequent // loads from anything _memory_mgr_obj points to or implies.
--- a/src/share/vm/services/memoryManager.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/services/memoryManager.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -63,7 +63,7 @@ void add_pool(MemoryPool* pool); - bool is_manager(instanceHandle mh) { return mh() == _memory_mgr_obj; } + bool is_manager(instanceHandle mh) { return mh() == oopDesc::bs()->resolve_oop(_memory_mgr_obj); } virtual instanceOop get_memory_manager_instance(TRAPS); virtual bool is_gc_memory_manager() { return false; } @@ -83,6 +83,7 @@ static GCMemoryManager* get_psMarkSweep_memory_manager(); static GCMemoryManager* get_g1YoungGen_memory_manager(); static GCMemoryManager* get_g1OldGen_memory_manager(); + static GCMemoryManager* get_shenandoah_memory_manager(); }; class CodeCacheMemoryManager : public MemoryManager { @@ -253,4 +254,13 @@ const char* name() { return "G1 Old Generation"; } }; +class ShenandoahMemoryManager : public GCMemoryManager { +private: +public: + ShenandoahMemoryManager() : GCMemoryManager() {} + + const char* name() { return "Shenandoah";} + +}; + #endif // SHARE_VM_SERVICES_MEMORYMANAGER_HPP
--- a/src/share/vm/services/memoryPool.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/services/memoryPool.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -140,7 +140,7 @@ } } - return pool_obj; + return (instanceOop) oopDesc::bs()->resolve_oop((oop) pool_obj); } inline static size_t get_max_value(size_t val1, size_t val2) {
--- a/src/share/vm/services/memoryService.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/services/memoryService.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -46,6 +46,7 @@ #include "utilities/growableArray.hpp" #include "utilities/macros.hpp" #if INCLUDE_ALL_GCS +#include "gc/shenandoah/shenandoahHeap.hpp" #include "gc/cms/concurrentMarkSweepGeneration.hpp" #include "gc/cms/parNewGeneration.hpp" #include "gc/g1/g1CollectedHeap.inline.hpp" @@ -54,6 +55,7 @@ #include "gc/parallel/psYoungGen.hpp" #include "services/g1MemoryPool.hpp" #include "services/psMemoryPool.hpp" +#include "services/shenandoahMemoryPool.hpp" #endif // INCLUDE_ALL_GCS GrowableArray<MemoryPool*>* MemoryService::_pools_list = @@ -98,6 +100,10 @@ add_g1_heap_info(G1CollectedHeap::heap()); break; } + case CollectedHeap::ShenandoahHeap : { + add_shenandoah_heap_info(ShenandoahHeap::heap()); + break; + } #endif // INCLUDE_ALL_GCS default: { guarantee(false, "Unrecognized kind of heap"); @@ -115,8 +121,12 @@ // All memory pools and memory managers are initialized. // + if (UseShenandoahGC) { + _major_gc_manager->initialize_gc_stat_info(); + } else { _minor_gc_manager->initialize_gc_stat_info(); _major_gc_manager->initialize_gc_stat_info(); + } } // Add memory pools for GenCollectedHeap @@ -187,6 +197,15 @@ add_g1YoungGen_memory_pool(g1h, _major_gc_manager, _minor_gc_manager); add_g1OldGen_memory_pool(g1h, _major_gc_manager); } + +void MemoryService::add_shenandoah_heap_info(ShenandoahHeap* pgch) { + assert(UseShenandoahGC, "sanity"); + _major_gc_manager = MemoryManager::get_shenandoah_memory_manager(); + _minor_gc_manager = MemoryManager::get_shenandoah_memory_manager(); + _managers_list->append(_major_gc_manager); + add_shenandoah_memory_pool(pgch, _major_gc_manager); +} + #endif // INCLUDE_ALL_GCS MemoryPool* MemoryService::add_gen(Generation* gen, @@ -385,6 +404,19 @@ mgr->add_pool(old_gen); _pools_list->append(old_gen); } + +void MemoryService::add_shenandoah_memory_pool(ShenandoahHeap* pgc, + MemoryManager* mgr) { + ShenandoahMemoryPool* pool = new ShenandoahMemoryPool(pgc, + "Shenandoah", + MemoryPool::Heap, + false /* support_usage_threshold */); + + mgr->add_pool(pool); + _pools_list->append(pool); +} + + #endif // INCLUDE_ALL_GCS void MemoryService::add_code_heap_memory_pool(CodeHeap* heap, const char* name) {
--- a/src/share/vm/services/memoryService.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/services/memoryService.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -46,6 +46,7 @@ class GenCollectedHeap; class ParallelScavengeHeap; class G1CollectedHeap; +class ShenandoahHeap; // VM Monitoring and Management Support @@ -92,6 +93,9 @@ static void add_g1OldGen_memory_pool(G1CollectedHeap* g1h, MemoryManager* mgr); + static void add_shenandoah_memory_pool(ShenandoahHeap* pgc, + MemoryManager* mgr); + static MemoryPool* add_space(ContiguousSpace* space, const char* name, bool is_heap, @@ -112,9 +116,11 @@ size_t max_size, bool support_usage_threshold); + static void add_gen_collected_heap_info(GenCollectedHeap* heap); static void add_parallel_scavenge_heap_info(ParallelScavengeHeap* heap); static void add_g1_heap_info(G1CollectedHeap* g1h); + static void add_shenandoah_heap_info(ShenandoahHeap* heap); public: static void set_universe_heap(CollectedHeap* heap);
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/services/shenandoahMemoryPool.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,23 @@ +/* +Copyright 2014 Red Hat, Inc. and/or its affiliates. + */ + +#include "services/shenandoahMemoryPool.hpp" + +ShenandoahMemoryPool::ShenandoahMemoryPool(ShenandoahHeap* gen, + const char* name, + PoolType type, + bool support_usage_threshold) : + CollectedMemoryPool(name, type, gen->capacity(), + gen->max_capacity(), + support_usage_threshold), + _gen(gen) { +} + +MemoryUsage ShenandoahMemoryPool::get_memory_usage() { + size_t maxSize = max_size(); + size_t used = used_in_bytes(); + size_t committed = _gen->capacity(); + + return MemoryUsage(initial_size(), used, committed, maxSize); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/services/shenandoahMemoryPool.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,32 @@ +/* +Copyright 2014 Red Hat, Inc. and/or its affiliates. + */ + + +#ifndef SHARE_VM_SERVICES_SHENANDOAHMEMORYPOOL_HPP +#define SHARE_VM_SERVICES_SHENANDOAHMEMORYPOOL_HPP + +#ifndef SERIALGC +#include "gc/shenandoah/shenandoahHeap.hpp" +#include "services/memoryPool.hpp" +#include "services/memoryUsage.hpp" +#endif + +class ShenandoahMemoryPool : public CollectedMemoryPool { +private: + + ShenandoahHeap* _gen; + +public: + + ShenandoahMemoryPool(ShenandoahHeap* pool, + const char* name, + PoolType type, + bool support_usage_threshold); + MemoryUsage get_memory_usage(); + size_t used_in_bytes() { return _gen->used(); } + size_t max_size() const { return _gen->max_capacity(); } +}; + + +#endif //SHARE_VM_SERVICES_SHENANDOAHMEMORYPOOL_HPP
--- a/src/share/vm/services/threadService.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/services/threadService.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -507,6 +507,7 @@ int len = (_locked_monitors != NULL ? _locked_monitors->length() : 0); for (int i = 0; i < len; i++) { oop o = _locked_monitors->at(i); + o = oopDesc::bs()->resolve_oop(o); InstanceKlass* ik = InstanceKlass::cast(o->klass()); st->print_cr("\t- locked <" INTPTR_FORMAT "> (a %s)", (address)o, ik->external_name()); }
--- a/src/share/vm/services/threadService.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/services/threadService.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -218,7 +218,7 @@ java_lang_Thread::ThreadStatus thread_status() { return _thread_status; } - oop threadObj() const { return _threadObj; } + oop threadObj() const { return oopDesc::bs()->resolve_oop(_threadObj); } void set_next(ThreadSnapshot* n) { _next = n; } @@ -233,8 +233,8 @@ jlong sleep_ticks() { return _sleep_ticks; } - oop blocker_object() { return _blocker_object; } - oop blocker_object_owner() { return _blocker_object_owner; } + oop blocker_object() { return oopDesc::bs()->resolve_oop(_blocker_object); } + oop blocker_object_owner() { return oopDesc::bs()->resolve_oop(_blocker_object_owner); } ThreadSnapshot* next() const { return _next; } ThreadStackTrace* get_stack_trace() { return _stack_trace; }
--- a/src/share/vm/utilities/exceptions.cpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/utilities/exceptions.cpp Tue Sep 29 18:47:52 2015 +0200 @@ -52,6 +52,10 @@ _exception_line = line; } +oop ThreadShadow::pending_exception() const { + return Universe::heap()->barrier_set()->resolve_oop(_pending_exception); +} + void ThreadShadow::clear_pending_exception() { if (TraceClearedExceptions) { if (_pending_exception != NULL) {
--- a/src/share/vm/utilities/exceptions.hpp Fri Sep 25 22:59:24 2015 -0700 +++ b/src/share/vm/utilities/exceptions.hpp Tue Sep 29 18:47:52 2015 +0200 @@ -75,7 +75,7 @@ virtual void unused_initial_virtual() { } public: - oop pending_exception() const { return _pending_exception; } + oop pending_exception() const; bool has_pending_exception() const { return _pending_exception != NULL; } const char* exception_file() const { return _exception_file; } int exception_line() const { return _exception_line; }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/gc/shenandoah/TestBarrierDependencies.java Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,43 @@ + +/* + * @test TestBarrierDependencies + * @run main/othervm -XX:ShenandoahGCHeuristics=aggressive -XX:+UseShenandoahGC -XX:-UseCompressedOops -XX:CompileOnly=TestBarrierDependencies.test TestBarrierDependencies + */ +public class TestBarrierDependencies { + + public static class Fluff { + public Test test1; + public Test test2; + } + + public static class Test { + public int x; + } + + public static Fluff fluff; + private static void setup() { + Test obj = new Test(); + fluff = new Fluff(); + fluff.test1 = obj; + fluff.test2 = obj; + } + public static void main(String[] args) { + setup(); + for (int x = 0; x < 1000000; x++) { + int val = (int) (Math.random() * Integer.MAX_VALUE); + String garbage = new String("hello world"); + test(fluff, val); + } + } + + private static void test(Fluff fluff, int value) { + Test b = fluff.test2; + if (b == null) throw new NullPointerException("b"); + Test a = fluff.test1; + if (a == null) throw new NullPointerException("a"); + a.x = value; + if (b.x != value) { + throw new RuntimeException("got wrong value"); + } + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/gc/shenandoah/TestBarrierDependencies2.java Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,51 @@ + +/** + * @test TestBarrierDependencies2 + * @run main/othervm -XX:ShenandoahGCHeuristics=aggressive -XX:+UseShenandoahGC -XX:-UseCompressedOops -XX:CompileOnly=TestBarrierDependencies.test TestBarrierDependencies2 + */ +public class TestBarrierDependencies2 { + + public static class Fluff { + public TestA test1; + public TestA test2; + public TestB test3; + } + + public static class TestA { + public int x; + } + + public static class TestB { + public int x; + } + + public static Fluff fluff; + private static void setup() { + TestA obj = new TestA(); + fluff = new Fluff(); + fluff.test1 = obj; + fluff.test2 = obj; + fluff.test3 = new TestB(); + } + public static void main(String[] args) { + setup(); + for (int x = 0; x < 1000000; x++) { + int val = (int) (Math.random() * Integer.MAX_VALUE); + String garbage = new String("hello world"); + test(fluff, val); + } + } + + private static void test(Fluff fluff, int value) { + TestA b = fluff.test2; + if (b == null) throw new NullPointerException("b"); + TestA a = fluff.test1; + if (a == null) throw new NullPointerException("a"); + TestB x = fluff.test3; + a.x = value; + x.x = value + 1; //(int) (Math.random() * Integer.MAX_VALUE); // This should have its write barrier elided. + if (b.x != value) { + throw new RuntimeException("got wrong value"); + } + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/gc/shenandoah/TestImplicitNullChecks.java Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,27 @@ + +/* + * @test TestImplicitNullChecks + * @run main/othervm -XX:+UseShenandoahGC -XX:-UseCompressedOops -XX:CompileOnly=TestImplicitNullChecks.test TestImplicitNullChecks + */ +public class TestImplicitNullChecks { + + public static class Test { + public int x; + } + + public static void main(String[] args) { + try { + for (int x = 0; x < 1000000; x++) { + Test t = new Test(); + test(t); + } + test(null); + } catch (NullPointerException ex) { + // Ok. + } + } + + private static void test(Test t) { + t.x = (int) (Math.random() * Integer.MAX_VALUE); + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/gc/shenandoah/TestImplicitNullChecks2.java Tue Sep 29 18:47:52 2015 +0200 @@ -0,0 +1,27 @@ + +/* + * @test TestImplicitNullChecks2 + * @run main/othervm -XX:+UseShenandoahGC -XX:-UseCompressedOops -XX:CompileOnly=TestImplicitNullChecks2.test TestImplicitNullChecks2 + */ +public class TestImplicitNullChecks2 { + + public static class Test { + public int x; + } + + public static void main(String[] args) { + try { + for (int x = 0; x < 1000000; x++) { + Test t = new Test(); + test(t); + } + test(null); + } catch (NullPointerException ex) { + // Ok. + } + } + + private static int test(Test t) { + return t.x; + } +}