Mercurial > hg > shenandoah-preopenjdk-archive > openjdk8 > hotspot
changeset 7446:675763177043
Reapply no-leaf-calls for write barriers.
line wrap: on
line diff
--- a/src/cpu/x86/vm/c1_CodeStubs_x86.cpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/cpu/x86/vm/c1_CodeStubs_x86.cpp Thu Jan 15 20:22:42 2015 +0100 @@ -517,23 +517,6 @@ ///////////////////////////////////////////////////////////////////////////// #if INCLUDE_ALL_GCS -void ShenandoahWriteBarrierStub::emit_code(LIR_Assembler* ce) { - - __ bind(_entry); - - // Do the runtime call. - bool need_swap_regs = rax != obj()->as_register(); - if (need_swap_regs) { - __ xchgq(rax, obj()->as_register()); - } - __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::shenandoah_write_barrier_slow_id))); - if (need_swap_regs) { - __ xchgq(rax, obj()->as_register()); - } - - __ jmp(_continuation); -} - void G1PreBarrierStub::emit_code(LIR_Assembler* ce) { // At this point we know that marking is in progress. // If do_load() is true then we have to emit the
--- a/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp Thu Jan 15 20:22:42 2015 +0100 @@ -32,6 +32,7 @@ #include "c1/c1_ValueStack.hpp" #include "ci/ciArrayKlass.hpp" #include "ci/ciInstance.hpp" +#include "gc_implementation/shenandoah/shenandoahHeap.hpp" #include "gc_interface/collectedHeap.hpp" #include "memory/barrierSet.hpp" #include "memory/cardTableModRefBS.hpp" @@ -1517,6 +1518,48 @@ } } +void LIR_Assembler::emit_opShenandoahWriteBarrier(LIR_OpShenandoahWriteBarrier* op) { + Label done; + Register obj = op->in_opr()->as_register(); + Register res = op->result_opr()->as_register(); + Register tmp1 = op->tmp_opr()->as_register(); + assert_different_registers(res, tmp1); + + if (res != obj) { + __ mov(res, obj); + } + + // Check for null. + if (op->need_null_check()) { + __ testptr(res, res); + __ jcc(Assembler::zero, done); + } + + // The read-barrier. + __ movptr(res, Address(res, -8)); + + // Check for evacuation-in-progress + ExternalAddress evacuation_in_progress = ExternalAddress(ShenandoahHeap::evacuation_in_progress_addr()); + __ movptr(tmp1, evacuation_in_progress); + __ cmpl(tmp1, 0); + __ jcc(Assembler::equal, done); + + if (res != rax) { + __ xchgptr(res, rax); // Move obj into rax and save rax into obj. + } + + __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::shenandoah_write_barrier_slow_id))); + add_call_info_here(op->info()); + verify_oop_map(op->info()); + + if (res != rax) { + __ xchgptr(rax, res); // Swap back obj with rax. + } + + __ bind(done); + +} + void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) { LIR_Opr src = op->in_opr(); LIR_Opr dest = op->result_opr();
--- a/src/cpu/x86/vm/c1_LIRGenerator_x86.cpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/cpu/x86/vm/c1_LIRGenerator_x86.cpp Thu Jan 15 20:22:42 2015 +0100 @@ -301,7 +301,7 @@ } LIR_Opr ary = array.result(); - ary = shenandoah_write_barrier(ary, null_check_info, x->needs_null_check()); + ary = shenandoah_write_barrier(ary, range_check_info, x->needs_null_check()); LIR_Opr val = value.result(); if (obj_store) { if (! val->is_register()) { @@ -747,6 +747,9 @@ obj.load_item(); offset.load_nonconstant(); + LIR_Opr obj_op = obj.result(); + obj_op = shenandoah_write_barrier(obj_op, state_for(x, x->state_before()), false); + if (type == objectType) { cmp.load_item_force(FrameMap::rax_oop_opr); val.load_item(); @@ -763,9 +766,6 @@ LIR_Opr addr = new_pointer_register(); LIR_Address* a; - LIR_Opr obj_op = obj.result(); - obj_op = shenandoah_write_barrier(obj_op, NULL, false); - if(offset.result()->is_constant()) { #ifdef _LP64 jlong c = offset.result()->as_jlong(); @@ -1343,7 +1343,8 @@ LIR_Opr left = xin->result(); LIR_Opr right = yin->result(); if (tag == objectTag && UseShenandoahGC && x->y()->type() != objectNull) { // Don't need to resolve for ifnull. - left = shenandoah_write_barrier(left, NULL, true); + CodeEmitInfo* info = state_for(x, x->state_before()); + left = shenandoah_write_barrier(left, info, true); right = shenandoah_read_barrier(right, NULL, true); } __ cmp(lir_cond(cond), left, right); @@ -1442,8 +1443,8 @@ void LIRGenerator::put_Object_unsafe(LIR_Opr src, LIR_Opr offset, LIR_Opr data, - BasicType type, bool is_volatile) { - src = shenandoah_write_barrier(src, NULL, false); + BasicType type, bool is_volatile, CodeEmitInfo* info) { + src = shenandoah_write_barrier(src, info, false); if (is_volatile && type == T_LONG) { LIR_Address* addr = new LIR_Address(src, offset, T_DOUBLE); LIR_Opr tmp = new_register(T_DOUBLE); @@ -1488,7 +1489,7 @@ assert (type == T_INT || (!x->is_add() && is_obj) LP64_ONLY( || type == T_LONG ), "unexpected type"); LIR_Opr src_op = src.result(); - src_op = shenandoah_write_barrier(src_op, NULL, false); + src_op = shenandoah_write_barrier(src_op, state_for(x, x->state_before()), false); if (is_obj) { data = shenandoah_read_barrier(data, NULL, true); }
--- a/src/cpu/x86/vm/c1_MacroAssembler_x86.cpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/cpu/x86/vm/c1_MacroAssembler_x86.cpp Thu Jan 15 20:22:42 2015 +0100 @@ -43,7 +43,7 @@ Label done; int null_check_offset = -1; - oopDesc::bs()->compile_resolve_oop_for_write(this, obj, true, 1, BarrierSet::ss_all); + oopDesc::bs()->compile_resolve_oop_for_write(this, obj, true, 0, 1, BarrierSet::ss_all); verify_oop(obj); // save object being locked into the BasicObjectLock @@ -109,7 +109,7 @@ if (UseBiasedLocking) { // load object movptr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes())); - oopDesc::bs()->compile_resolve_oop_for_write(this, obj, false, 1, BarrierSet::ss_all); + oopDesc::bs()->compile_resolve_oop_for_write(this, obj, false, 0, 1, BarrierSet::ss_all); biased_locking_exit(obj, hdr, done); } @@ -122,7 +122,7 @@ if (!UseBiasedLocking) { // load object movptr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes())); - oopDesc::bs()->compile_resolve_oop_for_write(this, obj, false, 1, BarrierSet::ss_all); + oopDesc::bs()->compile_resolve_oop_for_write(this, obj, false, 0, 1, BarrierSet::ss_all); } verify_oop(obj); // test if object header is pointing to the displaced header, and if so, restore
--- a/src/cpu/x86/vm/c1_Runtime1_x86.cpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/cpu/x86/vm/c1_Runtime1_x86.cpp Thu Jan 15 20:22:42 2015 +0100 @@ -1655,9 +1655,12 @@ __ jcc(Assembler::zero, done); - save_live_registers(sasm, 1); - __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_static2), rax); + OopMap* map = save_live_registers(sasm, 2); + int call_offset = __ call_RT(rax, noreg, CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_c1), rax); + oop_maps = new OopMapSet(); + oop_maps->add_gc_map(call_offset, map); restore_live_registers_except_rax(sasm); + __ verify_oop(rax); __ bind(done); }
--- a/src/cpu/x86/vm/interp_masm_x86_64.cpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/cpu/x86/vm/interp_masm_x86_64.cpp Thu Jan 15 20:22:42 2015 +0100 @@ -708,7 +708,7 @@ // Load object pointer into obj_reg %c_rarg3 movptr(obj_reg, Address(lock_reg, obj_offset)); // Need to preemptively evacuate obj because we CAS the mark word later. - oopDesc::bs()->compile_resolve_oop_for_write(this, obj_reg, false, 1, BarrierSet::ss_c_rarg1); + oopDesc::bs()->compile_resolve_oop_for_write(this, obj_reg, false, 0, 1, BarrierSet::ss_c_rarg1); if (UseBiasedLocking) { biased_locking_enter(lock_reg, obj_reg, swap_reg, rscratch1, false, done, &slow_case); @@ -794,15 +794,15 @@ save_bcp(); // Save in case of exception + // Load oop into obj_reg(%c_rarg3) + movptr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes())); + // We need to preemptively evacuate obj, because we later CAS the mark word into it. + oopDesc::bs()->compile_resolve_oop_for_write(this, obj_reg, false, 0, 1, BarrierSet::ss_c_rarg1); + // Convert from BasicObjectLock structure to object and BasicLock // structure Store the BasicLock address into %rax lea(swap_reg, Address(lock_reg, BasicObjectLock::lock_offset_in_bytes())); - // Load oop into obj_reg(%c_rarg3) - movptr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes())); - // We need to preemptively evacuate obj, because we later CAS the mark word into it. - oopDesc::bs()->compile_resolve_oop_for_write(this, obj_reg, false, 2, BarrierSet::ss_c_rarg1, BarrierSet::ss_rax); - // Free entry movptr(Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()), (int32_t)NULL_WORD);
--- a/src/cpu/x86/vm/templateTable_x86_64.cpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/cpu/x86/vm/templateTable_x86_64.cpp Thu Jan 15 20:22:42 2015 +0100 @@ -925,7 +925,7 @@ // ebx: index // rdx: array index_check(rdx, rbx); // prefer index in ebx - oopDesc::bs()->compile_resolve_oop_for_write(_masm, rdx, false, 2, BarrierSet::ss_rax, BarrierSet::ss_rbx); + oopDesc::bs()->compile_resolve_oop_for_write(_masm, rdx, false, 0, 3, BarrierSet::noreg, BarrierSet::ss_rbx, BarrierSet::ss_rax); __ movl(Address(rdx, rbx, Address::times_4, arrayOopDesc::base_offset_in_bytes(T_INT)), @@ -940,7 +940,7 @@ // ebx: index // rdx: array index_check(rdx, rbx); // prefer index in ebx - oopDesc::bs()->compile_resolve_oop_for_write(_masm, rdx, false, 2, BarrierSet::ss_rax, BarrierSet::ss_rbx); + oopDesc::bs()->compile_resolve_oop_for_write(_masm, rdx, false, 0, 3, BarrierSet::noreg, BarrierSet::ss_rbx, BarrierSet::ss_rax); __ movq(Address(rdx, rbx, Address::times_8, arrayOopDesc::base_offset_in_bytes(T_LONG)), @@ -955,7 +955,7 @@ // ebx: index // rdx: array index_check(rdx, rbx); // prefer index in ebx - oopDesc::bs()->compile_resolve_oop_for_write(_masm, rdx, false, 2, BarrierSet::ss_ftos, BarrierSet::ss_rbx); + oopDesc::bs()->compile_resolve_oop_for_write(_masm, rdx, false, 0, 3, BarrierSet::noreg, BarrierSet::ss_rbx, BarrierSet::ss_ftos); __ movflt(Address(rdx, rbx, Address::times_4, arrayOopDesc::base_offset_in_bytes(T_FLOAT)), @@ -970,7 +970,7 @@ // ebx: index // rdx: array index_check(rdx, rbx); // prefer index in ebx - oopDesc::bs()->compile_resolve_oop_for_write(_masm, rdx, false, 2, BarrierSet::ss_dtos, BarrierSet::ss_rbx); + oopDesc::bs()->compile_resolve_oop_for_write(_masm, rdx, false, 0, 3, BarrierSet::noreg, BarrierSet::ss_rbx, BarrierSet::ss_dtos); __ movdbl(Address(rdx, rbx, Address::times_8, arrayOopDesc::base_offset_in_bytes(T_DOUBLE)), @@ -990,7 +990,7 @@ arrayOopDesc::base_offset_in_bytes(T_OBJECT)); index_check(rdx, rcx); // kills rbx - oopDesc::bs()->compile_resolve_oop_for_write(_masm, rdx, false, 2, BarrierSet::ss_rax, BarrierSet::ss_rcx); + oopDesc::bs()->compile_resolve_oop_for_write(_masm, rdx, false, 3, 3, BarrierSet::noreg, BarrierSet::ss_rcx, BarrierSet::ss_rax); // do array store check - check for NULL value first __ testptr(rax, rax); @@ -1042,7 +1042,7 @@ // ebx: index // rdx: array index_check(rdx, rbx); // prefer index in ebx - oopDesc::bs()->compile_resolve_oop_for_write(_masm, rdx, false, 2, BarrierSet::ss_rax, BarrierSet::ss_rbx); + oopDesc::bs()->compile_resolve_oop_for_write(_masm, rdx, false, 0, 3, BarrierSet::noreg, BarrierSet::ss_rbx, BarrierSet::ss_rax); __ movb(Address(rdx, rbx, Address::times_1, arrayOopDesc::base_offset_in_bytes(T_BYTE)), @@ -1057,7 +1057,7 @@ // ebx: index // rdx: array index_check(rdx, rbx); // prefer index in ebx - oopDesc::bs()->compile_resolve_oop_for_write(_masm, rdx, false, 2, BarrierSet::ss_rax, BarrierSet::ss_rbx); + oopDesc::bs()->compile_resolve_oop_for_write(_masm, rdx, false, 0, 3, BarrierSet::noreg, BarrierSet::ss_rbx, BarrierSet::ss_rax); __ movw(Address(rdx, rbx, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)), @@ -1861,12 +1861,12 @@ transition(atos, vtos); // assume branch is more often taken than not (loops use backward branches) Label not_taken; - __ pop_ptr(rdx); // NOTE: We need to preemptively evacuate the oops here (for Shenandoah GC), because // we might get false negatives if both operands are the same, but another // thread evacuates between the load of the 1st and 2nd operand. - oopDesc::bs()->compile_resolve_oop_for_write(_masm, rdx, true, 1, BarrierSet::ss_rax); - oopDesc::bs()->compile_resolve_oop(_masm, rax); + oopDesc::bs()->compile_resolve_oop_for_write(_masm, rax, true, 0, 0); + __ pop_ptr(rdx); + oopDesc::bs()->compile_resolve_oop(_masm, rdx); __ cmpptr(rdx, rax); __ jcc(j_not(cc), not_taken); branch(false, false); @@ -2541,8 +2541,12 @@ // btos { __ pop(btos); - if (!is_static) pop_and_check_object(obj); - oopDesc::bs()->compile_resolve_oop_for_write(_masm, obj, false, 3, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + if (!is_static) { + pop_and_check_object(obj); + oopDesc::bs()->compile_resolve_oop_for_write(_masm, obj, false, 0, 4, BarrierSet::noreg, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + } else { + oopDesc::bs()->compile_resolve_oop_for_write(_masm, obj, false, 0, 3, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + } __ movb(field, rax); if (!is_static) { patch_bytecode(Bytecodes::_fast_bputfield, bc, rbx, true, byte_no); @@ -2557,8 +2561,12 @@ // atos { __ pop(atos); - if (!is_static) pop_and_check_object(obj); - oopDesc::bs()->compile_resolve_oop_for_write(_masm, obj, false, 3, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + if (!is_static) { + pop_and_check_object(obj); + oopDesc::bs()->compile_resolve_oop_for_write(_masm, obj, false, 0, 4, BarrierSet::noreg, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + } else { + oopDesc::bs()->compile_resolve_oop_for_write(_masm, obj, false, 0, 3, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + } // Store into the field do_oop_store(_masm, field, rax, _bs->kind(), false); if (!is_static) { @@ -2574,8 +2582,12 @@ // itos { __ pop(itos); - if (!is_static) pop_and_check_object(obj); - oopDesc::bs()->compile_resolve_oop_for_write(_masm, obj, false, 3, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + if (!is_static) { + pop_and_check_object(obj); + oopDesc::bs()->compile_resolve_oop_for_write(_masm, obj, false, 0, 4, BarrierSet::noreg, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + } else { + oopDesc::bs()->compile_resolve_oop_for_write(_masm, obj, false, 0, 3, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + } __ movl(field, rax); if (!is_static) { patch_bytecode(Bytecodes::_fast_iputfield, bc, rbx, true, byte_no); @@ -2590,8 +2602,12 @@ // ctos { __ pop(ctos); - if (!is_static) pop_and_check_object(obj); - oopDesc::bs()->compile_resolve_oop_for_write(_masm, obj, false, 3, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + if (!is_static) { + pop_and_check_object(obj); + oopDesc::bs()->compile_resolve_oop_for_write(_masm, obj, false, 0, 4, BarrierSet::noreg, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + } else { + oopDesc::bs()->compile_resolve_oop_for_write(_masm, obj, false, 0, 3, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + } __ movw(field, rax); if (!is_static) { patch_bytecode(Bytecodes::_fast_cputfield, bc, rbx, true, byte_no); @@ -2606,8 +2622,12 @@ // stos { __ pop(stos); - if (!is_static) pop_and_check_object(obj); - oopDesc::bs()->compile_resolve_oop_for_write(_masm, obj, false, 3, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + if (!is_static) { + pop_and_check_object(obj); + oopDesc::bs()->compile_resolve_oop_for_write(_masm, obj, false, 0, 4, BarrierSet::noreg, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + } else { + oopDesc::bs()->compile_resolve_oop_for_write(_masm, obj, false, 0, 3, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + } __ movw(field, rax); if (!is_static) { patch_bytecode(Bytecodes::_fast_sputfield, bc, rbx, true, byte_no); @@ -2622,8 +2642,12 @@ // ltos { __ pop(ltos); - if (!is_static) pop_and_check_object(obj); - oopDesc::bs()->compile_resolve_oop_for_write(_masm, obj, false, 3, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + if (!is_static) { + pop_and_check_object(obj); + oopDesc::bs()->compile_resolve_oop_for_write(_masm, obj, false, 0, 4, BarrierSet::noreg, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + } else { + oopDesc::bs()->compile_resolve_oop_for_write(_masm, obj, false, 0, 3, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + } __ movq(field, rax); if (!is_static) { patch_bytecode(Bytecodes::_fast_lputfield, bc, rbx, true, byte_no); @@ -2638,8 +2662,12 @@ // ftos { __ pop(ftos); - if (!is_static) pop_and_check_object(obj); - oopDesc::bs()->compile_resolve_oop_for_write(_masm, obj, false, 4, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx, BarrierSet::ss_ftos); + if (!is_static) { + pop_and_check_object(obj); + oopDesc::bs()->compile_resolve_oop_for_write(_masm, obj, false, 0, 4, BarrierSet::noreg, BarrierSet::ss_ftos, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + } else { + oopDesc::bs()->compile_resolve_oop_for_write(_masm, obj, false, 0, 3, BarrierSet::ss_ftos, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + } __ movflt(field, xmm0); if (!is_static) { patch_bytecode(Bytecodes::_fast_fputfield, bc, rbx, true, byte_no); @@ -2656,8 +2684,12 @@ // dtos { __ pop(dtos); - if (!is_static) pop_and_check_object(obj); - oopDesc::bs()->compile_resolve_oop_for_write(_masm, obj, false, 4, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx, BarrierSet::ss_dtos); + if (!is_static) { + pop_and_check_object(obj); + oopDesc::bs()->compile_resolve_oop_for_write(_masm, obj, false, 0, 4, BarrierSet::noreg, BarrierSet::ss_dtos, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + } else { + oopDesc::bs()->compile_resolve_oop_for_write(_masm, obj, false, 0, 3, BarrierSet::ss_dtos, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + } __ movdbl(field, xmm0); if (!is_static) { patch_bytecode(Bytecodes::_fast_dputfield, bc, rbx, true, byte_no); @@ -2778,33 +2810,33 @@ // access field switch (bytecode()) { case Bytecodes::_fast_aputfield: - oopDesc::bs()->compile_resolve_oop_for_write(_masm, rcx, false, 3, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + oopDesc::bs()->compile_resolve_oop_for_write(_masm, rcx, false, 0, 4, BarrierSet::noreg, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); do_oop_store(_masm, field, rax, _bs->kind(), false); break; case Bytecodes::_fast_lputfield: - oopDesc::bs()->compile_resolve_oop_for_write(_masm, rcx, false, 3, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + oopDesc::bs()->compile_resolve_oop_for_write(_masm, rcx, false, 0, 4, BarrierSet::noreg, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); __ movq(field, rax); break; case Bytecodes::_fast_iputfield: - oopDesc::bs()->compile_resolve_oop_for_write(_masm, rcx, false, 3, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + oopDesc::bs()->compile_resolve_oop_for_write(_masm, rcx, false, 0, 4, BarrierSet::noreg, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); __ movl(field, rax); break; case Bytecodes::_fast_bputfield: - oopDesc::bs()->compile_resolve_oop_for_write(_masm, rcx, false, 3, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + oopDesc::bs()->compile_resolve_oop_for_write(_masm, rcx, false, 0, 4, BarrierSet::noreg, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); __ movb(field, rax); break; case Bytecodes::_fast_sputfield: // fall through case Bytecodes::_fast_cputfield: - oopDesc::bs()->compile_resolve_oop_for_write(_masm, rcx, false, 3, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); + oopDesc::bs()->compile_resolve_oop_for_write(_masm, rcx, false, 0, 4, BarrierSet::noreg, BarrierSet::ss_rax, BarrierSet::ss_rbx, BarrierSet::ss_rdx); __ movw(field, rax); break; case Bytecodes::_fast_fputfield: - oopDesc::bs()->compile_resolve_oop_for_write(_masm, rcx, false, 3, BarrierSet::ss_rbx, BarrierSet::ss_rdx, BarrierSet::ss_ftos); + oopDesc::bs()->compile_resolve_oop_for_write(_masm, rcx, false, 0, 4, BarrierSet::noreg, BarrierSet::ss_ftos, BarrierSet::ss_rbx, BarrierSet::ss_rdx); __ movflt(field, xmm0); break; case Bytecodes::_fast_dputfield: - oopDesc::bs()->compile_resolve_oop_for_write(_masm, rcx, false, 3, BarrierSet::ss_rbx, BarrierSet::ss_rdx, BarrierSet::ss_dtos); + oopDesc::bs()->compile_resolve_oop_for_write(_masm, rcx, false, 0, 4, BarrierSet::noreg, BarrierSet::ss_dtos, BarrierSet::ss_rbx, BarrierSet::ss_rdx); __ movdbl(field, xmm0); break; default: @@ -3668,7 +3700,7 @@ // We need to preemptively evacuate the object, because we later compare // it to objects in the BasicObjectLock list, and we might get false negatives // if another thread evacuates the object in the meantime. See acmp. - oopDesc::bs()->compile_resolve_oop_for_write(_masm, rax, false); + oopDesc::bs()->compile_resolve_oop_for_write(_masm, rax, false, 0, 0); const Address monitor_block_top( rbp, frame::interpreter_frame_monitor_block_top_offset * wordSize); @@ -3773,7 +3805,7 @@ // We need to preemptively evacuate the object, because we later compare // it to objects in the BasicObjectLock list, and we might get false negatives // if another thread evacuates the object in the meantime. See acmp. - oopDesc::bs()->compile_resolve_oop_for_write(_masm, rax, false); + oopDesc::bs()->compile_resolve_oop_for_write(_masm, rax, false, 0, 0); const Address monitor_block_top( rbp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
--- a/src/share/vm/c1/c1_CodeStubs.hpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/share/vm/c1/c1_CodeStubs.hpp Thu Jan 15 20:22:42 2015 +0100 @@ -535,28 +535,6 @@ ////////////////////////////////////////////////////////////////////////////////////////// #if INCLUDE_ALL_GCS -class ShenandoahWriteBarrierStub : public CodeStub { -private: - LIR_Opr _obj; - bool _need_null_check; - -public: - ShenandoahWriteBarrierStub(LIR_Opr obj) : - _obj(obj) { - } - LIR_Opr obj() const { return _obj; } - virtual void emit_code(LIR_Assembler* e); - virtual void visit(LIR_OpVisitState* visitor) { - visitor->do_slow_case(); - visitor->do_input(_obj); - - } -#ifndef PRODUCT - virtual void print_name(outputStream* out) const { out->print("ShenandoahWriteBarrierStub"); } -#endif // PRODUCT - -}; - // Code stubs for Garbage-First barriers. class G1PreBarrierStub: public CodeStub { private:
--- a/src/share/vm/c1/c1_GraphBuilder.cpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/share/vm/c1/c1_GraphBuilder.cpp Thu Jan 15 20:22:42 2015 +0100 @@ -1185,7 +1185,7 @@ bool is_bb = tsux->bci() < stream()->cur_bci() || fsux->bci() < stream()->cur_bci(); // In case of loop invariant code motion or predicate insertion // before the body of a loop the state is needed - Instruction *i = append(new If(x, cond, false, y, tsux, fsux, (is_bb || compilation()->is_optimistic()) ? state_before : NULL, is_bb)); + Instruction *i = append(new If(x, cond, false, y, tsux, fsux, (is_bb || compilation()->is_optimistic() || UseShenandoahGC) ? state_before : NULL, is_bb)); assert(i->as_Goto() == NULL || (i->as_Goto()->sux_at(0) == tsux && i->as_Goto()->is_safepoint() == tsux->bci() < stream()->cur_bci()) || @@ -4216,13 +4216,16 @@ bool GraphBuilder::append_unsafe_put_obj(ciMethod* callee, BasicType t, bool is_volatile) { if (InlineUnsafeOps) { + ValueStack* stack = copy_state_before(); Values* args = state()->pop_arguments(callee->arg_size()); null_check(args->at(0)); Instruction* offset = args->at(2); #ifndef _LP64 offset = append(new Convert(Bytecodes::_l2i, offset, as_ValueType(T_INT))); #endif - Instruction* op = append(new UnsafePutObject(t, args->at(1), offset, args->at(3), is_volatile)); + UnsafePutObject* unsafe_put_object = new UnsafePutObject(t, args->at(1), offset, args->at(3), is_volatile); + unsafe_put_object->set_state_before(stack); + Instruction* op = append(unsafe_put_object); compilation()->set_has_unsafe_access(true); kill_all(); } @@ -4345,7 +4348,9 @@ #ifndef _LP64 offset = append(new Convert(Bytecodes::_l2i, offset, as_ValueType(T_INT))); #endif - Instruction* op = append(new UnsafeGetAndSetObject(t, args->at(1), offset, args->at(3), is_add)); + UnsafeGetAndSetObject* inst = new UnsafeGetAndSetObject(t, args->at(1), offset, args->at(3), is_add); + inst->set_state_before(copy_state_before()); + Instruction* op = append(inst); compilation()->set_has_unsafe_access(true); kill_all(); push(op->type(), op);
--- a/src/share/vm/c1/c1_Instruction.hpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/share/vm/c1/c1_Instruction.hpp Thu Jan 15 20:22:42 2015 +0100 @@ -1116,8 +1116,8 @@ public: // creation - IfOp(Value x, Condition cond, Value y, Value tval, Value fval) - : Op2(tval->type()->meet(fval->type()), (Bytecodes::Code)cond, x, y) + IfOp(Value x, Condition cond, Value y, Value tval, Value fval, ValueStack* state_before) + : Op2(tval->type()->meet(fval->type()), (Bytecodes::Code)cond, x, y, state_before) , _tval(tval) , _fval(fval) {
--- a/src/share/vm/c1/c1_LIR.cpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/share/vm/c1/c1_LIR.cpp Thu Jan 15 20:22:42 2015 +0100 @@ -1012,6 +1012,15 @@ do_temp(opProfileType->_tmp); break; } + case lir_shenandoah_wb: { + assert(op->as_OpShenandoahWriteBarrier() != NULL, "must be"); + LIR_OpShenandoahWriteBarrier* opShenandoahWB = (LIR_OpShenandoahWriteBarrier*) op; + do_input(opShenandoahWB->_opr); + do_output(opShenandoahWB->_result); + do_temp(opShenandoahWB->_tmp); + do_info(opShenandoahWB->_info); + break; + } default: ShouldNotReachHere(); } @@ -1110,6 +1119,10 @@ } } +void LIR_OpShenandoahWriteBarrier::emit_code(LIR_Assembler* masm) { + masm->emit_opShenandoahWriteBarrier(this); +} + void LIR_OpConvert::emit_code(LIR_Assembler* masm) { masm->emit_opConvert(this); if (stub() != NULL) { @@ -1820,6 +1833,7 @@ case lir_profile_call: s = "profile_call"; break; // LIR_OpProfileType case lir_profile_type: s = "profile_type"; break; + case lir_shenandoah_wb: s = "shenandoah_wb"; break; // LIR_OpAssert #ifdef ASSERT case lir_assert: s = "assert"; break; @@ -1830,6 +1844,12 @@ return s; } +void LIR_OpShenandoahWriteBarrier::print_instr(outputStream* out) const { + out->print("[obj: "); in_opr()->print(out); out->print("]"); + out->print("[res: "); result_opr()->print(out); out->print("]"); + out->print("[tmp: "); tmp_opr()->print(out); out->print("]"); +} + // LIR_OpJavaCall void LIR_OpJavaCall::print_instr(outputStream* out) const { out->print("call: ");
--- a/src/share/vm/c1/c1_LIR.hpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/share/vm/c1/c1_LIR.hpp Thu Jan 15 20:22:42 2015 +0100 @@ -869,6 +869,7 @@ class LIR_OpConvert; class LIR_OpAllocObj; class LIR_OpRoundFP; +class LIR_OpShenandoahWriteBarrier; class LIR_Op2; class LIR_OpDelay; class LIR_Op3; @@ -935,6 +936,7 @@ , lir_pack64 , lir_unpack64 , lir_unwind + , lir_shenandoah_wb , end_op1 , begin_op2 , lir_cmp @@ -1149,6 +1151,7 @@ virtual LIR_OpCompareAndSwap* as_OpCompareAndSwap() { return NULL; } virtual LIR_OpProfileCall* as_OpProfileCall() { return NULL; } virtual LIR_OpProfileType* as_OpProfileType() { return NULL; } + virtual LIR_OpShenandoahWriteBarrier* as_OpShenandoahWriteBarrier() { return NULL; } #ifdef ASSERT virtual LIR_OpAssert* as_OpAssert() { return NULL; } #endif @@ -1465,6 +1468,23 @@ virtual void print_instr(outputStream* out) const PRODUCT_RETURN; }; +class LIR_OpShenandoahWriteBarrier : public LIR_Op1 { + friend class LIR_OpVisitState; + +private: + bool _need_null_check; + LIR_Opr _tmp; + +public: + LIR_OpShenandoahWriteBarrier(LIR_Opr obj, LIR_Opr result, LIR_Opr tmp, CodeEmitInfo* info, bool need_null_check) : LIR_Op1(lir_shenandoah_wb, obj, result, T_OBJECT, lir_patch_none, info), _tmp(tmp), _need_null_check(need_null_check) { + } + LIR_Opr tmp_opr() const { return _tmp; } + bool need_null_check() const { return _need_null_check; } + virtual void emit_code(LIR_Assembler* masm); + virtual LIR_OpShenandoahWriteBarrier* as_OpShenandoahWriteBarrier() { return this; } + virtual void print_instr(outputStream* out) const PRODUCT_RETURN; + +}; class ConversionStub; @@ -2146,6 +2166,8 @@ #endif void convert(Bytecodes::Code code, LIR_Opr left, LIR_Opr dst, ConversionStub* stub = NULL/*, bool is_32bit = false*/) { append(new LIR_OpConvert(code, left, dst, stub)); } + void shenandoah_wb(LIR_Opr obj, LIR_Opr result, LIR_Opr tmp, CodeEmitInfo* info, bool need_null_check) { append(new LIR_OpShenandoahWriteBarrier(obj, result, tmp, info, need_null_check)); } + void logical_and (LIR_Opr left, LIR_Opr right, LIR_Opr dst) { append(new LIR_Op2(lir_logic_and, left, right, dst)); } void logical_or (LIR_Opr left, LIR_Opr right, LIR_Opr dst) { append(new LIR_Op2(lir_logic_or, left, right, dst)); } void logical_xor (LIR_Opr left, LIR_Opr right, LIR_Opr dst) { append(new LIR_Op2(lir_logic_xor, left, right, dst)); }
--- a/src/share/vm/c1/c1_LIRAssembler.hpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/share/vm/c1/c1_LIRAssembler.hpp Thu Jan 15 20:22:42 2015 +0100 @@ -199,6 +199,7 @@ void emit_opLabel(LIR_OpLabel* op); void emit_arraycopy(LIR_OpArrayCopy* op); void emit_updatecrc32(LIR_OpUpdateCRC32* op); + void emit_opShenandoahWriteBarrier(LIR_OpShenandoahWriteBarrier* op); void emit_opConvert(LIR_OpConvert* op); void emit_alloc_obj(LIR_OpAllocObj* op); void emit_alloc_array(LIR_OpAllocArray* op);
--- a/src/share/vm/c1/c1_LIRGenerator.cpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/share/vm/c1/c1_LIRGenerator.cpp Thu Jan 15 20:22:42 2015 +0100 @@ -1738,7 +1738,7 @@ __ null_check(obj, new CodeEmitInfo(info)); } - obj = shenandoah_write_barrier(obj, info, x->needs_null_check()); + obj = shenandoah_write_barrier(obj, state_for(x, x->state_before()), x->needs_null_check()); LIR_Opr val = value.result(); if (is_oop) { if (! val->is_register()) { @@ -1880,35 +1880,12 @@ LIR_Opr LIRGenerator::shenandoah_write_barrier(LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) { if (UseShenandoahGC) { - LabelObj* done = new LabelObj(); LIR_Opr result = new_register(T_OBJECT); - __ move(obj, result); - - // Usual read barrier with optional explicit null check. - if (need_null_check) { - __ cmp(lir_cond_equal, result, LIR_OprFact::oopConst(NULL)); - __ branch(lir_cond_equal, T_LONG, done->label()); - } - - // Check if evacuation in progress. - LIR_Opr evac_in_progress_addr = new_pointer_register(); - __ move(LIR_OprFact::intptrConst(ShenandoahHeap::evacuation_in_progress_addr()), - evac_in_progress_addr); - - LIR_Address* brooks_ptr_address = generate_address(result, -8, T_ADDRESS); - __ load(brooks_ptr_address, result, info ? new CodeEmitInfo(info) : NULL, lir_patch_none); - - __ cmp(lir_cond_notEqual, LIR_OprFact::address(new LIR_Address(evac_in_progress_addr, T_INT)), - LIR_OprFact::intConst(0)); - - // Do the slow-path runtime call. - CodeStub* slow = new ShenandoahWriteBarrierStub(result); - __ branch(lir_cond_notEqual, T_INT, slow); - __ branch_destination(slow->continuation()); - __ branch_destination(done->label()); - + LIR_Opr tmp = new_register(T_INT); + __ shenandoah_wb(obj, result, tmp, new CodeEmitInfo(info), need_null_check); return result; + } else { return obj; } @@ -2463,7 +2440,8 @@ set_no_result(x); if (x->is_volatile() && os::is_MP()) __ membar_release(); - put_Object_unsafe(src.result(), off.result(), data.result(), type, x->is_volatile()); + CodeEmitInfo* info = state_for(x, x->state_before()); + put_Object_unsafe(src.result(), off.result(), data.result(), type, x->is_volatile(), info); if (x->is_volatile() && os::is_MP()) __ membar(); } @@ -3140,8 +3118,9 @@ LIR_Opr left_opr = left.result(); LIR_Opr right_opr = right.result(); if (xtag == objectTag && UseShenandoahGC && x->y()->type() != objectNull) { // Don't need to resolve for ifnull. - left_opr = shenandoah_write_barrier(left_opr, NULL, true); - right_opr = shenandoah_read_barrier(right_opr, NULL, true); + CodeEmitInfo* info = state_for(x, x->state_before()); + left_opr = shenandoah_write_barrier(left_opr, info, true); + right_opr = shenandoah_read_barrier(right_opr, info, true); } __ cmp(lir_cond(x->cond()), left_opr, right_opr);
--- a/src/share/vm/c1/c1_LIRGenerator.hpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/share/vm/c1/c1_LIRGenerator.hpp Thu Jan 15 20:22:42 2015 +0100 @@ -299,7 +299,7 @@ void volatile_field_store(LIR_Opr value, LIR_Address* address, CodeEmitInfo* info); void volatile_field_load(LIR_Address* address, LIR_Opr result, CodeEmitInfo* info); - void put_Object_unsafe(LIR_Opr src, LIR_Opr offset, LIR_Opr data, BasicType type, bool is_volatile); + void put_Object_unsafe(LIR_Opr src, LIR_Opr offset, LIR_Opr data, BasicType type, bool is_volatile, CodeEmitInfo* info); void get_Object_unsafe(LIR_Opr dest, LIR_Opr src, LIR_Opr offset, BasicType type, bool is_volatile); void arithmetic_call_op (Bytecodes::Code code, LIR_Opr result, LIR_OprList* args);
--- a/src/share/vm/c1/c1_Optimizer.cpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/share/vm/c1/c1_Optimizer.cpp Thu Jan 15 20:22:42 2015 +0100 @@ -90,7 +90,7 @@ virtual void block_do(BlockBegin* block); private: - Value make_ifop(Value x, Instruction::Condition cond, Value y, Value tval, Value fval); + Value make_ifop(Value x, Instruction::Condition cond, Value y, Value tval, Value fval, ValueStack* state_before); }; void CE_Eliminator::block_do(BlockBegin* block) { @@ -194,7 +194,8 @@ cur_end = cur_end->set_next(f_value); } - Value result = make_ifop(if_->x(), if_->cond(), if_->y(), t_value, f_value); + ValueStack* state_before = if_->state_before(); + Value result = make_ifop(if_->x(), if_->cond(), if_->y(), t_value, f_value, state_before); assert(result != NULL, "make_ifop must return a non-null instruction"); if (!result->is_linked() && result->can_be_linked()) { NOT_PRODUCT(result->set_printable_bci(if_->printable_bci())); @@ -202,7 +203,6 @@ } // append Goto to successor - ValueStack* state_before = if_->state_before(); Goto* goto_ = new Goto(sux, state_before, if_->is_safepoint() || t_goto->is_safepoint() || f_goto->is_safepoint()); // prepare state for Goto @@ -246,9 +246,9 @@ _hir->verify(); } -Value CE_Eliminator::make_ifop(Value x, Instruction::Condition cond, Value y, Value tval, Value fval) { +Value CE_Eliminator::make_ifop(Value x, Instruction::Condition cond, Value y, Value tval, Value fval, ValueStack* state_before) { if (!OptimizeIfOps) { - return new IfOp(x, cond, y, tval, fval); + return new IfOp(x, cond, y, tval, fval, state_before); } tval = tval->subst(); @@ -283,7 +283,7 @@ if (new_tval == new_fval) { return new_tval; } else { - return new IfOp(x_ifop->x(), x_ifop_cond, x_ifop->y(), new_tval, new_fval); + return new IfOp(x_ifop->x(), x_ifop_cond, x_ifop->y(), new_tval, new_fval, state_before); } } } @@ -299,7 +299,7 @@ } } } - return new IfOp(x, cond, y, tval, fval); + return new IfOp(x, cond, y, tval, fval, state_before); } void Optimizer::eliminate_conditional_expressions() {
--- a/src/share/vm/c1/c1_Runtime1.cpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/share/vm/c1/c1_Runtime1.cpp Thu Jan 15 20:22:42 2015 +0100 @@ -197,7 +197,6 @@ switch (id) { // These stubs don't need to have an oopmap case dtrace_object_alloc_id: - case shenandoah_write_barrier_slow_id: case g1_pre_barrier_slow_id: case g1_post_barrier_slow_id: case slow_subtype_check_id: @@ -302,7 +301,7 @@ FUNCTION_CASE(entry, TRACE_TIME_METHOD); #endif FUNCTION_CASE(entry, StubRoutines::updateBytesCRC32()); - FUNCTION_CASE(entry, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_static); + FUNCTION_CASE(entry, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_c1); #undef FUNCTION_CASE
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.cpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.cpp Thu Jan 15 20:22:42 2015 +0100 @@ -305,17 +305,26 @@ } } -IRT_LEAF(oopDesc*, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_static(oopDesc* src)) - oop result = ((ShenandoahBarrierSet*)oopDesc::bs())->resolve_and_maybe_copy_oop_work(oop(src)); - // tty->print_cr("called static write barrier with: %p result: %p copy: %d", (oopDesc*) src, (oopDesc*) result, src != result); - return (oopDesc*) result; +JRT_ENTRY(void, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_c2(oopDesc* src, JavaThread* thread)) + oop result = ((ShenandoahBarrierSet*) oopDesc::bs())->resolve_and_maybe_copy_oop_work(oop(src)); + // tty->print_cr("called C2 write barrier with: %p result: %p copy: %d", (oopDesc*) src, (oopDesc*) result, src != result); + thread->set_vm_result(result); + // eturn (oopDesc*) result; +JRT_END + +IRT_ENTRY(void, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_static2(JavaThread* thread, oopDesc* src)) + oop result = ((ShenandoahBarrierSet*)oopDesc::bs())->resolve_and_maybe_copy_oop_work2(oop(src)); + // tty->print_cr("called interpreter write barrier with: %p result: %p", src, result); + thread->set_vm_result(result); + //return (oopDesc*) result; IRT_END -IRT_LEAF(oopDesc*, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_static2(oopDesc* src)) +JRT_ENTRY(void, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_c1(JavaThread* thread, oopDesc* src)) oop result = ((ShenandoahBarrierSet*)oopDesc::bs())->resolve_and_maybe_copy_oop_work2(oop(src)); - // tty->print_cr("called write barrier with: %p result: %p", src, result); - return (oopDesc*) result; -IRT_END + // tty->print_cr("called C1 write barrier with: %p result: %p", src, result); + thread->set_vm_result(result); + //return (oopDesc*) result; +JRT_END oop ShenandoahBarrierSet::resolve_and_maybe_copy_oop(oop src) { ShenandoahHeap *sh = (ShenandoahHeap*) Universe::heap(); @@ -488,7 +497,7 @@ } } -void ShenandoahBarrierSet::compile_resolve_oop_for_write(MacroAssembler* masm, Register dst, bool explicit_null_check, int num_state_save, ...) { +void ShenandoahBarrierSet::compile_resolve_oop_for_write(MacroAssembler* masm, Register dst, bool explicit_null_check, int stack_adjust, int num_state_save, ...) { if (! ShenandoahWriteBarrier) { assert(! ShenandoahConcurrentEvacuation, "Can only do this without concurrent evacuation"); @@ -503,12 +512,13 @@ // Resolve oop first. // TODO: Make this not-null-checking as soon as we have implicit null checks in c1! - __ push(rscratch1); if (explicit_null_check) { __ testptr(dst, dst); __ jcc(Assembler::zero, done); } + __ push(rscratch1); + // Now check if evacuation is in progress. ExternalAddress evacuation_in_progress = ExternalAddress(ShenandoahHeap::evacuation_in_progress_addr()); __ movptr(rscratch1, evacuation_in_progress); @@ -516,15 +526,16 @@ compile_resolve_oop_not_null(masm, dst); __ cmpl(rscratch1, 0); + __ pop(rscratch1); __ jcc(Assembler::equal, done); - + __ push(rscratch1); __ push(rscratch2); ExternalAddress heap_address = ExternalAddress((address) Universe::heap_addr()); __ movptr(rscratch1, heap_address); // Compute index into regions array. __ movq(rscratch2, dst); - __ andq(rscratch2, ~(ShenandoahHeapRegion::RegionSizeBytes - 1)); + // __ andq(rscratch2, ~(ShenandoahHeapRegion::RegionSizeBytes - 1)); Address first_region_bottom_addr = Address(rscratch1, ShenandoahHeap::first_region_bottom_offset()); __ subq(rscratch2, first_region_bottom_addr); __ shrq(rscratch2, ShenandoahHeapRegion::RegionSizeShift); @@ -540,6 +551,7 @@ __ testb(rscratch1, 0x1); __ pop(rscratch2); + __ pop(rscratch1); __ jcc(Assembler::zero, done); @@ -547,12 +559,18 @@ va_list vl; va_start(vl, num_state_save); for (int i = 0; i < num_state_save; i++) { - save_states.at_put(i, va_arg(vl, int /* SaveState */)); + save_states.at_put(i, va_arg(vl, int)); } va_end(vl); + if (stack_adjust != 0) { + __ addptr(rsp, stack_adjust * Interpreter::stackElementSize); + } for (int i = 0; i < num_state_save; i++) { switch (save_states[i]) { + case noreg: + __ subptr(rsp, Interpreter::stackElementSize); + break; case ss_rax: __ push(rax); break; @@ -616,9 +634,9 @@ if (dst != rsi) { __ push(rsi); } - if (dst != rbp) { - __ push(rbp); - } + //if (dst != rbp) { + // __ push(rbp); + // } if (dst != r8) { __ push(r8); } @@ -666,13 +684,13 @@ } } - - __ mov(c_rarg1, dst); - __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_static2), c_rarg1); - __ mov(rscratch1, rax); + __ call_VM(dst, CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_static2), dst, false); for (int i = num_state_save - 1; i >= 0; i--) { switch (save_states[i]) { + case noreg: + __ addptr(rsp, Interpreter::stackElementSize); + break; case ss_rax: __ pop(rax); break; @@ -757,9 +775,9 @@ if (dst != r8) { __ pop(r8); } - if (dst != rbp) { - __ pop(rbp); - } + // if (dst != rbp) { + // __ pop(rbp); + // } if (dst != rsi) { __ pop(rsi); } @@ -784,10 +802,10 @@ } } - __ mov(dst, rscratch1); - + if (stack_adjust != 0) { + __ subptr(rsp, stack_adjust * Interpreter::stackElementSize); + } __ bind(done); - __ pop(rscratch1); } /*
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.hpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.hpp Thu Jan 15 20:22:42 2015 +0100 @@ -132,8 +132,10 @@ oop resolve_and_maybe_copy_oop_work(oop src); oop resolve_and_maybe_copy_oop_work2(oop src); virtual oop resolve_and_maybe_copy_oop(oop src); - static oopDesc* resolve_and_maybe_copy_oop_static(oopDesc* src); - static oopDesc* resolve_and_maybe_copy_oop_static2(oopDesc* src); + + static void resolve_and_maybe_copy_oop_c2(oopDesc* src, JavaThread* thread); + static void resolve_and_maybe_copy_oop_static2(JavaThread* thread, oopDesc* src); + static void resolve_and_maybe_copy_oop_c1(JavaThread* thread, oopDesc* src); private: bool need_update_refs_barrier(); @@ -143,7 +145,7 @@ // TODO: The following should really live in an X86 specific subclass. virtual void compile_resolve_oop(MacroAssembler* masm, Register dst); virtual void compile_resolve_oop_not_null(MacroAssembler* masm, Register dst); - void compile_resolve_oop_for_write(MacroAssembler* masm, Register dst, bool explicit_null_check, int num_save_state = 0, ...); + void compile_resolve_oop_for_write(MacroAssembler* masm, Register dst, bool explicit_null_check, int stack_adjust, int num_save_state, ...); private: void compile_resolve_oop_runtime(MacroAssembler* masm, Register dst);
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.cpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.cpp Thu Jan 15 20:22:42 2015 +0100 @@ -418,22 +418,34 @@ HeapWord* ShenandoahHeap::allocate_memory(size_t word_size, bool evacuation) { HeapWord* result = NULL; - { - MutexLockerEx ml(ShenandoahHeap_lock, true); - result = allocate_memory_work(word_size, evacuation); - } + result = allocate_memory_with_lock(word_size, evacuation); if (result == NULL && ! evacuation) { // Allocation failed, try full-GC, then retry allocation. collect(GCCause::_allocation_failure); - { - MutexLockerEx ml(ShenandoahHeap_lock, true); - result = allocate_memory_work(word_size, evacuation); - } + result = allocate_memory_with_lock(word_size, evacuation); } return result; } +HeapWord* ShenandoahHeap::allocate_memory_with_lock(size_t word_size, bool evacuation) { + if (Thread::current()->is_GC_task_thread() && SafepointSynchronize::is_at_safepoint()) { + return allocate_memory_shenandoah_lock(word_size, evacuation); + } else { + return allocate_memory_heap_lock(word_size, evacuation); + } +} + +HeapWord* ShenandoahHeap::allocate_memory_heap_lock(size_t word_size, bool evacuation) { + MutexLocker ml(Heap_lock); + return allocate_memory_work(word_size, evacuation); +} + +HeapWord* ShenandoahHeap::allocate_memory_shenandoah_lock(size_t word_size, bool evacuation) { + MutexLocker ml(ShenandoahHeap_lock); + return allocate_memory_work(word_size, evacuation); +} + ShenandoahHeapRegion* ShenandoahHeap::check_skip_humonguous(ShenandoahHeapRegion* region, bool evacuation) { while (region != NULL && region->is_humonguous()) { region = _free_regions->get_next(evacuation); @@ -655,8 +667,8 @@ return result; } else { tty->print_cr("Out of memory. Requested number of words: %d used heap: %ld, bytes allocated since last CM: %ld", size, used(), _bytesAllocSinceCM); - MutexLockerEx ml(ShenandoahHeap_lock, true); { + MutexLocker ml(Heap_lock); print_heap_regions(); tty->print("Printing %d free regions:\n", _free_regions->length()); _free_regions->print();
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.hpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.hpp Thu Jan 15 20:22:42 2015 +0100 @@ -371,6 +371,9 @@ void verify_live(); void verify_liveness_after_concurrent_mark(); + HeapWord* allocate_memory_with_lock(size_t word_size, bool evacuation); + HeapWord* allocate_memory_heap_lock(size_t word_size, bool evacuation); + HeapWord* allocate_memory_shenandoah_lock(size_t word_size, bool evacuation); HeapWord* allocate_memory_work(size_t word_size, bool evacuation); HeapWord* allocate_large_memory(size_t word_size); ShenandoahHeapRegion* check_skip_humonguous(ShenandoahHeapRegion* region, bool evacuation);
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahMarkCompact.cpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/share/vm/gc_implementation/shenandoah/shenandoahMarkCompact.cpp Thu Jan 15 20:22:42 2015 +0100 @@ -84,8 +84,8 @@ // TODO: Move prepare_unmarked_root_objs() into SCM! // TODO: Make this whole sequence a separate method in SCM! - _heap->concurrentMark()->prepare_unmarked_root_objs_no_derived_ptrs(true /* update references */); - _heap->concurrentMark()->mark_from_roots(true /* update-refs */, true /* full-gc */); + _heap->concurrentMark()->prepare_unmarked_root_objs_no_derived_ptrs(false /* update references */); + _heap->concurrentMark()->mark_from_roots(false /* update-refs */, true /* full-gc */); _heap->concurrentMark()->finish_mark_from_roots(true); }
--- a/src/share/vm/memory/barrierSet.hpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/share/vm/memory/barrierSet.hpp Thu Jan 15 20:22:42 2015 +0100 @@ -54,6 +54,7 @@ }; enum SaveState { + noreg, ss_rax, ss_rbx, ss_rcx, @@ -224,7 +225,7 @@ // Default implementation does nothing. } - virtual void compile_resolve_oop_for_write(MacroAssembler* masm, Register dst, bool explicit_null_check, int num_save_state = 0, ...) { + virtual void compile_resolve_oop_for_write(MacroAssembler* masm, Register dst, bool explicit_null_check, int stack_adjust, int num_save_state, ...) { // Default implementation does nothing. } };
--- a/src/share/vm/opto/escape.cpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/share/vm/opto/escape.cpp Thu Jan 15 20:22:42 2015 +0100 @@ -825,7 +825,7 @@ ciMethod* meth = call->as_CallJava()->method(); if (meth == NULL) { const char* name = call->as_CallStaticJava()->_name; - assert(strncmp(name, "_multianewarray", 15) == 0, "TODO: add failed case check"); + assert(strncmp(name, "_multianewarray", 15) == 0 || strncmp(name, "shenandoah_write_barrier", 24) == 0, "TODO: add failed case check"); // Returns a newly allocated unescaped object. add_java_object(call, PointsToNode::NoEscape); ptnode_adr(call_idx)->set_scalar_replaceable(false); @@ -943,7 +943,6 @@ (strcmp(call->as_CallLeaf()->_name, "g1_wb_pre") == 0 || strcmp(call->as_CallLeaf()->_name, "g1_wb_post") == 0 || strcmp(call->as_CallLeaf()->_name, "shenandoah_clone_barrier") == 0 || - strcmp(call->as_CallLeaf()->_name, "shenandoah_write_barrier") == 0 || strcmp(call->as_CallLeaf()->_name, "shenandoah_read_barrier") == 0 || strcmp(call->as_CallLeaf()->_name, "shenandoah_cas_obj") == 0 || strcmp(call->as_CallLeaf()->_name, "updateBytesCRC32") == 0 || @@ -1463,7 +1462,7 @@ #ifdef ASSERT if (alloc->as_CallStaticJava()->method() == NULL) { const char* name = alloc->as_CallStaticJava()->_name; - assert(strncmp(name, "_multianewarray", 15) == 0, "sanity"); + assert(strncmp(name, "_multianewarray", 15) == 0 || strncmp(name, "shenandoah_write_barrier", 24) == 0, "sanity"); } #endif // Non-escaped allocation returned from Java or runtime call have
--- a/src/share/vm/opto/graphKit.cpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/share/vm/opto/graphKit.cpp Thu Jan 15 20:22:42 2015 +0100 @@ -813,7 +813,7 @@ // Helper function for enforcing certain bytecodes to reexecute if // deoptimization happens static bool should_reexecute_implied_by_bytecode(JVMState *jvms, bool is_anewarray) { - ciMethod* cur_method = jvms->method(); + ciMethod* cur_method = jvms->has_method() ? jvms->method() : NULL; int cur_bci = jvms->bci(); if (cur_method != NULL && cur_bci != InvocationEntryBci) { Bytecodes::Code code = cur_method->java_code_at_bci(cur_bci); @@ -3231,8 +3231,6 @@ return; } - obj = shenandoah_write_barrier(obj); - // Memory barrier to avoid floating things down past the locked region insert_mem_bar(Op_MemBarReleaseLock); @@ -4299,9 +4297,10 @@ // Evacuation path. set_control(iffalse); - Node *call = make_runtime_call(RC_LEAF | RC_NO_IO, + kill_dead_locals(); + Node *call = make_runtime_call(RC_NO_LEAF | RC_NO_IO, OptoRuntime::shenandoah_barrier_Type(obj_type), - CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_static), + OptoRuntime::shenandoah_write_barrier_Java(), "shenandoah_write_barrier", obj_type->is_ptr()->add_offset(-8), obj);
--- a/src/share/vm/opto/output.cpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/share/vm/opto/output.cpp Thu Jan 15 20:22:42 2015 +0100 @@ -905,8 +905,8 @@ !method->is_synchronized() || method->is_native() || num_mon > 0 || - !GenerateSynchronizationCode, - "monitors must always exist for synchronized methods"); + !GenerateSynchronizationCode || (UseShenandoahGC && jvms->bci() < 0), + err_msg("monitors must always exist for synchronized methods, bci: %d", jvms->bci())); // Build the growable array of ScopeValues for exp stack GrowableArray<MonitorValue*> *monarray = new GrowableArray<MonitorValue*>(num_mon);
--- a/src/share/vm/opto/runtime.cpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/share/vm/opto/runtime.cpp Thu Jan 15 20:22:42 2015 +0100 @@ -36,6 +36,7 @@ #include "compiler/oopMap.hpp" #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" #include "gc_implementation/g1/heapRegion.hpp" +#include "gc_implementation/shenandoah/shenandoahBarrierSet.hpp" #include "gc_interface/collectedHeap.hpp" #include "interpreter/bytecode.hpp" #include "interpreter/interpreter.hpp" @@ -111,6 +112,7 @@ address OptoRuntime::_multianewarray4_Java = NULL; address OptoRuntime::_multianewarray5_Java = NULL; address OptoRuntime::_multianewarrayN_Java = NULL; +address OptoRuntime::_shenandoah_write_barrier_Java = NULL; address OptoRuntime::_g1_wb_pre_Java = NULL; address OptoRuntime::_g1_wb_post_Java = NULL; address OptoRuntime::_vtable_must_compile_Java = NULL; @@ -160,6 +162,7 @@ gen(env, _multianewarray4_Java , multianewarray4_Type , multianewarray4_C , 0 , true , false, false); gen(env, _multianewarray5_Java , multianewarray5_Type , multianewarray5_C , 0 , true , false, false); gen(env, _multianewarrayN_Java , multianewarrayN_Type , multianewarrayN_C , 0 , true , false, false); + gen(env, _shenandoah_write_barrier_Java , shenandoah_write_barrier_Type, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_c2, 0, true, false, false); gen(env, _g1_wb_pre_Java , g1_wb_pre_Type , SharedRuntime::g1_wb_pre , 0 , false, false, false); gen(env, _g1_wb_post_Java , g1_wb_post_Type , SharedRuntime::g1_wb_post , 0 , false, false, false); gen(env, _complete_monitor_locking_Java , complete_monitor_enter_Type , SharedRuntime::complete_monitor_locking_C, 0, false, false, false); @@ -1386,6 +1389,20 @@ return TypeFunc::make(domain, range); } +const TypeFunc* OptoRuntime::shenandoah_write_barrier_Type() { + // create input type (domain) + const Type** fields = TypeTuple::fields(1); + fields[TypeFunc::Parms+0] = TypeOopPtr::NOTNULL; + + const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+1, fields); + + // result type needed + fields = TypeTuple::fields(1); + fields[TypeFunc::Parms+0] = TypeOopPtr::NOTNULL; + const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields); + return TypeFunc::make(domain, range); +} + JRT_ENTRY_NO_ASYNC(void, OptoRuntime::register_finalizer(oopDesc* obj, JavaThread* thread)) assert(obj->is_oop(), "must be a valid oop"); assert(obj->klass()->has_finalizer(), "shouldn't be here otherwise");
--- a/src/share/vm/opto/runtime.hpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/share/vm/opto/runtime.hpp Thu Jan 15 20:22:42 2015 +0100 @@ -134,6 +134,7 @@ static address _multianewarray4_Java; static address _multianewarray5_Java; static address _multianewarrayN_Java; + static address _shenandoah_write_barrier_Java; static address _g1_wb_pre_Java; static address _g1_wb_post_Java; static address _vtable_must_compile_Java; @@ -234,6 +235,7 @@ static address multianewarray4_Java() { return _multianewarray4_Java; } static address multianewarray5_Java() { return _multianewarray5_Java; } static address multianewarrayN_Java() { return _multianewarrayN_Java; } + static address shenandoah_write_barrier_Java() { return _shenandoah_write_barrier_Java; } static address g1_wb_pre_Java() { return _g1_wb_pre_Java; } static address g1_wb_post_Java() { return _g1_wb_post_Java; } static address vtable_must_compile_stub() { return _vtable_must_compile_Java; } @@ -331,6 +333,7 @@ // Shenandoah support static const TypeFunc* shenandoah_barrier_Type(const Type* type); + static const TypeFunc* shenandoah_write_barrier_Type(); # ifdef ENABLE_ZAP_DEAD_LOCALS static const TypeFunc* zap_dead_locals_Type();
--- a/src/share/vm/runtime/mutexLocker.cpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/share/vm/runtime/mutexLocker.cpp Thu Jan 15 20:22:42 2015 +0100 @@ -210,7 +210,6 @@ def(SATB_Q_FL_lock , Mutex , special, true ); def(SATB_Q_CBL_mon , Monitor, nonleaf, true ); def(Shared_SATB_Q_lock , Mutex, nonleaf, true ); - def(ShenandoahHeap_lock , Monitor, special, false); def(ShenandoahFullGC_lock , Monitor, leaf, true); def(ShenandoahJNICritical_lock , Monitor, nonleaf+1, true); def(ShenandoahMemProtect_lock , Monitor, native, false); @@ -269,6 +268,7 @@ // used in CMS GC for locking PLL lock } def(Heap_lock , Monitor, nonleaf+1, false); + def(ShenandoahHeap_lock , Monitor, nonleaf+1, false); def(JfieldIdCreation_lock , Mutex , nonleaf+1, true ); // jfieldID, Used in VM_Operation def(MemberNameTable_lock , Mutex , nonleaf+1, false); // Used to protect MemberNameTable
--- a/src/share/vm/runtime/mutexLocker.hpp Mon Jan 12 18:14:19 2015 +0100 +++ b/src/share/vm/runtime/mutexLocker.hpp Thu Jan 15 20:22:42 2015 +0100 @@ -60,7 +60,7 @@ extern Mutex* JvmtiThreadState_lock; // a lock on modification of JVMTI thread data extern Monitor* JvmtiPendingEvent_lock; // a lock on the JVMTI pending events list extern Monitor* Heap_lock; // a lock on the heap -extern Monitor* ShenandoahHeap_lock; // a lock on the heap, used by ShenandoahGC when growing the heap +extern Monitor* ShenandoahHeap_lock; // a lock on the heap, used by ShenandoahGC when evacuating at a safepoint extern Monitor* ShenandoahFullGC_lock; // a monitor to wait/notify the Shenandoah background thread on full-GC requests extern Monitor* ShenandoahJNICritical_lock; // a monitor to wait/notify the Shenandoah background thread on full-GC requests extern Mutex* ExpandHeap_lock; // a lock on expanding the heap