# HG changeset patch # User Roman Kennke # Date 1422007638 -3600 # Node ID c782867202052b2804420011698a82526815b0f0 # Parent 14cbd68c1a64dca7a742545e177f8c1d0ea6d1ad Check for object in collection set before calling into the runtime write barrier. diff -r 14cbd68c1a64 -r c78286720205 src/cpu/x86/vm/c1_LIRAssembler_x86.cpp --- a/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp Fri Jan 23 09:44:58 2015 +0100 +++ b/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp Fri Jan 23 11:07:18 2015 +0100 @@ -1522,8 +1522,9 @@ Label done; Register obj = op->in_opr()->as_register(); Register res = op->result_opr()->as_register(); - Register tmp1 = op->tmp_opr()->as_register(); - assert_different_registers(res, tmp1); + Register tmp1 = op->tmp1_opr()->as_register(); + Register tmp2 = op->tmp2_opr()->as_register(); + assert_different_registers(res, tmp1, tmp2); if (res != obj) { __ mov(res, obj); @@ -1544,6 +1545,14 @@ __ cmpl(tmp1, 0); __ jcc(Assembler::equal, done); + // Check for object in collection set. + __ movptr(tmp1, res); + __ shrptr(tmp1, ShenandoahHeapRegion::RegionSizeShift); + __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr()); + __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1)); + __ testb(tmp2, 0x1); + __ jcc(Assembler::zero, done); + if (res != rax) { __ xchgptr(res, rax); // Move obj into rax and save rax into obj. } diff -r 14cbd68c1a64 -r c78286720205 src/cpu/x86/vm/c1_Runtime1_x86.cpp --- a/src/cpu/x86/vm/c1_Runtime1_x86.cpp Fri Jan 23 09:44:58 2015 +0100 +++ b/src/cpu/x86/vm/c1_Runtime1_x86.cpp Fri Jan 23 11:07:18 2015 +0100 @@ -1625,35 +1625,6 @@ case shenandoah_write_barrier_slow_id: { StubFrame f(sasm, "shenandoah_write_barrier", dont_gc_arguments); - Label done; - - __ push(rscratch1); - __ push(rscratch2); - - ExternalAddress heap_address = ExternalAddress((address) Universe::heap_addr()); - __ movptr(rscratch1, heap_address); - // Compute index into regions array. - __ movq(rscratch2, rax); - __ andq(rscratch2, ~(ShenandoahHeapRegion::RegionSizeBytes - 1)); - Address first_region_bottom_addr = Address(rscratch1, ShenandoahHeap::first_region_bottom_offset()); - __ subq(rscratch2, first_region_bottom_addr); - __ shrq(rscratch2, ShenandoahHeapRegion::RegionSizeShift); - - Address regions_address = Address(rscratch1, ShenandoahHeap::ordered_regions_offset()); - __ movptr(rscratch1, regions_address); - - Address heap_region_containing_addr = Address(rscratch1, rscratch2, Address::times_ptr); - __ movptr(rscratch1, heap_region_containing_addr); - - Address is_in_coll_set_addr = Address(rscratch1, ShenandoahHeapRegion::is_in_collection_set_offset()); - __ movb(rscratch1, is_in_coll_set_addr); - __ testb(rscratch1, 0x1); - - __ pop(rscratch2); - __ pop(rscratch1); - - __ jcc(Assembler::zero, done); - OopMap* map = save_live_registers(sasm, 2); int call_offset = __ call_RT(rax, noreg, CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_c1), rax); @@ -1661,7 +1632,6 @@ oop_maps->add_gc_map(call_offset, map); restore_live_registers_except_rax(sasm); __ verify_oop(rax); - __ bind(done); } break; diff -r 14cbd68c1a64 -r c78286720205 src/share/vm/c1/c1_LIR.cpp --- a/src/share/vm/c1/c1_LIR.cpp Fri Jan 23 09:44:58 2015 +0100 +++ b/src/share/vm/c1/c1_LIR.cpp Fri Jan 23 11:07:18 2015 +0100 @@ -1017,7 +1017,8 @@ LIR_OpShenandoahWriteBarrier* opShenandoahWB = (LIR_OpShenandoahWriteBarrier*) op; do_input(opShenandoahWB->_opr); do_output(opShenandoahWB->_result); - do_temp(opShenandoahWB->_tmp); + do_temp(opShenandoahWB->_tmp1); + do_temp(opShenandoahWB->_tmp2); do_info(opShenandoahWB->_info); break; } @@ -1847,7 +1848,8 @@ void LIR_OpShenandoahWriteBarrier::print_instr(outputStream* out) const { out->print("[obj: "); in_opr()->print(out); out->print("]"); out->print("[res: "); result_opr()->print(out); out->print("]"); - out->print("[tmp: "); tmp_opr()->print(out); out->print("]"); + out->print("[tmp1: "); tmp1_opr()->print(out); out->print("]"); + out->print("[tmp2: "); tmp2_opr()->print(out); out->print("]"); } // LIR_OpJavaCall diff -r 14cbd68c1a64 -r c78286720205 src/share/vm/c1/c1_LIR.hpp --- a/src/share/vm/c1/c1_LIR.hpp Fri Jan 23 09:44:58 2015 +0100 +++ b/src/share/vm/c1/c1_LIR.hpp Fri Jan 23 11:07:18 2015 +0100 @@ -1473,12 +1473,14 @@ private: bool _need_null_check; - LIR_Opr _tmp; + LIR_Opr _tmp1; + LIR_Opr _tmp2; public: - LIR_OpShenandoahWriteBarrier(LIR_Opr obj, LIR_Opr result, LIR_Opr tmp, CodeEmitInfo* info, bool need_null_check) : LIR_Op1(lir_shenandoah_wb, obj, result, T_OBJECT, lir_patch_none, info), _tmp(tmp), _need_null_check(need_null_check) { + LIR_OpShenandoahWriteBarrier(LIR_Opr obj, LIR_Opr result, LIR_Opr tmp1, LIR_Opr tmp2, CodeEmitInfo* info, bool need_null_check) : LIR_Op1(lir_shenandoah_wb, obj, result, T_OBJECT, lir_patch_none, info), _tmp1(tmp1), _tmp2(tmp2), _need_null_check(need_null_check) { } - LIR_Opr tmp_opr() const { return _tmp; } + LIR_Opr tmp1_opr() const { return _tmp1; } + LIR_Opr tmp2_opr() const { return _tmp2; } bool need_null_check() const { return _need_null_check; } virtual void emit_code(LIR_Assembler* masm); virtual LIR_OpShenandoahWriteBarrier* as_OpShenandoahWriteBarrier() { return this; } @@ -2166,7 +2168,7 @@ #endif void convert(Bytecodes::Code code, LIR_Opr left, LIR_Opr dst, ConversionStub* stub = NULL/*, bool is_32bit = false*/) { append(new LIR_OpConvert(code, left, dst, stub)); } - void shenandoah_wb(LIR_Opr obj, LIR_Opr result, LIR_Opr tmp, CodeEmitInfo* info, bool need_null_check) { append(new LIR_OpShenandoahWriteBarrier(obj, result, tmp, info, need_null_check)); } + void shenandoah_wb(LIR_Opr obj, LIR_Opr result, LIR_Opr tmp1, LIR_Opr tmp2, CodeEmitInfo* info, bool need_null_check) { append(new LIR_OpShenandoahWriteBarrier(obj, result, tmp1, tmp2, info, need_null_check)); } void logical_and (LIR_Opr left, LIR_Opr right, LIR_Opr dst) { append(new LIR_Op2(lir_logic_and, left, right, dst)); } void logical_or (LIR_Opr left, LIR_Opr right, LIR_Opr dst) { append(new LIR_Op2(lir_logic_or, left, right, dst)); } diff -r 14cbd68c1a64 -r c78286720205 src/share/vm/c1/c1_LIRGenerator.cpp --- a/src/share/vm/c1/c1_LIRGenerator.cpp Fri Jan 23 09:44:58 2015 +0100 +++ b/src/share/vm/c1/c1_LIRGenerator.cpp Fri Jan 23 11:07:18 2015 +0100 @@ -1882,8 +1882,9 @@ if (UseShenandoahGC) { LIR_Opr result = new_register(T_OBJECT); - LIR_Opr tmp = new_register(T_INT); - __ shenandoah_wb(obj, result, tmp, new CodeEmitInfo(info), need_null_check); + LIR_Opr tmp1 = new_register(T_INT); + LIR_Opr tmp2 = new_register(T_INT); + __ shenandoah_wb(obj, result, tmp1, tmp2, new CodeEmitInfo(info), need_null_check); return result; } else { diff -r 14cbd68c1a64 -r c78286720205 src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.cpp --- a/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.cpp Fri Jan 23 09:44:58 2015 +0100 +++ b/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.cpp Fri Jan 23 11:07:18 2015 +0100 @@ -311,13 +311,13 @@ } JRT_ENTRY(void, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_c2(oopDesc* src, JavaThread* thread)) - oop result = ((ShenandoahBarrierSet*) oopDesc::bs())->resolve_and_maybe_copy_oop_work(oop(src)); + oop result = ((ShenandoahBarrierSet*) oopDesc::bs())->resolve_and_maybe_copy_oop_work2(oop(src)); // tty->print_cr("called C2 write barrier with: %p result: %p copy: %d", (oopDesc*) src, (oopDesc*) result, src != result); thread->set_vm_result(result); // eturn (oopDesc*) result; JRT_END -IRT_ENTRY(void, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_static2(JavaThread* thread, oopDesc* src)) +IRT_ENTRY(void, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_interp(JavaThread* thread, oopDesc* src)) oop result = ((ShenandoahBarrierSet*)oopDesc::bs())->resolve_and_maybe_copy_oop_work2(oop(src)); // tty->print_cr("called interpreter write barrier with: %p result: %p", src, result); thread->set_vm_result(result); @@ -536,24 +536,11 @@ __ push(rscratch1); __ push(rscratch2); - ExternalAddress heap_address = ExternalAddress((address) Universe::heap_addr()); - __ movptr(rscratch1, heap_address); - // Compute index into regions array. - __ movq(rscratch2, dst); - // __ andq(rscratch2, ~(ShenandoahHeapRegion::RegionSizeBytes - 1)); - Address first_region_bottom_addr = Address(rscratch1, ShenandoahHeap::first_region_bottom_offset()); - __ subq(rscratch2, first_region_bottom_addr); - __ shrq(rscratch2, ShenandoahHeapRegion::RegionSizeShift); - - Address regions_address = Address(rscratch1, ShenandoahHeap::ordered_regions_offset()); - __ movptr(rscratch1, regions_address); - - Address heap_region_containing_addr = Address(rscratch1, rscratch2, Address::times_ptr); - __ movptr(rscratch1, heap_region_containing_addr); - - Address is_in_coll_set_addr = Address(rscratch1, ShenandoahHeapRegion::is_in_collection_set_offset()); - __ movb(rscratch1, is_in_coll_set_addr); - __ testb(rscratch1, 0x1); + __ movptr(rscratch1, dst); + __ shrptr(rscratch1, ShenandoahHeapRegion::RegionSizeShift); + __ movptr(rscratch2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr()); + __ movbool(rscratch2, Address(rscratch2, rscratch1, Address::times_1)); + __ testb(rscratch2, 0x1); __ pop(rscratch2); __ pop(rscratch1); @@ -626,7 +613,7 @@ } } - __ call_VM(dst, CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_static2), dst, false); + __ call_VM(dst, CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_interp), dst, false); for (int i = num_state_save - 1; i >= 0; i--) { switch (save_states[i]) { diff -r 14cbd68c1a64 -r c78286720205 src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.hpp --- a/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.hpp Fri Jan 23 09:44:58 2015 +0100 +++ b/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.hpp Fri Jan 23 11:07:18 2015 +0100 @@ -134,7 +134,7 @@ virtual oop resolve_and_maybe_copy_oop(oop src); static void resolve_and_maybe_copy_oop_c2(oopDesc* src, JavaThread* thread); - static void resolve_and_maybe_copy_oop_static2(JavaThread* thread, oopDesc* src); + static void resolve_and_maybe_copy_oop_interp(JavaThread* thread, oopDesc* src); static void resolve_and_maybe_copy_oop_c1(JavaThread* thread, oopDesc* src); private: diff -r 14cbd68c1a64 -r c78286720205 src/share/vm/opto/graphKit.cpp --- a/src/share/vm/opto/graphKit.cpp Fri Jan 23 09:44:58 2015 +0100 +++ b/src/share/vm/opto/graphKit.cpp Fri Jan 23 11:07:18 2015 +0100 @@ -4282,12 +4282,12 @@ Node* oldmem = map()->memory(); // Make the merge point. - enum { _evac_path = 1, _no_evac_path, PATH_LIMIT }; + enum { _evac_path = 1, _no_cset_path, _no_evac_path, PATH_LIMIT }; RegionNode* region = new(C) RegionNode(PATH_LIMIT); Node* phi = new(C) PhiNode(region, obj_type); // Make the actual if-branch. - IfNode* iff = create_and_map_if(control(), test, PROB_LIKELY_MAG(3), COUNT_UNKNOWN); + IfNode* iff = create_and_map_if(control(), test, PROB_LIKELY_MAG(1), COUNT_UNKNOWN); Node* iftrue = _gvn.transform(new (C) IfTrueNode(iff)); Node* iffalse = _gvn.transform(new (C) IfFalseNode(iff)); @@ -4297,6 +4297,28 @@ // Evacuation path. set_control(iffalse); + + // Check for in-cset. + Node* in_cset_fast_test_addr = makecon(TypeRawPtr::make(ShenandoahHeap::in_cset_fast_test_addr())); + Node* region_size_shift = intcon(ShenandoahHeapRegion::RegionSizeShift); + Node* objX = _gvn.transform(new (C) CastP2XNode(control(), obj)); + Node* in_cset_fast_test_idx = _gvn.transform(new (C) URShiftXNode(objX, region_size_shift)); + Node* test_addr = basic_plus_adr(top(), in_cset_fast_test_addr, in_cset_fast_test_idx); + Node* in_cset_test = make_load(control(), test_addr, TypeInt::BOOL, T_BOOLEAN, Compile::AliasIdxRaw, MemNode::unordered, false); + Node* chk2 = _gvn.transform(new (C) CmpINode(in_cset_test, intcon(0))); + Node* test2 = _gvn.transform(new (C) BoolNode(chk2, BoolTest::eq)); + + // The if-branch for in-cset-test. + IfNode* iff2 = create_and_map_if(control(), test2, PROB_LIKELY_MAG(1), COUNT_UNKNOWN); + Node* iftrue2 = _gvn.transform(new (C) IfTrueNode(iff2)); + Node* iffalse2 = _gvn.transform(new (C) IfFalseNode(iff2)); + + region->init_req(_no_cset_path, iftrue2); + phi->init_req(_no_cset_path, obj); + + // Evacuation path. + set_control(iffalse2); + kill_dead_locals(); Node *call = make_runtime_call(RC_NO_LEAF | RC_NO_IO, OptoRuntime::shenandoah_barrier_Type(obj_type), @@ -4314,6 +4336,7 @@ record_for_igvn(region); phi = _gvn.transform(phi); + merge_memory(oldmem, region, _no_cset_path); merge_memory(oldmem, region, _no_evac_path); return phi;