Mercurial > hg > release > icedtea7-forest-2.2 > hotspot
changeset 3187:1dc233a8c7fe
7121140: Allocation paths require explicit memory synchronization operations for RMO systems
Summary: adds store store barrier after initialization of header and body of objects.
Reviewed-by: never, kvn
| author | roland |
|---|---|
| date | Tue, 20 Dec 2011 16:56:50 +0100 |
| parents | 069ab3f976d3 |
| children | e5ac210043cd |
| files | src/cpu/sparc/vm/sparc.ad src/cpu/x86/vm/x86_32.ad src/cpu/x86/vm/x86_64.ad src/share/vm/adlc/formssel.cpp src/share/vm/opto/callnode.hpp src/share/vm/opto/classes.hpp src/share/vm/opto/escape.cpp src/share/vm/opto/graphKit.cpp src/share/vm/opto/library_call.cpp src/share/vm/opto/macro.cpp src/share/vm/opto/memnode.cpp src/share/vm/opto/memnode.hpp src/share/vm/opto/node.hpp |
| diffstat | 13 files changed, 196 insertions(+), 7 deletions(-) [+] |
line wrap: on
line diff
--- a/src/cpu/sparc/vm/sparc.ad Wed Dec 07 11:35:03 2011 +0100 +++ b/src/cpu/sparc/vm/sparc.ad Tue Dec 20 16:56:50 2011 +0100 @@ -6773,6 +6773,16 @@ ins_pipe(empty); %} +instruct membar_storestore() %{ + match(MemBarStoreStore); + ins_cost(0); + + size(0); + format %{ "!MEMBAR-storestore (empty encoding)" %} + ins_encode( ); + ins_pipe(empty); +%} + //----------Register Move Instructions----------------------------------------- instruct roundDouble_nop(regD dst) %{ match(Set dst (RoundDouble dst));
--- a/src/cpu/x86/vm/x86_32.ad Wed Dec 07 11:35:03 2011 +0100 +++ b/src/cpu/x86/vm/x86_32.ad Tue Dec 20 16:56:50 2011 +0100 @@ -7368,6 +7368,16 @@ ins_pipe(empty); %} +instruct membar_storestore() %{ + match(MemBarStoreStore); + ins_cost(0); + + size(0); + format %{ "MEMBAR-storestore (empty encoding)" %} + ins_encode( ); + ins_pipe(empty); +%} + //----------Move Instructions-------------------------------------------------- instruct castX2P(eAXRegP dst, eAXRegI src) %{ match(Set dst (CastX2P src));
--- a/src/cpu/x86/vm/x86_64.ad Wed Dec 07 11:35:03 2011 +0100 +++ b/src/cpu/x86/vm/x86_64.ad Tue Dec 20 16:56:50 2011 +0100 @@ -6810,6 +6810,16 @@ ins_pipe(empty); %} +instruct membar_storestore() %{ + match(MemBarStoreStore); + ins_cost(0); + + size(0); + format %{ "MEMBAR-storestore (empty encoding)" %} + ins_encode( ); + ins_pipe(empty); +%} + //----------Move Instructions-------------------------------------------------- instruct castX2P(rRegP dst, rRegL src)
--- a/src/share/vm/adlc/formssel.cpp Wed Dec 07 11:35:03 2011 +0100 +++ b/src/share/vm/adlc/formssel.cpp Tue Dec 20 16:56:50 2011 +0100 @@ -627,6 +627,7 @@ if( strcmp(_matrule->_opType,"MemBarAcquire") == 0 ) return true; if( strcmp(_matrule->_opType,"MemBarReleaseLock") == 0 ) return true; if( strcmp(_matrule->_opType,"MemBarAcquireLock") == 0 ) return true; + if( strcmp(_matrule->_opType,"MemBarStoreStore") == 0 ) return true; return false; } @@ -3978,7 +3979,8 @@ !strcmp(_opType,"MemBarAcquireLock") || !strcmp(_opType,"MemBarReleaseLock") || !strcmp(_opType,"MemBarVolatile" ) || - !strcmp(_opType,"MemBarCPUOrder" ) ; + !strcmp(_opType,"MemBarCPUOrder" ) || + !strcmp(_opType,"MemBarStoreStore" ); } bool MatchRule::is_ideal_loadPC() const {
--- a/src/share/vm/opto/callnode.hpp Wed Dec 07 11:35:03 2011 +0100 +++ b/src/share/vm/opto/callnode.hpp Tue Dec 20 16:56:50 2011 +0100 @@ -791,6 +791,10 @@ // are defined in graphKit.cpp, which sets up the bidirectional relation.) InitializeNode* initialization(); + // Return the corresponding storestore barrier (or null if none). + // Walks out edges to find it... + MemBarStoreStoreNode* storestore(); + // Convenience for initialization->maybe_set_complete(phase) bool maybe_set_complete(PhaseGVN* phase); };
--- a/src/share/vm/opto/classes.hpp Wed Dec 07 11:35:03 2011 +0100 +++ b/src/share/vm/opto/classes.hpp Tue Dec 20 16:56:50 2011 +0100 @@ -166,6 +166,7 @@ macro(MemBarRelease) macro(MemBarReleaseLock) macro(MemBarVolatile) +macro(MemBarStoreStore) macro(MergeMem) macro(MinI) macro(ModD)
--- a/src/share/vm/opto/escape.cpp Wed Dec 07 11:35:03 2011 +0100 +++ b/src/share/vm/opto/escape.cpp Tue Dec 20 16:56:50 2011 +0100 @@ -1595,6 +1595,7 @@ GrowableArray<Node*> alloc_worklist; GrowableArray<Node*> addp_worklist; GrowableArray<Node*> ptr_cmp_worklist; + GrowableArray<Node*> storestore_worklist; PhaseGVN* igvn = _igvn; // Push all useful nodes onto CG list and set their type. @@ -1618,6 +1619,11 @@ (n->Opcode() == Op_CmpP || n->Opcode() == Op_CmpN)) { // Compare pointers nodes ptr_cmp_worklist.append(n); + } else if (n->is_MemBarStoreStore()) { + // Collect all MemBarStoreStore nodes so that depending on the + // escape status of the associated Allocate node some of them + // may be eliminated. + storestore_worklist.append(n); } for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { Node* m = n->fast_out(i); // Get user @@ -1724,11 +1730,20 @@ uint alloc_length = alloc_worklist.length(); for (uint next = 0; next < alloc_length; ++next) { Node* n = alloc_worklist.at(next); - if (ptnode_adr(n->_idx)->escape_state() == PointsToNode::NoEscape) { + PointsToNode::EscapeState es = ptnode_adr(n->_idx)->escape_state(); + if (es == PointsToNode::NoEscape) { has_non_escaping_obj = true; if (n->is_Allocate()) { find_init_values(n, &visited, igvn); + // The object allocated by this Allocate node will never be + // seen by an other thread. Mark it so that when it is + // expanded no MemBarStoreStore is added. + n->as_Allocate()->initialization()->set_does_not_escape(); } + } else if ((es == PointsToNode::ArgEscape) && n->is_Allocate()) { + // Same as above. Mark this Allocate node so that when it is + // expanded no MemBarStoreStore is added. + n->as_Allocate()->initialization()->set_does_not_escape(); } } @@ -1874,6 +1889,25 @@ igvn->hash_delete(_pcmp_eq); } + // For MemBarStoreStore nodes added in library_call.cpp, check + // escape status of associated AllocateNode and optimize out + // MemBarStoreStore node if the allocated object never escapes. + while (storestore_worklist.length() != 0) { + Node *n = storestore_worklist.pop(); + MemBarStoreStoreNode *storestore = n ->as_MemBarStoreStore(); + Node *alloc = storestore->in(MemBarNode::Precedent)->in(0); + assert (alloc->is_Allocate(), "storestore should point to AllocateNode"); + PointsToNode::EscapeState es = ptnode_adr(alloc->_idx)->escape_state(); + if (es == PointsToNode::NoEscape || es == PointsToNode::ArgEscape) { + MemBarNode* mb = MemBarNode::make(C, Op_MemBarCPUOrder, Compile::AliasIdxBot); + mb->init_req(TypeFunc::Memory, storestore->in(TypeFunc::Memory)); + mb->init_req(TypeFunc::Control, storestore->in(TypeFunc::Control)); + + _igvn->register_new_node_with_optimizer(mb); + _igvn->replace_node(storestore, mb); + } + } + #ifndef PRODUCT if (PrintEscapeAnalysis) { dump(); // Dump ConnectionGraph
--- a/src/share/vm/opto/graphKit.cpp Wed Dec 07 11:35:03 2011 +0100 +++ b/src/share/vm/opto/graphKit.cpp Tue Dec 20 16:56:50 2011 +0100 @@ -3337,6 +3337,19 @@ return NULL; } +// Trace Allocate -> Proj[Parm] -> MemBarStoreStore +MemBarStoreStoreNode* AllocateNode::storestore() { + ProjNode* rawoop = proj_out(AllocateNode::RawAddress); + if (rawoop == NULL) return NULL; + for (DUIterator_Fast imax, i = rawoop->fast_outs(imax); i < imax; i++) { + Node* storestore = rawoop->fast_out(i); + if (storestore->is_MemBarStoreStore()) { + return storestore->as_MemBarStoreStore(); + } + } + return NULL; +} + //----------------------------- loop predicates --------------------------- //------------------------------add_predicate_impl----------------------------
--- a/src/share/vm/opto/library_call.cpp Wed Dec 07 11:35:03 2011 +0100 +++ b/src/share/vm/opto/library_call.cpp Tue Dec 20 16:56:50 2011 +0100 @@ -4193,12 +4193,17 @@ Node* raw_obj = alloc_obj->in(1); assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), ""); + AllocateNode* alloc = NULL; if (ReduceBulkZeroing) { // We will be completely responsible for initializing this object - // mark Initialize node as complete. - AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn); + alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn); // The object was just allocated - there should be no any stores! guarantee(alloc != NULL && alloc->maybe_set_complete(&_gvn), ""); + // Mark as complete_with_arraycopy so that on AllocateNode + // expansion, we know this AllocateNode is initialized by an array + // copy and a StoreStore barrier exists after the array copy. + alloc->initialization()->set_complete_with_arraycopy(); } // Copy the fastest available way. @@ -4260,7 +4265,18 @@ } // Do not let reads from the cloned object float above the arraycopy. - insert_mem_bar(Op_MemBarCPUOrder); + if (alloc != NULL) { + // Do not let stores that initialize this object be reordered with + // a subsequent store that would make this object accessible by + // other threads. + // Record what AllocateNode this StoreStore protects so that + // escape analysis can go from the MemBarStoreStoreNode to the + // AllocateNode and eliminate the MemBarStoreStoreNode if possible + // based on the escape status of the AllocateNode. + insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress)); + } else { + insert_mem_bar(Op_MemBarCPUOrder); + } } //------------------------inline_native_clone---------------------------- @@ -5003,7 +5019,16 @@ // the membar also. // // Do not let reads from the cloned object float above the arraycopy. - if (InsertMemBarAfterArraycopy || alloc != NULL) + if (alloc != NULL) { + // Do not let stores that initialize this object be reordered with + // a subsequent store that would make this object accessible by + // other threads. + // Record what AllocateNode this StoreStore protects so that + // escape analysis can go from the MemBarStoreStoreNode to the + // AllocateNode and eliminate the MemBarStoreStoreNode if possible + // based on the escape status of the AllocateNode. + insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress)); + } else if (InsertMemBarAfterArraycopy) insert_mem_bar(Op_MemBarCPUOrder); }
--- a/src/share/vm/opto/macro.cpp Wed Dec 07 11:35:03 2011 +0100 +++ b/src/share/vm/opto/macro.cpp Tue Dec 20 16:56:50 2011 +0100 @@ -1088,6 +1088,12 @@ Node* klass_node = alloc->in(AllocateNode::KlassNode); Node* initial_slow_test = alloc->in(AllocateNode::InitialTest); + Node* storestore = alloc->storestore(); + if (storestore != NULL) { + // Break this link that is no longer useful and confuses register allocation + storestore->set_req(MemBarNode::Precedent, top()); + } + assert(ctrl != NULL, "must have control"); // We need a Region and corresponding Phi's to merge the slow-path and fast-path results. // they will not be used if "always_slow" is set @@ -1289,10 +1295,66 @@ 0, new_alloc_bytes, T_LONG); } + InitializeNode* init = alloc->initialization(); fast_oop_rawmem = initialize_object(alloc, fast_oop_ctrl, fast_oop_rawmem, fast_oop, klass_node, length, size_in_bytes); + // If initialization is performed by an array copy, any required + // MemBarStoreStore was already added. If the object does not + // escape no need for a MemBarStoreStore. Otherwise we need a + // MemBarStoreStore so that stores that initialize this object + // can't be reordered with a subsequent store that makes this + // object accessible by other threads. + if (init == NULL || (!init->is_complete_with_arraycopy() && !init->does_not_escape())) { + if (init == NULL || init->req() < InitializeNode::RawStores) { + // No InitializeNode or no stores captured by zeroing + // elimination. Simply add the MemBarStoreStore after object + // initialization. + MemBarNode* mb = MemBarNode::make(C, Op_MemBarStoreStore, Compile::AliasIdxBot, fast_oop_rawmem); + transform_later(mb); + + mb->init_req(TypeFunc::Memory, fast_oop_rawmem); + mb->init_req(TypeFunc::Control, fast_oop_ctrl); + fast_oop_ctrl = new (C, 1) ProjNode(mb,TypeFunc::Control); + transform_later(fast_oop_ctrl); + fast_oop_rawmem = new (C, 1) ProjNode(mb,TypeFunc::Memory); + transform_later(fast_oop_rawmem); + } else { + // Add the MemBarStoreStore after the InitializeNode so that + // all stores performing the initialization that were moved + // before the InitializeNode happen before the storestore + // barrier. + + Node* init_ctrl = init->proj_out(TypeFunc::Control); + Node* init_mem = init->proj_out(TypeFunc::Memory); + + MemBarNode* mb = MemBarNode::make(C, Op_MemBarStoreStore, Compile::AliasIdxBot); + transform_later(mb); + + Node* ctrl = new (C, 1) ProjNode(init,TypeFunc::Control); + transform_later(ctrl); + Node* mem = new (C, 1) ProjNode(init,TypeFunc::Memory); + transform_later(mem); + + // The MemBarStoreStore depends on control and memory coming + // from the InitializeNode + mb->init_req(TypeFunc::Memory, mem); + mb->init_req(TypeFunc::Control, ctrl); + + ctrl = new (C, 1) ProjNode(mb,TypeFunc::Control); + transform_later(ctrl); + mem = new (C, 1) ProjNode(mb,TypeFunc::Memory); + transform_later(mem); + + // All nodes that depended on the InitializeNode for control + // and memory must now depend on the MemBarNode that itself + // depends on the InitializeNode + _igvn.replace_node(init_ctrl, ctrl); + _igvn.replace_node(init_mem, mem); + } + } + if (C->env()->dtrace_extended_probes()) { // Slow-path call int size = TypeFunc::Parms + 2;
--- a/src/share/vm/opto/memnode.cpp Wed Dec 07 11:35:03 2011 +0100 +++ b/src/share/vm/opto/memnode.cpp Tue Dec 20 16:56:50 2011 +0100 @@ -2721,6 +2721,7 @@ case Op_MemBarVolatile: return new(C, len) MemBarVolatileNode(C, atp, pn); case Op_MemBarCPUOrder: return new(C, len) MemBarCPUOrderNode(C, atp, pn); case Op_Initialize: return new(C, len) InitializeNode(C, atp, pn); + case Op_MemBarStoreStore: return new(C, len) MemBarStoreStoreNode(C, atp, pn); default: ShouldNotReachHere(); return NULL; } } @@ -2870,7 +2871,7 @@ //---------------------------InitializeNode------------------------------------ InitializeNode::InitializeNode(Compile* C, int adr_type, Node* rawoop) - : _is_complete(Incomplete), + : _is_complete(Incomplete), _does_not_escape(false), MemBarNode(C, adr_type, rawoop) { init_class_id(Class_Initialize);
--- a/src/share/vm/opto/memnode.hpp Wed Dec 07 11:35:03 2011 +0100 +++ b/src/share/vm/opto/memnode.hpp Tue Dec 20 16:56:50 2011 +0100 @@ -918,6 +918,15 @@ virtual int Opcode() const; }; +class MemBarStoreStoreNode: public MemBarNode { +public: + MemBarStoreStoreNode(Compile* C, int alias_idx, Node* precedent) + : MemBarNode(C, alias_idx, precedent) { + init_class_id(Class_MemBarStoreStore); + } + virtual int Opcode() const; +}; + // Ordering between a volatile store and a following volatile load. // Requires multi-CPU visibility? class MemBarVolatileNode: public MemBarNode { @@ -950,6 +959,8 @@ }; int _is_complete; + bool _does_not_escape; + public: enum { Control = TypeFunc::Control, @@ -989,6 +1000,9 @@ void set_complete(PhaseGVN* phase); void set_complete_with_arraycopy() { _is_complete = Complete | WithArraycopy; } + bool does_not_escape() { return _does_not_escape; } + void set_does_not_escape() { _does_not_escape = true; } + #ifdef ASSERT // ensure all non-degenerate stores are ordered and non-overlapping bool stores_are_sane(PhaseTransform* phase);
--- a/src/share/vm/opto/node.hpp Wed Dec 07 11:35:03 2011 +0100 +++ b/src/share/vm/opto/node.hpp Tue Dec 20 16:56:50 2011 +0100 @@ -97,6 +97,7 @@ class MachTempNode; class Matcher; class MemBarNode; +class MemBarStoreStoreNode; class MemNode; class MergeMemNode; class MultiNode; @@ -564,7 +565,8 @@ DEFINE_CLASS_ID(NeverBranch, MultiBranch, 2) DEFINE_CLASS_ID(Start, Multi, 2) DEFINE_CLASS_ID(MemBar, Multi, 3) - DEFINE_CLASS_ID(Initialize, MemBar, 0) + DEFINE_CLASS_ID(Initialize, MemBar, 0) + DEFINE_CLASS_ID(MemBarStoreStore, MemBar, 1) DEFINE_CLASS_ID(Mach, Node, 1) DEFINE_CLASS_ID(MachReturn, Mach, 0) @@ -744,6 +746,7 @@ DEFINE_CLASS_QUERY(MachTemp) DEFINE_CLASS_QUERY(Mem) DEFINE_CLASS_QUERY(MemBar) + DEFINE_CLASS_QUERY(MemBarStoreStore) DEFINE_CLASS_QUERY(MergeMem) DEFINE_CLASS_QUERY(Multi) DEFINE_CLASS_QUERY(MultiBranch)