# HG changeset patch # User Roman Kennke # Date 1421931122 -3600 # Node ID b51c76b380cd7c8ec6637a982fa4cf76faee4a16 # Parent 0086e1d4fe0c5301fbd0e7912aa04a42fb2e14de Implement resizable TLABs in Shenandoah. diff -r 0086e1d4fe0c -r b51c76b380cd src/share/vm/gc_implementation/shenandoah/shenandoahHeap.cpp --- a/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.cpp Thu Jan 22 13:51:37 2015 +0100 +++ b/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.cpp Thu Jan 22 13:52:02 2015 +0100 @@ -361,7 +361,7 @@ } HeapWord* ShenandoahHeap::allocate_new_tlab(size_t word_size) { - HeapWord* result = allocate_memory(word_size, false); + HeapWord* result = allocate_memory(word_size); if (result != NULL) { if (_concurrent_mark_in_progress || (ShenandoahUpdateRefsEarly && _evacuation_in_progress)) { @@ -381,30 +381,6 @@ return result; } -HeapWord* ShenandoahHeap::allocate_new_gclab(size_t word_size) { - HeapWord* result = allocate_memory(word_size, true); - if (result == NULL) { - oom_during_evacuation(); - return NULL; - } - assert(! _concurrent_mark_in_progress, "no new gclabs during marking"); - assert(_evacuation_in_progress, "new gclabs only during evacuation"); - if (ShenandoahUpdateRefsEarly) { - // We mark the whole tlab here, this way we avoid marking every single - // allocated object. - _next_mark_bit_map->parMarkRange(MemRegion(result, word_size)); - } - assert(! heap_region_containing(result)->is_in_collection_set(), "Never allocate in dirty region"); - if (result != NULL) { - if (ShenandoahTraceTLabs) { - tty->print("allocating new gclab of size "SIZE_FORMAT" at addr %p\n", word_size, result); - } - - } - return result; -} - - ShenandoahHeap* ShenandoahHeap::heap() { assert(_pgc != NULL, "Unitialized access to ShenandoahHeap::heap()"); assert(_pgc->kind() == CollectedHeap::ShenandoahHeap, "not a shenandoah heap"); @@ -449,58 +425,58 @@ }; -HeapWord* ShenandoahHeap::allocate_memory(size_t word_size, bool evacuation) { +HeapWord* ShenandoahHeap::allocate_memory(size_t word_size) { HeapWord* result = NULL; - result = allocate_memory_with_lock(word_size, evacuation); - - if (result == NULL && ! evacuation) { // Allocation failed, try full-GC, then retry allocation. + result = allocate_memory_with_lock(word_size); + + if (result == NULL && ! Thread::current()->is_evacuating()) { // Allocation failed, try full-GC, then retry allocation. collect(GCCause::_allocation_failure); - result = allocate_memory_with_lock(word_size, evacuation); + result = allocate_memory_with_lock(word_size); } return result; } -HeapWord* ShenandoahHeap::allocate_memory_with_lock(size_t word_size, bool evacuation) { +HeapWord* ShenandoahHeap::allocate_memory_with_lock(size_t word_size) { if (Thread::current()->is_GC_task_thread() && SafepointSynchronize::is_at_safepoint()) { - return allocate_memory_shenandoah_lock(word_size, evacuation); + return allocate_memory_shenandoah_lock(word_size); } else { - return allocate_memory_heap_lock(word_size, evacuation); + return allocate_memory_heap_lock(word_size); } } -HeapWord* ShenandoahHeap::allocate_memory_heap_lock(size_t word_size, bool evacuation) { +HeapWord* ShenandoahHeap::allocate_memory_heap_lock(size_t word_size) { MutexLocker ml(Heap_lock); - return allocate_memory_work(word_size, evacuation); + return allocate_memory_work(word_size); } -HeapWord* ShenandoahHeap::allocate_memory_shenandoah_lock(size_t word_size, bool evacuation) { +HeapWord* ShenandoahHeap::allocate_memory_shenandoah_lock(size_t word_size) { MutexLocker ml(ShenandoahHeap_lock); - return allocate_memory_work(word_size, evacuation); + return allocate_memory_work(word_size); } -ShenandoahHeapRegion* ShenandoahHeap::check_skip_humonguous(ShenandoahHeapRegion* region, bool evacuation) { +ShenandoahHeapRegion* ShenandoahHeap::check_skip_humonguous(ShenandoahHeapRegion* region) { while (region != NULL && region->is_humonguous()) { - region = _free_regions->get_next(evacuation); + region = _free_regions->get_next(); } return region; } -ShenandoahHeapRegion* ShenandoahHeap::get_next_region_skip_humonguous(bool evacuation) { - ShenandoahHeapRegion* next = _free_regions->get_next(evacuation); - return check_skip_humonguous(next, evacuation); +ShenandoahHeapRegion* ShenandoahHeap::get_next_region_skip_humonguous() { + ShenandoahHeapRegion* next = _free_regions->get_next(); + return check_skip_humonguous(next); } -ShenandoahHeapRegion* ShenandoahHeap::get_current_region_skip_humonguous(bool evacuation) { - ShenandoahHeapRegion* current = _free_regions->current(evacuation); - return check_skip_humonguous(current, evacuation); +ShenandoahHeapRegion* ShenandoahHeap::get_current_region_skip_humonguous() { + ShenandoahHeapRegion* current = _free_regions->current(); + return check_skip_humonguous(current); } -ShenandoahHeapRegion* ShenandoahHeap::check_grow_heap(ShenandoahHeapRegion* current, bool evacuation) { +ShenandoahHeapRegion* ShenandoahHeap::check_grow_heap(ShenandoahHeapRegion* current) { if (current == NULL) { if (grow_heap_by()) { - current = _free_regions->get_next(evacuation); + current = _free_regions->get_next(); assert(current != NULL, "After successfully growing the heap we should have a region"); assert(! current->is_humonguous(), "new region must not be humonguous"); } else { @@ -510,25 +486,25 @@ return current; } -ShenandoahHeapRegion* ShenandoahHeap::get_current_region(bool evacuation) { - ShenandoahHeapRegion* current = get_current_region_skip_humonguous(evacuation); - return check_grow_heap(current, evacuation); +ShenandoahHeapRegion* ShenandoahHeap::get_current_region() { + ShenandoahHeapRegion* current = get_current_region_skip_humonguous(); + return check_grow_heap(current); } -ShenandoahHeapRegion* ShenandoahHeap::get_next_region(bool evacuation) { - ShenandoahHeapRegion* current = get_next_region_skip_humonguous(evacuation); - return check_grow_heap(current, evacuation); +ShenandoahHeapRegion* ShenandoahHeap::get_next_region() { + ShenandoahHeapRegion* current = get_next_region_skip_humonguous(); + return check_grow_heap(current); } -HeapWord* ShenandoahHeap::allocate_memory_work(size_t word_size, bool evacuation) { +HeapWord* ShenandoahHeap::allocate_memory_work(size_t word_size) { if (word_size * HeapWordSize > ShenandoahHeapRegion::RegionSizeBytes) { - assert(! evacuation, "no evacuation of humonguous objects"); + assert(! Thread::current()->is_evacuating(), "no humonguous allocation for evacuating thread"); return allocate_large_memory(word_size); } - ShenandoahHeapRegion* my_current_region = get_current_region(false); + ShenandoahHeapRegion* my_current_region = get_current_region(); if (my_current_region == NULL) { return NULL; // No more room to make a new region. OOM. } @@ -547,7 +523,7 @@ result = my_current_region->par_allocate(word_size); while (result == NULL && my_current_region != NULL) { // 2nd attempt. Try next region. - my_current_region = get_next_region(false); + my_current_region = get_next_region(); if (my_current_region == NULL) { return NULL; // No more room to make a new region. OOM. } @@ -679,7 +655,7 @@ // This was used for allocation while holding the Heap_lock. // HeapWord* filler = allocate_memory(BrooksPointer::BROOKS_POINTER_OBJ_SIZE + size); - HeapWord* filler = allocate_memory(BrooksPointer::BROOKS_POINTER_OBJ_SIZE + size, false); + HeapWord* filler = allocate_memory(BrooksPointer::BROOKS_POINTER_OBJ_SIZE + size); HeapWord* result = filler + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; if (filler != NULL) { initialize_brooks_ptr(filler, result); @@ -2252,57 +2228,6 @@ #endif } -HeapWord* ShenandoahHeap::allocate_from_gclab(Thread* thread, size_t size) { - HeapWord* obj = thread->tlab().allocate(size); - if (obj != NULL) { - return obj; - } - // Otherwise... - return allocate_from_gclab_slow(thread, size); -} - -HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) { - // Retain tlab and allocate object in shared space if - // the amount free in the tlab is too large to discard. - if (thread->tlab().free() > thread->tlab().refill_waste_limit()) { - thread->tlab().record_slow_allocation(size); - return NULL; - } - - // Discard tlab and allocate a new one. - // To minimize fragmentation, the last TLAB may be smaller than the rest. - size_t new_gclab_size = thread->tlab().compute_size(size); - - thread->tlab().clear_before_allocation(); - - if (new_gclab_size == 0) { - return NULL; - } - - // Allocate a new TLAB... - HeapWord* obj = allocate_new_gclab(new_gclab_size); - if (obj == NULL) { - return NULL; - } - fill_with_object(obj, size); - - if (ZeroTLAB) { - // ..and clear it. - Copy::zero_to_words(obj, new_gclab_size); - } else { - // ...and zap just allocated object. -#ifdef ASSERT - // Skip mangling the space corresponding to the object header to - // ensure that the returned space is not considered parsable by - // any concurrent GC thread. - size_t hdr_size = oopDesc::header_size(); - Copy::fill_to_words(obj + hdr_size, new_gclab_size - hdr_size, badHeapWordVal); -#endif // ASSERT - } - thread->tlab().fill(obj, obj + size, new_gclab_size); - return obj; -} - oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { ShenandoahHeapRegion* hr; size_t required; @@ -2330,11 +2255,13 @@ } bool alloc_from_gclab = true; - HeapWord* filler = allocate_from_gclab(thread, required); + thread->set_evacuating(true); + HeapWord* filler = allocate_from_tlab_work(SystemDictionary::Object_klass(), thread, required); if (filler == NULL) { - filler = allocate_memory(required, true); + filler = allocate_memory(required); alloc_from_gclab = false; } + thread->set_evacuating(false); if (filler == NULL) { oom_during_evacuation(); @@ -2390,10 +2317,6 @@ return return_val; } -HeapWord* ShenandoahHeap::allocate_from_tlab_work(Thread* thread, size_t size) { - return CollectedHeap::allocate_from_tlab_work(SystemDictionary::Object_klass(), thread, size); -} - HeapWord* ShenandoahHeap::tlab_post_allocation_setup(HeapWord* obj, bool new_obj) { HeapWord* result = obj + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; initialize_brooks_ptr(obj, result, new_obj); diff -r 0086e1d4fe0c -r b51c76b380cd src/share/vm/gc_implementation/shenandoah/shenandoahHeap.hpp --- a/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.hpp Thu Jan 22 13:51:37 2015 +0100 +++ b/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.hpp Thu Jan 22 13:52:02 2015 +0100 @@ -108,7 +108,7 @@ ShenandoahHeap(ShenandoahCollectorPolicy* policy); HeapWord* allocate_new_tlab(size_t word_size); - HeapWord* allocate_memory(size_t word_size, bool evacuation); + HeapWord* allocate_memory(size_t word_size); bool find_contiguous_free_regions(uint num_free_regions, ShenandoahHeapRegion** free_regions); bool allocate_contiguous_free_regions(uint num_free_regions, ShenandoahHeapRegion** free_regions); @@ -271,9 +271,6 @@ void verify_heap_after_update_refs(); void verify_regions_after_update_refs(); - // This is here to get access to the otherwise protected method in CollectedHeap. - static HeapWord* allocate_from_tlab_work(Thread* thread, size_t size); - static ByteSize ordered_regions_offset() { return byte_offset_of(ShenandoahHeap, _ordered_regions); } static ByteSize first_region_bottom_offset() { return byte_offset_of(ShenandoahHeap, _first_region_bottom); } static address evacuation_in_progress_addr() { @@ -361,10 +358,6 @@ void verify_evacuation(ShenandoahHeapRegion* from_region); void set_concurrent_mark_in_progress(bool in_progress); - HeapWord* allocate_new_gclab(size_t word_size); - HeapWord* allocate_from_gclab(Thread* thread, size_t size); - HeapWord* allocate_from_gclab_slow(Thread* thread, size_t size); - void oom_during_evacuation(); void cancel_evacuation(); public: @@ -378,17 +371,17 @@ void verify_live(); void verify_liveness_after_concurrent_mark(); - HeapWord* allocate_memory_with_lock(size_t word_size, bool evacuation); - HeapWord* allocate_memory_heap_lock(size_t word_size, bool evacuation); - HeapWord* allocate_memory_shenandoah_lock(size_t word_size, bool evacuation); - HeapWord* allocate_memory_work(size_t word_size, bool evacuation); + HeapWord* allocate_memory_with_lock(size_t word_size); + HeapWord* allocate_memory_heap_lock(size_t word_size); + HeapWord* allocate_memory_shenandoah_lock(size_t word_size); + HeapWord* allocate_memory_work(size_t word_size); HeapWord* allocate_large_memory(size_t word_size); - ShenandoahHeapRegion* check_skip_humonguous(ShenandoahHeapRegion* region, bool evacuation); - ShenandoahHeapRegion* get_next_region_skip_humonguous(bool evacuation); - ShenandoahHeapRegion* get_current_region_skip_humonguous(bool evacuation); - ShenandoahHeapRegion* check_grow_heap(ShenandoahHeapRegion* current, bool evacuation); - ShenandoahHeapRegion* get_next_region(bool evacuation); - ShenandoahHeapRegion* get_current_region(bool evacuation); + ShenandoahHeapRegion* check_skip_humonguous(ShenandoahHeapRegion* region); + ShenandoahHeapRegion* get_next_region_skip_humonguous(); + ShenandoahHeapRegion* get_current_region_skip_humonguous(); + ShenandoahHeapRegion* check_grow_heap(ShenandoahHeapRegion* current); + ShenandoahHeapRegion* get_next_region(); + ShenandoahHeapRegion* get_current_region(); void set_from_region_protection(bool protect); }; diff -r 0086e1d4fe0c -r b51c76b380cd src/share/vm/gc_implementation/shenandoah/shenandoahHeapRegionSet.cpp --- a/src/share/vm/gc_implementation/shenandoah/shenandoahHeapRegionSet.cpp Thu Jan 22 13:51:37 2015 +0100 +++ b/src/share/vm/gc_implementation/shenandoah/shenandoahHeapRegionSet.cpp Thu Jan 22 13:52:02 2015 +0100 @@ -15,8 +15,7 @@ _free_threshold(ShenandoahHeapRegion::RegionSizeBytes / 2) { _next = &_regions[0]; - _current_allocation = NULL; - _current_evacuation = NULL; + _current = NULL; _next_free = &_regions[0]; } @@ -30,8 +29,7 @@ memcpy(_regions, regions, sizeof(ShenandoahHeapRegion*) * num_regions); _next = &_regions[0]; - _current_allocation = NULL; - _current_evacuation = NULL; + _current = NULL; _next_free = &_regions[num_regions]; } @@ -60,10 +58,10 @@ else return 0; } -ShenandoahHeapRegion* ShenandoahHeapRegionSet::current(bool evacuation) { - ShenandoahHeapRegion** current = evacuation ? _current_evacuation : _current_allocation; +ShenandoahHeapRegion* ShenandoahHeapRegionSet::current() { + ShenandoahHeapRegion** current = _current; if (current == NULL) { - return get_next(evacuation); + return get_next(); } else { return *(limit_region(current)); } @@ -90,8 +88,7 @@ } void ShenandoahHeapRegionSet::clear() { - _current_allocation = NULL; - _current_evacuation = NULL; + _current = NULL; _next = _regions; _next_free = _regions; } @@ -106,15 +103,11 @@ } } -ShenandoahHeapRegion* ShenandoahHeapRegionSet::get_next(bool evacuation) { +ShenandoahHeapRegion* ShenandoahHeapRegionSet::get_next() { ShenandoahHeapRegion** next = _next; if (next < _next_free) { - if (evacuation) { - _current_evacuation = next; - } else { - _current_allocation = next; - } + _current = next; _next++; return *next; } else { @@ -132,11 +125,8 @@ void ShenandoahHeapRegionSet::print() { for (ShenandoahHeapRegion** i = _regions; i < _next_free; i++) { - if (i == _current_allocation) { - tty->print_cr("A->"); - } - if (i == _current_evacuation) { - tty->print_cr("E->"); + if (i == _current) { + tty->print_cr("C->"); } if (i == _next) { tty->print_cr("N->"); diff -r 0086e1d4fe0c -r b51c76b380cd src/share/vm/gc_implementation/shenandoah/shenandoahHeapRegionSet.hpp --- a/src/share/vm/gc_implementation/shenandoah/shenandoahHeapRegionSet.hpp Thu Jan 22 13:51:37 2015 +0100 +++ b/src/share/vm/gc_implementation/shenandoah/shenandoahHeapRegionSet.hpp Thu Jan 22 13:52:02 2015 +0100 @@ -11,8 +11,7 @@ private: ShenandoahHeapRegion** _regions; // current region to be returned from get_next() - ShenandoahHeapRegion** _current_allocation; - ShenandoahHeapRegion** _current_evacuation; + ShenandoahHeapRegion** _current; ShenandoahHeapRegion** _next; // last inserted region. @@ -57,7 +56,7 @@ /** * Returns a pointer to the current region. */ - ShenandoahHeapRegion* current(bool evacuation); + ShenandoahHeapRegion* current(); /** * Gets the next region for allocation (from free-list). @@ -65,7 +64,7 @@ * increment to the next region, the others will fail and return * the region that the succeeding thread got. */ - ShenandoahHeapRegion* get_next(bool evacuation); + ShenandoahHeapRegion* get_next(); /** * Claims next region for processing. This is implemented to be concurrency-safe. diff -r 0086e1d4fe0c -r b51c76b380cd src/share/vm/runtime/thread.cpp --- a/src/share/vm/runtime/thread.cpp Thu Jan 22 13:51:37 2015 +0100 +++ b/src/share/vm/runtime/thread.cpp Thu Jan 22 13:52:02 2015 +0100 @@ -289,6 +289,8 @@ _MutexEvent = ParkEvent::Allocate (this) ; _MuxEvent = ParkEvent::Allocate (this) ; + _evacuating = false; + #ifdef CHECK_UNHANDLED_OOPS if (CheckUnhandledOops) { _unhandled_oops = new UnhandledOops(this); diff -r 0086e1d4fe0c -r b51c76b380cd src/share/vm/runtime/thread.hpp --- a/src/share/vm/runtime/thread.hpp Thu Jan 22 13:51:37 2015 +0100 +++ b/src/share/vm/runtime/thread.hpp Thu Jan 22 13:52:02 2015 +0100 @@ -276,6 +276,8 @@ // ObjectMonitor on which this thread called Object.wait() ObjectMonitor* _current_waiting_monitor; + bool _evacuating; + // Private thread-local objectmonitor list - a simple cache organized as a SLL. public: ObjectMonitor* omFreeList; @@ -475,6 +477,14 @@ _current_waiting_monitor = monitor; } + bool is_evacuating() { + return _evacuating; + } + + void set_evacuating(bool evacuating) { + _evacuating = evacuating; + } + // GC support // Apply "f->do_oop" to all root oops in "this". // Apply "cld_f->do_cld" to CLDs that are otherwise not kept alive.