changeset 7455:b51c76b380cd

Implement resizable TLABs in Shenandoah.
author Roman Kennke <rkennke@redhat.com>
date Thu, 22 Jan 2015 13:52:02 +0100
parents 0086e1d4fe0c
children ca617ad7ee11
files src/share/vm/gc_implementation/shenandoah/shenandoahHeap.cpp src/share/vm/gc_implementation/shenandoah/shenandoahHeap.hpp src/share/vm/gc_implementation/shenandoah/shenandoahHeapRegionSet.cpp src/share/vm/gc_implementation/shenandoah/shenandoahHeapRegionSet.hpp src/share/vm/runtime/thread.cpp src/share/vm/runtime/thread.hpp
diffstat 6 files changed, 74 insertions(+), 157 deletions(-) [+]
line wrap: on
line diff
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.cpp	Thu Jan 22 13:51:37 2015 +0100
+++ b/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.cpp	Thu Jan 22 13:52:02 2015 +0100
@@ -361,7 +361,7 @@
 }
 
 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t word_size) {
-  HeapWord* result = allocate_memory(word_size, false);
+  HeapWord* result = allocate_memory(word_size);
 
   if (result != NULL) {
     if (_concurrent_mark_in_progress || (ShenandoahUpdateRefsEarly && _evacuation_in_progress)) {
@@ -381,30 +381,6 @@
   return result;
 }
 
-HeapWord* ShenandoahHeap::allocate_new_gclab(size_t word_size) {
-  HeapWord* result = allocate_memory(word_size, true);
-  if (result == NULL) {
-    oom_during_evacuation();
-    return NULL;
-  }
-  assert(! _concurrent_mark_in_progress, "no new gclabs during marking");
-  assert(_evacuation_in_progress, "new gclabs only during evacuation");
-  if (ShenandoahUpdateRefsEarly) {
-    // We mark the whole tlab here, this way we avoid marking every single
-    // allocated object.
-    _next_mark_bit_map->parMarkRange(MemRegion(result, word_size));
-  }
-  assert(! heap_region_containing(result)->is_in_collection_set(), "Never allocate in dirty region");
-  if (result != NULL) {
-    if (ShenandoahTraceTLabs) {
-      tty->print("allocating new gclab of size "SIZE_FORMAT" at addr %p\n", word_size, result);
-    }
-
-  }
-  return result;
-}
-
-  
 ShenandoahHeap* ShenandoahHeap::heap() {
   assert(_pgc != NULL, "Unitialized access to ShenandoahHeap::heap()");
   assert(_pgc->kind() == CollectedHeap::ShenandoahHeap, "not a shenandoah heap");
@@ -449,58 +425,58 @@
 
 };
 
-HeapWord* ShenandoahHeap::allocate_memory(size_t word_size, bool evacuation) {
+HeapWord* ShenandoahHeap::allocate_memory(size_t word_size) {
   HeapWord* result = NULL;
-  result = allocate_memory_with_lock(word_size, evacuation);
-
-  if (result == NULL && ! evacuation) { // Allocation failed, try full-GC, then retry allocation.
+  result = allocate_memory_with_lock(word_size);
+
+  if (result == NULL && ! Thread::current()->is_evacuating()) { // Allocation failed, try full-GC, then retry allocation.
     collect(GCCause::_allocation_failure);
-    result = allocate_memory_with_lock(word_size, evacuation);
+    result = allocate_memory_with_lock(word_size);
   }
 
   return result;
 }
 
-HeapWord* ShenandoahHeap::allocate_memory_with_lock(size_t word_size, bool evacuation) {
+HeapWord* ShenandoahHeap::allocate_memory_with_lock(size_t word_size) {
   if (Thread::current()->is_GC_task_thread() && SafepointSynchronize::is_at_safepoint()) {
-    return allocate_memory_shenandoah_lock(word_size, evacuation);
+    return allocate_memory_shenandoah_lock(word_size);
   } else {
-    return allocate_memory_heap_lock(word_size, evacuation);
+    return allocate_memory_heap_lock(word_size);
   }
 }
 
-HeapWord* ShenandoahHeap::allocate_memory_heap_lock(size_t word_size, bool evacuation) {
+HeapWord* ShenandoahHeap::allocate_memory_heap_lock(size_t word_size) {
   MutexLocker ml(Heap_lock);
-  return allocate_memory_work(word_size, evacuation);
+  return allocate_memory_work(word_size);
 }
 
-HeapWord* ShenandoahHeap::allocate_memory_shenandoah_lock(size_t word_size, bool evacuation) {
+HeapWord* ShenandoahHeap::allocate_memory_shenandoah_lock(size_t word_size) {
   MutexLocker ml(ShenandoahHeap_lock);
-  return allocate_memory_work(word_size, evacuation);
+  return allocate_memory_work(word_size);
 }
 
-ShenandoahHeapRegion* ShenandoahHeap::check_skip_humonguous(ShenandoahHeapRegion* region, bool evacuation) {
+ShenandoahHeapRegion* ShenandoahHeap::check_skip_humonguous(ShenandoahHeapRegion* region) {
   while (region != NULL && region->is_humonguous()) {
-    region = _free_regions->get_next(evacuation);
+    region = _free_regions->get_next();
   }
   return region;
 }
 
-ShenandoahHeapRegion* ShenandoahHeap::get_next_region_skip_humonguous(bool evacuation) {
-  ShenandoahHeapRegion* next = _free_regions->get_next(evacuation);
-  return check_skip_humonguous(next, evacuation);
+ShenandoahHeapRegion* ShenandoahHeap::get_next_region_skip_humonguous() {
+  ShenandoahHeapRegion* next = _free_regions->get_next();
+  return check_skip_humonguous(next);
 }
 
-ShenandoahHeapRegion* ShenandoahHeap::get_current_region_skip_humonguous(bool evacuation) {
-  ShenandoahHeapRegion* current = _free_regions->current(evacuation);
-  return check_skip_humonguous(current, evacuation);
+ShenandoahHeapRegion* ShenandoahHeap::get_current_region_skip_humonguous() {
+  ShenandoahHeapRegion* current = _free_regions->current();
+  return check_skip_humonguous(current);
 }
 
 
-ShenandoahHeapRegion* ShenandoahHeap::check_grow_heap(ShenandoahHeapRegion* current, bool evacuation) {
+ShenandoahHeapRegion* ShenandoahHeap::check_grow_heap(ShenandoahHeapRegion* current) {
   if (current == NULL) {
     if (grow_heap_by()) {
-      current = _free_regions->get_next(evacuation);
+      current = _free_regions->get_next();
       assert(current != NULL, "After successfully growing the heap we should have a region");
       assert(! current->is_humonguous(), "new region must not be humonguous");
     } else {
@@ -510,25 +486,25 @@
   return current;
 }
 
-ShenandoahHeapRegion* ShenandoahHeap::get_current_region(bool evacuation) {
-  ShenandoahHeapRegion* current = get_current_region_skip_humonguous(evacuation);
-  return check_grow_heap(current, evacuation);
+ShenandoahHeapRegion* ShenandoahHeap::get_current_region() {
+  ShenandoahHeapRegion* current = get_current_region_skip_humonguous();
+  return check_grow_heap(current);
 }
 
-ShenandoahHeapRegion* ShenandoahHeap::get_next_region(bool evacuation) {
-  ShenandoahHeapRegion* current = get_next_region_skip_humonguous(evacuation);
-  return check_grow_heap(current, evacuation);
+ShenandoahHeapRegion* ShenandoahHeap::get_next_region() {
+  ShenandoahHeapRegion* current = get_next_region_skip_humonguous();
+  return check_grow_heap(current);
 }
 
 
-HeapWord* ShenandoahHeap::allocate_memory_work(size_t word_size, bool evacuation) {
+HeapWord* ShenandoahHeap::allocate_memory_work(size_t word_size) {
 
   if (word_size * HeapWordSize > ShenandoahHeapRegion::RegionSizeBytes) {
-    assert(! evacuation, "no evacuation of humonguous objects");
+    assert(! Thread::current()->is_evacuating(), "no humonguous allocation for evacuating thread");
     return allocate_large_memory(word_size);
   }
 
-  ShenandoahHeapRegion* my_current_region = get_current_region(false);
+  ShenandoahHeapRegion* my_current_region = get_current_region();
   if (my_current_region == NULL) {
     return NULL; // No more room to make a new region. OOM.
   }
@@ -547,7 +523,7 @@
   result = my_current_region->par_allocate(word_size);
   while (result == NULL && my_current_region != NULL) {
     // 2nd attempt. Try next region.
-    my_current_region = get_next_region(false);
+    my_current_region = get_next_region();
     if (my_current_region == NULL) {
       return NULL; // No more room to make a new region. OOM.
     }
@@ -679,7 +655,7 @@
   // This was used for allocation while holding the Heap_lock.
   // HeapWord* filler = allocate_memory(BrooksPointer::BROOKS_POINTER_OBJ_SIZE + size);
 
-  HeapWord* filler = allocate_memory(BrooksPointer::BROOKS_POINTER_OBJ_SIZE + size, false);
+  HeapWord* filler = allocate_memory(BrooksPointer::BROOKS_POINTER_OBJ_SIZE + size);
   HeapWord* result = filler + BrooksPointer::BROOKS_POINTER_OBJ_SIZE;
   if (filler != NULL) {
     initialize_brooks_ptr(filler, result);
@@ -2252,57 +2228,6 @@
 #endif
 }
 
-HeapWord* ShenandoahHeap::allocate_from_gclab(Thread* thread, size_t size) {
-  HeapWord* obj = thread->tlab().allocate(size);
-  if (obj != NULL) {
-    return obj;
-  }
-  // Otherwise...
-  return allocate_from_gclab_slow(thread, size);
-}
-
-HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
-  // Retain tlab and allocate object in shared space if
-  // the amount free in the tlab is too large to discard.
-  if (thread->tlab().free() > thread->tlab().refill_waste_limit()) {
-    thread->tlab().record_slow_allocation(size);
-    return NULL;
-  }
-
-  // Discard tlab and allocate a new one.
-  // To minimize fragmentation, the last TLAB may be smaller than the rest.
-  size_t new_gclab_size = thread->tlab().compute_size(size);
-
-  thread->tlab().clear_before_allocation();
-
-  if (new_gclab_size == 0) {
-    return NULL;
-  }
-
-  // Allocate a new TLAB...
-  HeapWord* obj = allocate_new_gclab(new_gclab_size);
-  if (obj == NULL) {
-    return NULL;
-  }
-  fill_with_object(obj, size);
-
-  if (ZeroTLAB) {
-    // ..and clear it.
-    Copy::zero_to_words(obj, new_gclab_size);
-  } else {
-    // ...and zap just allocated object.
-#ifdef ASSERT
-    // Skip mangling the space corresponding to the object header to
-    // ensure that the returned space is not considered parsable by
-    // any concurrent GC thread.
-    size_t hdr_size = oopDesc::header_size();
-    Copy::fill_to_words(obj + hdr_size, new_gclab_size - hdr_size, badHeapWordVal);
-#endif // ASSERT
-  }
-  thread->tlab().fill(obj, obj + size, new_gclab_size);
-  return obj;
-}
-
 oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
   ShenandoahHeapRegion* hr;
   size_t required;
@@ -2330,11 +2255,13 @@
   }
 
   bool alloc_from_gclab = true;
-  HeapWord* filler = allocate_from_gclab(thread, required);
+  thread->set_evacuating(true);
+  HeapWord* filler = allocate_from_tlab_work(SystemDictionary::Object_klass(), thread, required);
   if (filler == NULL) {
-    filler = allocate_memory(required, true);
+    filler = allocate_memory(required);
     alloc_from_gclab = false;
   }
+  thread->set_evacuating(false);
 
   if (filler == NULL) {
     oom_during_evacuation();
@@ -2390,10 +2317,6 @@
   return return_val;
 }
 
-HeapWord* ShenandoahHeap::allocate_from_tlab_work(Thread* thread, size_t size) {
-  return CollectedHeap::allocate_from_tlab_work(SystemDictionary::Object_klass(), thread, size);
-}
-
 HeapWord* ShenandoahHeap::tlab_post_allocation_setup(HeapWord* obj, bool new_obj) {
   HeapWord* result = obj + BrooksPointer::BROOKS_POINTER_OBJ_SIZE;
   initialize_brooks_ptr(obj, result, new_obj);
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.hpp	Thu Jan 22 13:51:37 2015 +0100
+++ b/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.hpp	Thu Jan 22 13:52:02 2015 +0100
@@ -108,7 +108,7 @@
   ShenandoahHeap(ShenandoahCollectorPolicy* policy);
   HeapWord* allocate_new_tlab(size_t word_size);
 
-  HeapWord* allocate_memory(size_t word_size, bool evacuation);
+  HeapWord* allocate_memory(size_t word_size);
 
   bool find_contiguous_free_regions(uint num_free_regions, ShenandoahHeapRegion** free_regions);
   bool allocate_contiguous_free_regions(uint num_free_regions, ShenandoahHeapRegion** free_regions);
@@ -271,9 +271,6 @@
   void verify_heap_after_update_refs();
   void verify_regions_after_update_refs();
 
-  // This is here to get access to the otherwise protected method in CollectedHeap.
-  static HeapWord* allocate_from_tlab_work(Thread* thread, size_t size);
-
   static ByteSize ordered_regions_offset() { return byte_offset_of(ShenandoahHeap, _ordered_regions); }
   static ByteSize first_region_bottom_offset() { return byte_offset_of(ShenandoahHeap, _first_region_bottom); }
   static address evacuation_in_progress_addr() {
@@ -361,10 +358,6 @@
   void verify_evacuation(ShenandoahHeapRegion* from_region);
   void set_concurrent_mark_in_progress(bool in_progress);
 
-  HeapWord* allocate_new_gclab(size_t word_size);
-  HeapWord* allocate_from_gclab(Thread* thread, size_t size);
-  HeapWord* allocate_from_gclab_slow(Thread* thread, size_t size);
-
   void oom_during_evacuation();
   void cancel_evacuation();
 public:
@@ -378,17 +371,17 @@
   void verify_live();
   void verify_liveness_after_concurrent_mark();
 
-  HeapWord* allocate_memory_with_lock(size_t word_size, bool evacuation);
-  HeapWord* allocate_memory_heap_lock(size_t word_size, bool evacuation);
-  HeapWord* allocate_memory_shenandoah_lock(size_t word_size, bool evacuation);
-  HeapWord* allocate_memory_work(size_t word_size, bool evacuation);
+  HeapWord* allocate_memory_with_lock(size_t word_size);
+  HeapWord* allocate_memory_heap_lock(size_t word_size);
+  HeapWord* allocate_memory_shenandoah_lock(size_t word_size);
+  HeapWord* allocate_memory_work(size_t word_size);
   HeapWord* allocate_large_memory(size_t word_size);
-  ShenandoahHeapRegion* check_skip_humonguous(ShenandoahHeapRegion* region, bool evacuation);
-  ShenandoahHeapRegion* get_next_region_skip_humonguous(bool evacuation);
-  ShenandoahHeapRegion* get_current_region_skip_humonguous(bool evacuation);
-  ShenandoahHeapRegion* check_grow_heap(ShenandoahHeapRegion* current, bool evacuation);
-  ShenandoahHeapRegion* get_next_region(bool evacuation);
-  ShenandoahHeapRegion* get_current_region(bool evacuation);
+  ShenandoahHeapRegion* check_skip_humonguous(ShenandoahHeapRegion* region);
+  ShenandoahHeapRegion* get_next_region_skip_humonguous();
+  ShenandoahHeapRegion* get_current_region_skip_humonguous();
+  ShenandoahHeapRegion* check_grow_heap(ShenandoahHeapRegion* current);
+  ShenandoahHeapRegion* get_next_region();
+  ShenandoahHeapRegion* get_current_region();
 
   void set_from_region_protection(bool protect);
 };
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahHeapRegionSet.cpp	Thu Jan 22 13:51:37 2015 +0100
+++ b/src/share/vm/gc_implementation/shenandoah/shenandoahHeapRegionSet.cpp	Thu Jan 22 13:52:02 2015 +0100
@@ -15,8 +15,7 @@
   _free_threshold(ShenandoahHeapRegion::RegionSizeBytes / 2) {
 
   _next = &_regions[0];
-  _current_allocation = NULL;
-  _current_evacuation = NULL;
+  _current = NULL;
   _next_free = &_regions[0];
 }
 
@@ -30,8 +29,7 @@
   memcpy(_regions, regions, sizeof(ShenandoahHeapRegion*) * num_regions);
 
   _next = &_regions[0];
-  _current_allocation = NULL;
-  _current_evacuation = NULL;
+  _current = NULL;
   _next_free = &_regions[num_regions];
 }
 
@@ -60,10 +58,10 @@
   else return 0;
 }
 
-ShenandoahHeapRegion* ShenandoahHeapRegionSet::current(bool evacuation) {
-  ShenandoahHeapRegion** current = evacuation ? _current_evacuation : _current_allocation;
+ShenandoahHeapRegion* ShenandoahHeapRegionSet::current() {
+  ShenandoahHeapRegion** current = _current;
   if (current == NULL) {
-    return get_next(evacuation);
+    return get_next();
   } else {
     return *(limit_region(current));
   }
@@ -90,8 +88,7 @@
 }
 
 void ShenandoahHeapRegionSet::clear() {
-  _current_allocation = NULL;
-  _current_evacuation = NULL;
+  _current = NULL;
   _next = _regions;
   _next_free = _regions;
 }
@@ -106,15 +103,11 @@
   }
 }
 
-ShenandoahHeapRegion* ShenandoahHeapRegionSet::get_next(bool evacuation) {
+ShenandoahHeapRegion* ShenandoahHeapRegionSet::get_next() {
 
   ShenandoahHeapRegion** next = _next;
   if (next < _next_free) {
-    if (evacuation) {
-      _current_evacuation = next;
-    } else {
-      _current_allocation = next;
-    }
+    _current = next;
     _next++;
     return *next;
   } else {
@@ -132,11 +125,8 @@
 
 void ShenandoahHeapRegionSet::print() {
   for (ShenandoahHeapRegion** i = _regions; i < _next_free; i++) {
-    if (i == _current_allocation) {
-      tty->print_cr("A->");
-    }
-    if (i == _current_evacuation) {
-      tty->print_cr("E->");
+    if (i == _current) {
+      tty->print_cr("C->");
     }
     if (i == _next) {
       tty->print_cr("N->");
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahHeapRegionSet.hpp	Thu Jan 22 13:51:37 2015 +0100
+++ b/src/share/vm/gc_implementation/shenandoah/shenandoahHeapRegionSet.hpp	Thu Jan 22 13:52:02 2015 +0100
@@ -11,8 +11,7 @@
 private:
   ShenandoahHeapRegion** _regions;
   // current region to be returned from get_next()
-  ShenandoahHeapRegion** _current_allocation;
-  ShenandoahHeapRegion** _current_evacuation;
+  ShenandoahHeapRegion** _current;
   ShenandoahHeapRegion** _next;
 
   // last inserted region.
@@ -57,7 +56,7 @@
   /**
    * Returns a pointer to the current region.
    */
-   ShenandoahHeapRegion* current(bool evacuation);
+   ShenandoahHeapRegion* current();
 
   /**
    * Gets the next region for allocation (from free-list).
@@ -65,7 +64,7 @@
    * increment to the next region, the others will fail and return
    * the region that the succeeding thread got.
    */
-  ShenandoahHeapRegion* get_next(bool evacuation);
+  ShenandoahHeapRegion* get_next();
 
   /**
    * Claims next region for processing. This is implemented to be concurrency-safe.
--- a/src/share/vm/runtime/thread.cpp	Thu Jan 22 13:51:37 2015 +0100
+++ b/src/share/vm/runtime/thread.cpp	Thu Jan 22 13:52:02 2015 +0100
@@ -289,6 +289,8 @@
   _MutexEvent  = ParkEvent::Allocate (this) ;
   _MuxEvent    = ParkEvent::Allocate (this) ;
 
+  _evacuating = false;
+
 #ifdef CHECK_UNHANDLED_OOPS
   if (CheckUnhandledOops) {
     _unhandled_oops = new UnhandledOops(this);
--- a/src/share/vm/runtime/thread.hpp	Thu Jan 22 13:51:37 2015 +0100
+++ b/src/share/vm/runtime/thread.hpp	Thu Jan 22 13:52:02 2015 +0100
@@ -276,6 +276,8 @@
   // ObjectMonitor on which this thread called Object.wait()
   ObjectMonitor* _current_waiting_monitor;
 
+  bool _evacuating;
+
   // Private thread-local objectmonitor list - a simple cache organized as a SLL.
  public:
   ObjectMonitor* omFreeList;
@@ -475,6 +477,14 @@
     _current_waiting_monitor = monitor;
   }
 
+  bool is_evacuating() {
+    return _evacuating;
+  }
+
+  void set_evacuating(bool evacuating) {
+    _evacuating = evacuating;
+  }
+
   // GC support
   // Apply "f->do_oop" to all root oops in "this".
   // Apply "cld_f->do_cld" to CLDs that are otherwise not kept alive.