Mercurial > hg > openjdk > hsx19
changeset 1639:1b0104ab1e5e
Merge
| author | tonyp |
|---|---|
| date | Thu, 19 Aug 2010 14:08:58 -0400 |
| parents | 495caa35b1b5 (current diff) effb55808a18 (diff) |
| children | 09cdb1e1c77b |
| files | |
| diffstat | 18 files changed, 440 insertions(+), 183 deletions(-) [+] |
line wrap: on
line diff
--- a/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.cpp Tue Aug 17 22:52:50 2010 -0700 +++ b/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.cpp Thu Aug 19 14:08:58 2010 -0400 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2009, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -402,6 +402,29 @@ return res; } +void LinearAllocBlock::print_on(outputStream* st) const { + st->print_cr(" LinearAllocBlock: ptr = " PTR_FORMAT ", word_size = " SIZE_FORMAT + ", refillsize = " SIZE_FORMAT ", allocation_size_limit = " SIZE_FORMAT, + _ptr, _word_size, _refillSize, _allocation_size_limit); +} + +void CompactibleFreeListSpace::print_on(outputStream* st) const { + st->print_cr("COMPACTIBLE FREELIST SPACE"); + st->print_cr(" Space:"); + Space::print_on(st); + + st->print_cr("promoInfo:"); + _promoInfo.print_on(st); + + st->print_cr("_smallLinearAllocBlock"); + _smallLinearAllocBlock.print_on(st); + + // dump_memory_block(_smallLinearAllocBlock->_ptr, 128); + + st->print_cr(" _fitStrategy = %s, _adaptive_freelists = %s", + _fitStrategy?"true":"false", _adaptive_freelists?"true":"false"); +} + void CompactibleFreeListSpace::print_indexed_free_lists(outputStream* st) const { reportIndexedFreeListStatistics(); @@ -557,13 +580,15 @@ void CompactibleFreeListSpace::set_end(HeapWord* value) { HeapWord* prevEnd = end(); assert(prevEnd != value, "unnecessary set_end call"); - assert(prevEnd == NULL || value >= unallocated_block(), "New end is below unallocated block"); + assert(prevEnd == NULL || !BlockOffsetArrayUseUnallocatedBlock || value >= unallocated_block(), + "New end is below unallocated block"); _end = value; if (prevEnd != NULL) { // Resize the underlying block offset table. _bt.resize(pointer_delta(value, bottom())); if (value <= prevEnd) { - assert(value >= unallocated_block(), "New end is below unallocated block"); + assert(!BlockOffsetArrayUseUnallocatedBlock || value >= unallocated_block(), + "New end is below unallocated block"); } else { // Now, take this new chunk and add it to the free blocks. // Note that the BOT has not yet been updated for this block. @@ -938,7 +963,6 @@ size_t CompactibleFreeListSpace::block_size(const HeapWord* p) const { NOT_PRODUCT(verify_objects_initialized()); - assert(MemRegion(bottom(), end()).contains(p), "p not in space"); // This must be volatile, or else there is a danger that the compiler // will compile the code below into a sometimes-infinite loop, by keeping // the value read the first time in a register. @@ -957,7 +981,7 @@ // must read from what 'p' points to in each loop. klassOop k = ((volatile oopDesc*)p)->klass_or_null(); if (k != NULL) { - assert(k->is_oop(true /* ignore mark word */), "Should really be klass oop."); + assert(k->is_oop(true /* ignore mark word */), "Should be klass oop"); oop o = (oop)p; assert(o->is_parsable(), "Should be parsable"); assert(o->is_oop(true /* ignore mark word */), "Should be an oop."); @@ -1231,7 +1255,6 @@ // satisfy the request. This is different that // evm. // Don't record chunk off a LinAB? smallSplitBirth(size); - } else { // Raid the exact free lists larger than size, even if they are not // overpopulated. @@ -1449,6 +1472,7 @@ // Update BOT last so that other (parallel) GC threads see a consistent // view of the BOT and free blocks. // Above must occur before BOT is updated below. + OrderAccess::storestore(); _bt.split_block(res, blk_size, size); // adjust block offset table } return res; @@ -1477,6 +1501,7 @@ // Update BOT last so that other (parallel) GC threads see a consistent // view of the BOT and free blocks. // Above must occur before BOT is updated below. + OrderAccess::storestore(); _bt.split_block(res, blk_size, size); // adjust block offset table _bt.allocated(res, size); } @@ -1856,6 +1881,8 @@ ffc->linkPrev(NULL); // Mark as a free block for other (parallel) GC threads. // Above must occur before BOT is updated below. // adjust block offset table + OrderAccess::storestore(); + assert(chunk->isFree() && ffc->isFree(), "Error"); _bt.split_block((HeapWord*)chunk, chunk->size(), new_size); if (rem_size < SmallForDictionary) { bool is_par = (SharedHeap::heap()->n_par_threads() > 0); @@ -1911,8 +1938,7 @@ // mark the "end" of the used space at the time of this call; // note, however, that promoted objects from this point // on are tracked in the _promoInfo below. - set_saved_mark_word(BlockOffsetArrayUseUnallocatedBlock ? - unallocated_block() : end()); + set_saved_mark_word(unallocated_block()); // inform allocator that promotions should be tracked. assert(_promoInfo.noPromotions(), "_promoInfo inconsistency"); _promoInfo.startTrackingPromotions(); @@ -2238,8 +2264,7 @@ } void CompactibleFreeListSpace::print() const { - tty->print(" CompactibleFreeListSpace"); - Space::print(); + Space::print_on(tty); } void CompactibleFreeListSpace::prepare_for_verify() { @@ -2253,18 +2278,28 @@ private: const CompactibleFreeListSpace* _sp; const MemRegion _span; + HeapWord* _last_addr; + size_t _last_size; + bool _last_was_obj; + bool _last_was_live; public: VerifyAllBlksClosure(const CompactibleFreeListSpace* sp, - MemRegion span) : _sp(sp), _span(span) { } + MemRegion span) : _sp(sp), _span(span), + _last_addr(NULL), _last_size(0), + _last_was_obj(false), _last_was_live(false) { } virtual size_t do_blk(HeapWord* addr) { size_t res; + bool was_obj = false; + bool was_live = false; if (_sp->block_is_obj(addr)) { + was_obj = true; oop p = oop(addr); guarantee(p->is_oop(), "Should be an oop"); res = _sp->adjustObjectSize(p->size()); if (_sp->obj_is_alive(addr)) { + was_live = true; p->verify(); } } else { @@ -2275,7 +2310,20 @@ "Chunk should be on a free list"); } } - guarantee(res != 0, "Livelock: no rank reduction!"); + if (res == 0) { + gclog_or_tty->print_cr("Livelock: no rank reduction!"); + gclog_or_tty->print_cr( + " Current: addr = " PTR_FORMAT ", size = " SIZE_FORMAT ", obj = %s, live = %s \n" + " Previous: addr = " PTR_FORMAT ", size = " SIZE_FORMAT ", obj = %s, live = %s \n", + addr, res, was_obj ?"true":"false", was_live ?"true":"false", + _last_addr, _last_size, _last_was_obj?"true":"false", _last_was_live?"true":"false"); + _sp->print_on(gclog_or_tty); + guarantee(false, "Seppuku!"); + } + _last_addr = addr; + _last_size = res; + _last_was_obj = was_obj; + _last_was_live = was_live; return res; } }; @@ -2521,7 +2569,7 @@ HeapWord* CFLS_LAB::alloc(size_t word_sz) { FreeChunk* res; - word_sz = _cfls->adjustObjectSize(word_sz); + guarantee(word_sz == _cfls->adjustObjectSize(word_sz), "Error"); if (word_sz >= CompactibleFreeListSpace::IndexSetSize) { // This locking manages sync with other large object allocations. MutexLockerEx x(_cfls->parDictionaryAllocLock(), @@ -2667,12 +2715,12 @@ (cur_sz < CompactibleFreeListSpace::IndexSetSize) && (CMSSplitIndexedFreeListBlocks || k <= 1); k++, cur_sz = k * word_sz) { - FreeList* gfl = &_indexedFreeList[cur_sz]; FreeList fl_for_cur_sz; // Empty. fl_for_cur_sz.set_size(cur_sz); { MutexLockerEx x(_indexedFreeListParLocks[cur_sz], Mutex::_no_safepoint_check_flag); + FreeList* gfl = &_indexedFreeList[cur_sz]; if (gfl->count() != 0) { // nn is the number of chunks of size cur_sz that // we'd need to split k-ways each, in order to create @@ -2685,9 +2733,9 @@ // we increment the split death count by the number of blocks // we just took from the cur_sz-size blocks list and which // we will be splitting below. - ssize_t deaths = _indexedFreeList[cur_sz].splitDeaths() + + ssize_t deaths = gfl->splitDeaths() + fl_for_cur_sz.count(); - _indexedFreeList[cur_sz].set_splitDeaths(deaths); + gfl->set_splitDeaths(deaths); } } } @@ -2703,18 +2751,25 @@ // access the main chunk sees it as a single free block until we // change it. size_t fc_size = fc->size(); + assert(fc->isFree(), "Error"); for (int i = k-1; i >= 0; i--) { FreeChunk* ffc = (FreeChunk*)((HeapWord*)fc + i * word_sz); + assert((i != 0) || + ((fc == ffc) && ffc->isFree() && + (ffc->size() == k*word_sz) && (fc_size == word_sz)), + "Counting error"); ffc->setSize(word_sz); + ffc->linkPrev(NULL); // Mark as a free block for other (parallel) GC threads. ffc->linkNext(NULL); - ffc->linkPrev(NULL); // Mark as a free block for other (parallel) GC threads. // Above must occur before BOT is updated below. - // splitting from the right, fc_size == (k - i + 1) * wordsize - _bt.mark_block((HeapWord*)ffc, word_sz); + OrderAccess::storestore(); + // splitting from the right, fc_size == i * word_sz + _bt.mark_block((HeapWord*)ffc, word_sz, true /* reducing */); fc_size -= word_sz; - _bt.verify_not_unallocated((HeapWord*)ffc, ffc->size()); + assert(fc_size == i*word_sz, "Error"); + _bt.verify_not_unallocated((HeapWord*)ffc, word_sz); _bt.verify_single_block((HeapWord*)fc, fc_size); - _bt.verify_single_block((HeapWord*)ffc, ffc->size()); + _bt.verify_single_block((HeapWord*)ffc, word_sz); // Push this on "fl". fl->returnChunkAtHead(ffc); } @@ -2744,7 +2799,7 @@ _dictionary->minSize()), FreeBlockDictionary::atLeast); if (fc != NULL) { - _bt.allocated((HeapWord*)fc, fc->size()); // update _unallocated_blk + _bt.allocated((HeapWord*)fc, fc->size(), true /* reducing */); // update _unallocated_blk dictionary()->dictCensusUpdate(fc->size(), true /*split*/, false /*birth*/); @@ -2754,8 +2809,10 @@ } } if (fc == NULL) return; + // Otherwise, split up that block. assert((ssize_t)n >= 1, "Control point invariant"); - // Otherwise, split up that block. + assert(fc->isFree(), "Error: should be a free block"); + _bt.verify_single_block((HeapWord*)fc, fc->size()); const size_t nn = fc->size() / word_sz; n = MIN2(nn, n); assert((ssize_t)n >= 1, "Control point invariant"); @@ -2773,6 +2830,7 @@ // dictionary and return, leaving "fl" empty. if (n == 0) { returnChunkToDictionary(fc); + assert(fl->count() == 0, "We never allocated any blocks"); return; } @@ -2785,11 +2843,14 @@ size_t prefix_size = n * word_sz; rem_fc = (FreeChunk*)((HeapWord*)fc + prefix_size); rem_fc->setSize(rem); + rem_fc->linkPrev(NULL); // Mark as a free block for other (parallel) GC threads. rem_fc->linkNext(NULL); - rem_fc->linkPrev(NULL); // Mark as a free block for other (parallel) GC threads. // Above must occur before BOT is updated below. assert((ssize_t)n > 0 && prefix_size > 0 && rem_fc > fc, "Error"); + OrderAccess::storestore(); _bt.split_block((HeapWord*)fc, fc->size(), prefix_size); + assert(fc->isFree(), "Error"); + fc->setSize(prefix_size); if (rem >= IndexSetSize) { returnChunkToDictionary(rem_fc); dictionary()->dictCensusUpdate(rem, true /*split*/, true /*birth*/); @@ -2815,11 +2876,12 @@ for (ssize_t i = n-1; i > 0; i--) { FreeChunk* ffc = (FreeChunk*)((HeapWord*)fc + i * word_sz); ffc->setSize(word_sz); + ffc->linkPrev(NULL); // Mark as a free block for other (parallel) GC threads. ffc->linkNext(NULL); - ffc->linkPrev(NULL); // Mark as a free block for other (parallel) GC threads. // Above must occur before BOT is updated below. + OrderAccess::storestore(); // splitting from the right, fc_size == (n - i + 1) * wordsize - _bt.mark_block((HeapWord*)ffc, word_sz); + _bt.mark_block((HeapWord*)ffc, word_sz, true /* reducing */); fc_size -= word_sz; _bt.verify_not_unallocated((HeapWord*)ffc, ffc->size()); _bt.verify_single_block((HeapWord*)ffc, ffc->size()); @@ -2828,9 +2890,11 @@ fl->returnChunkAtHead(ffc); } // First chunk + assert(fc->isFree() && fc->size() == n*word_sz, "Error: should still be a free block"); + // The blocks above should show their new sizes before the first block below fc->setSize(word_sz); + fc->linkPrev(NULL); // idempotent wrt free-ness, see assert above fc->linkNext(NULL); - fc->linkPrev(NULL); _bt.verify_not_unallocated((HeapWord*)fc, fc->size()); _bt.verify_single_block((HeapWord*)fc, fc->size()); fl->returnChunkAtHead(fc);
--- a/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp Tue Aug 17 22:52:50 2010 -0700 +++ b/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp Thu Aug 19 14:08:58 2010 -0400 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2009, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -48,6 +48,8 @@ size_t _word_size; size_t _refillSize; size_t _allocation_size_limit; // largest size that will be allocated + + void print_on(outputStream* st) const; }; // Concrete subclass of CompactibleSpace that implements @@ -249,10 +251,14 @@ size_t numFreeBlocksInIndexedFreeLists() const; // Accessor HeapWord* unallocated_block() const { - HeapWord* ub = _bt.unallocated_block(); - assert(ub >= bottom() && - ub <= end(), "space invariant"); - return ub; + if (BlockOffsetArrayUseUnallocatedBlock) { + HeapWord* ub = _bt.unallocated_block(); + assert(ub >= bottom() && + ub <= end(), "space invariant"); + return ub; + } else { + return end(); + } } void freed(HeapWord* start, size_t size) { _bt.freed(start, size); @@ -476,6 +482,7 @@ // Debugging support void print() const; + void print_on(outputStream* st) const; void prepare_for_verify(); void verify(bool allow_dirty) const; void verifyFreeLists() const PRODUCT_RETURN;
--- a/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp Tue Aug 17 22:52:50 2010 -0700 +++ b/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp Thu Aug 19 14:08:58 2010 -0400 @@ -1019,7 +1019,7 @@ } HeapWord* ConcurrentMarkSweepGeneration::have_lock_and_allocate(size_t size, - bool tlab) { + bool tlab /* ignored */) { assert_lock_strong(freelistLock()); size_t adjustedSize = CompactibleFreeListSpace::adjustObjectSize(size); HeapWord* res = cmsSpace()->allocate(adjustedSize); @@ -1032,6 +1032,11 @@ // allowing the object to be blackened (and its references scanned) // either during a preclean phase or at the final checkpoint. if (res != NULL) { + // We may block here with an uninitialized object with + // its mark-bit or P-bits not yet set. Such objects need + // to be safely navigable by block_start(). + assert(oop(res)->klass_or_null() == NULL, "Object should be uninitialized here."); + assert(!((FreeChunk*)res)->isFree(), "Error, block will look free but show wrong size"); collector()->direct_allocated(res, adjustedSize); _direct_allocated_words += adjustedSize; // allocation counters @@ -1061,8 +1066,14 @@ // [see comments preceding SweepClosure::do_blk() below for details] // 1. need to mark the object as live so it isn't collected // 2. need to mark the 2nd bit to indicate the object may be uninitialized - // 3. need to mark the end of the object so sweeper can skip over it - // if it's uninitialized when the sweeper reaches it. + // 3. need to mark the end of the object so marking, precleaning or sweeping + // can skip over uninitialized or unparsable objects. An allocated + // object is considered uninitialized for our purposes as long as + // its klass word is NULL. (Unparsable objects are those which are + // initialized in the sense just described, but whose sizes can still + // not be correctly determined. Note that the class of unparsable objects + // can only occur in the perm gen. All old gen objects are parsable + // as soon as they are initialized.) _markBitMap.mark(start); // object is live _markBitMap.mark(start + 1); // object is potentially uninitialized? _markBitMap.mark(start + size - 1); @@ -1088,7 +1099,13 @@ // We don't need to mark the object as uninitialized (as // in direct_allocated above) because this is being done with the // world stopped and the object will be initialized by the - // time the sweeper gets to look at it. + // time the marking, precleaning or sweeping get to look at it. + // But see the code for copying objects into the CMS generation, + // where we need to ensure that concurrent readers of the + // block offset table are able to safely navigate a block that + // is in flux from being free to being allocated (and in + // transition while being copied into) and subsequently + // becoming a bona-fide object when the copy/promotion is complete. assert(SafepointSynchronize::is_at_safepoint(), "expect promotion only at safepoints"); @@ -1304,6 +1321,48 @@ return collector()->allocation_limit_reached(space, top, word_sz); } +// IMPORTANT: Notes on object size recognition in CMS. +// --------------------------------------------------- +// A block of storage in the CMS generation is always in +// one of three states. A free block (FREE), an allocated +// object (OBJECT) whose size() method reports the correct size, +// and an intermediate state (TRANSIENT) in which its size cannot +// be accurately determined. +// STATE IDENTIFICATION: (32 bit and 64 bit w/o COOPS) +// ----------------------------------------------------- +// FREE: klass_word & 1 == 1; mark_word holds block size +// +// OBJECT: klass_word installed; klass_word != 0 && klass_word & 0 == 0; +// obj->size() computes correct size +// [Perm Gen objects needs to be "parsable" before they can be navigated] +// +// TRANSIENT: klass_word == 0; size is indeterminate until we become an OBJECT +// +// STATE IDENTIFICATION: (64 bit+COOPS) +// ------------------------------------ +// FREE: mark_word & CMS_FREE_BIT == 1; mark_word & ~CMS_FREE_BIT gives block_size +// +// OBJECT: klass_word installed; klass_word != 0; +// obj->size() computes correct size +// [Perm Gen comment above continues to hold] +// +// TRANSIENT: klass_word == 0; size is indeterminate until we become an OBJECT +// +// +// STATE TRANSITION DIAGRAM +// +// mut / parnew mut / parnew +// FREE --------------------> TRANSIENT ---------------------> OBJECT --| +// ^ | +// |------------------------ DEAD <------------------------------------| +// sweep mut +// +// While a block is in TRANSIENT state its size cannot be determined +// so readers will either need to come back later or stall until +// the size can be determined. Note that for the case of direct +// allocation, P-bits, when available, may be used to determine the +// size of an object that may not yet have been initialized. + // Things to support parallel young-gen collection. oop ConcurrentMarkSweepGeneration::par_promote(int thread_num, @@ -1331,33 +1390,39 @@ } } assert(promoInfo->has_spooling_space(), "Control point invariant"); - HeapWord* obj_ptr = ps->lab.alloc(word_sz); + const size_t alloc_sz = CompactibleFreeListSpace::adjustObjectSize(word_sz); + HeapWord* obj_ptr = ps->lab.alloc(alloc_sz); if (obj_ptr == NULL) { - obj_ptr = expand_and_par_lab_allocate(ps, word_sz); + obj_ptr = expand_and_par_lab_allocate(ps, alloc_sz); if (obj_ptr == NULL) { return NULL; } } oop obj = oop(obj_ptr); + OrderAccess::storestore(); assert(obj->klass_or_null() == NULL, "Object should be uninitialized here."); + assert(!((FreeChunk*)obj_ptr)->isFree(), "Error, block will look free but show wrong size"); + // IMPORTANT: See note on object initialization for CMS above. // Otherwise, copy the object. Here we must be careful to insert the // klass pointer last, since this marks the block as an allocated object. // Except with compressed oops it's the mark word. HeapWord* old_ptr = (HeapWord*)old; + // Restore the mark word copied above. + obj->set_mark(m); + assert(obj->klass_or_null() == NULL, "Object should be uninitialized here."); + assert(!((FreeChunk*)obj_ptr)->isFree(), "Error, block will look free but show wrong size"); + OrderAccess::storestore(); + + if (UseCompressedOops) { + // Copy gap missed by (aligned) header size calculation below + obj->set_klass_gap(old->klass_gap()); + } if (word_sz > (size_t)oopDesc::header_size()) { Copy::aligned_disjoint_words(old_ptr + oopDesc::header_size(), obj_ptr + oopDesc::header_size(), word_sz - oopDesc::header_size()); } - if (UseCompressedOops) { - // Copy gap missed by (aligned) header size calculation above - obj->set_klass_gap(old->klass_gap()); - } - - // Restore the mark word copied above. - obj->set_mark(m); - // Now we can track the promoted object, if necessary. We take care // to delay the transition from uninitialized to full object // (i.e., insertion of klass pointer) until after, so that it @@ -1365,18 +1430,22 @@ if (promoInfo->tracking()) { promoInfo->track((PromotedObject*)obj, old->klass()); } + assert(obj->klass_or_null() == NULL, "Object should be uninitialized here."); + assert(!((FreeChunk*)obj_ptr)->isFree(), "Error, block will look free but show wrong size"); + assert(old->is_oop(), "Will use and dereference old klass ptr below"); // Finally, install the klass pointer (this should be volatile). + OrderAccess::storestore(); obj->set_klass(old->klass()); - - assert(old->is_oop(), "Will dereference klass ptr below"); + // We should now be able to calculate the right size for this object + assert(obj->is_oop() && obj->size() == (int)word_sz, "Error, incorrect size computed for promoted object"); + collector()->promoted(true, // parallel obj_ptr, old->is_objArray(), word_sz); NOT_PRODUCT( - Atomic::inc(&_numObjectsPromoted); - Atomic::add((jint)CompactibleFreeListSpace::adjustObjectSize(obj->size()), - &_numWordsPromoted); + Atomic::inc_ptr(&_numObjectsPromoted); + Atomic::add_ptr(alloc_sz, &_numWordsPromoted); ) return obj; @@ -7868,14 +7937,20 @@ FreeChunk* fc = (FreeChunk*)addr; size_t res; - // check if we are done sweepinrg - if (addr == _limit) { // we have swept up to the limit, do nothing more + // Check if we are done sweeping. Below we check "addr >= _limit" rather + // than "addr == _limit" because although _limit was a block boundary when + // we started the sweep, it may no longer be one because heap expansion + // may have caused us to coalesce the block ending at the address _limit + // with a newly expanded chunk (this happens when _limit was set to the + // previous _end of the space), so we may have stepped past _limit; see CR 6977970. + if (addr >= _limit) { // we have swept up to or past the limit, do nothing more assert(_limit >= _sp->bottom() && _limit <= _sp->end(), "sweep _limit out of bounds"); + assert(addr < _sp->end(), "addr out of bounds"); // help the closure application finish - return pointer_delta(_sp->end(), _limit); - } - assert(addr <= _limit, "sweep invariant"); + return pointer_delta(_sp->end(), addr); + } + assert(addr < _limit, "sweep invariant"); // check if we should yield do_yield_check(addr);
--- a/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp Tue Aug 17 22:52:50 2010 -0700 +++ b/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp Thu Aug 19 14:08:58 2010 -0400 @@ -1010,10 +1010,10 @@ // Non-product stat counters NOT_PRODUCT( - int _numObjectsPromoted; - int _numWordsPromoted; - int _numObjectsAllocated; - int _numWordsAllocated; + size_t _numObjectsPromoted; + size_t _numWordsPromoted; + size_t _numObjectsAllocated; + size_t _numWordsAllocated; ) // Used for sizing decisions
--- a/src/share/vm/gc_implementation/concurrentMarkSweep/freeChunk.hpp Tue Aug 17 22:52:50 2010 -0700 +++ b/src/share/vm/gc_implementation/concurrentMarkSweep/freeChunk.hpp Thu Aug 19 14:08:58 2010 -0400 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2008, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -110,15 +110,21 @@ } void linkNext(FreeChunk* ptr) { _next = ptr; } void linkPrev(FreeChunk* ptr) { - LP64_ONLY(if (UseCompressedOops) _prev = ptr; else) - _prev = (FreeChunk*)((intptr_t)ptr | 0x1); + LP64_ONLY(if (UseCompressedOops) _prev = ptr; else) + _prev = (FreeChunk*)((intptr_t)ptr | 0x1); } void clearPrev() { _prev = NULL; } void clearNext() { _next = NULL; } void markNotFree() { - LP64_ONLY(if (UseCompressedOops) set_mark(markOopDesc::prototype());) - // Also set _prev to null - _prev = NULL; + // Set _prev (klass) to null before (if) clearing the mark word below + _prev = NULL; +#ifdef _LP64 + if (UseCompressedOops) { + OrderAccess::storestore(); + set_mark(markOopDesc::prototype()); + } +#endif + assert(!isFree(), "Error"); } // Return the address past the end of this chunk
--- a/src/share/vm/gc_implementation/concurrentMarkSweep/promotionInfo.cpp Tue Aug 17 22:52:50 2010 -0700 +++ b/src/share/vm/gc_implementation/concurrentMarkSweep/promotionInfo.cpp Thu Aug 19 14:08:58 2010 -0400 @@ -330,7 +330,7 @@ void PromotionInfo::print_on(outputStream* st) const { SpoolBlock* curSpool = NULL; size_t i = 0; - st->print_cr("start & end indices: [" SIZE_FORMAT ", " SIZE_FORMAT ")", + st->print_cr(" start & end indices: [" SIZE_FORMAT ", " SIZE_FORMAT ")", _firstIndex, _nextIndex); for (curSpool = _spoolHead; curSpool != _spoolTail && curSpool != NULL; curSpool = curSpool->nextSpoolBlock) { @@ -350,7 +350,7 @@ st->print_cr(" free "); i++; } - st->print_cr(SIZE_FORMAT " header spooling blocks", i); + st->print_cr(" " SIZE_FORMAT " header spooling blocks", i); } void SpoolBlock::print_on(outputStream* st) const {
--- a/src/share/vm/gc_implementation/g1/concurrentMark.cpp Tue Aug 17 22:52:50 2010 -0700 +++ b/src/share/vm/gc_implementation/g1/concurrentMark.cpp Thu Aug 19 14:08:58 2010 -0400 @@ -2586,9 +2586,6 @@ double end_time = os::elapsedTime(); double elapsed_time_ms = (end_time - start) * 1000.0; g1h->g1_policy()->record_mark_closure_time(elapsed_time_ms); - if (PrintGCDetails) { - gclog_or_tty->print_cr("Mark closure took %5.2f ms.", elapsed_time_ms); - } ClearMarksInHRClosure clr(nextMarkBitMap()); g1h->collection_set_iterate(&clr);
--- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Tue Aug 17 22:52:50 2010 -0700 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Thu Aug 19 14:08:58 2010 -0400 @@ -1044,29 +1044,56 @@ const size_t capacity_after_gc = capacity(); const size_t free_after_gc = capacity_after_gc - used_after_gc; + // This is enforced in arguments.cpp. + assert(MinHeapFreeRatio <= MaxHeapFreeRatio, + "otherwise the code below doesn't make sense"); + // We don't have floating point command-line arguments - const double minimum_free_percentage = (double) MinHeapFreeRatio / 100; + const double minimum_free_percentage = (double) MinHeapFreeRatio / 100.0; const double maximum_used_percentage = 1.0 - minimum_free_percentage; - const double maximum_free_percentage = (double) MaxHeapFreeRatio / 100; + const double maximum_free_percentage = (double) MaxHeapFreeRatio / 100.0; const double minimum_used_percentage = 1.0 - maximum_free_percentage; - size_t minimum_desired_capacity = (size_t) (used_after_gc / maximum_used_percentage); - size_t maximum_desired_capacity = (size_t) (used_after_gc / minimum_used_percentage); - - // Don't shrink less than the initial size. - minimum_desired_capacity = - MAX2(minimum_desired_capacity, - collector_policy()->initial_heap_byte_size()); - maximum_desired_capacity = - MAX2(maximum_desired_capacity, - collector_policy()->initial_heap_byte_size()); - - // We are failing here because minimum_desired_capacity is - assert(used_after_gc <= minimum_desired_capacity, "sanity check"); - assert(minimum_desired_capacity <= maximum_desired_capacity, "sanity check"); + const size_t min_heap_size = collector_policy()->min_heap_byte_size(); + const size_t max_heap_size = collector_policy()->max_heap_byte_size(); + + // We have to be careful here as these two calculations can overflow + // 32-bit size_t's. + double used_after_gc_d = (double) used_after_gc; + double minimum_desired_capacity_d = used_after_gc_d / maximum_used_percentage; + double maximum_desired_capacity_d = used_after_gc_d / minimum_used_percentage; + + // Let's make sure that they are both under the max heap size, which + // by default will make them fit into a size_t. + double desired_capacity_upper_bound = (double) max_heap_size; + minimum_desired_capacity_d = MIN2(minimum_desired_capacity_d, + desired_capacity_upper_bound); + maximum_desired_capacity_d = MIN2(maximum_desired_capacity_d, + desired_capacity_upper_bound); + + // We can now safely turn them into size_t's. + size_t minimum_desired_capacity = (size_t) minimum_desired_capacity_d; + size_t maximum_desired_capacity = (size_t) maximum_desired_capacity_d; + + // This assert only makes sense here, before we adjust them + // with respect to the min and max heap size. + assert(minimum_desired_capacity <= maximum_desired_capacity, + err_msg("minimum_desired_capacity = "SIZE_FORMAT", " + "maximum_desired_capacity = "SIZE_FORMAT, + minimum_desired_capacity, maximum_desired_capacity)); + + // Should not be greater than the heap max size. No need to adjust + // it with respect to the heap min size as it's a lower bound (i.e., + // we'll try to make the capacity larger than it, not smaller). + minimum_desired_capacity = MIN2(minimum_desired_capacity, max_heap_size); + // Should not be less than the heap min size. No need to adjust it + // with respect to the heap max size as it's an upper bound (i.e., + // we'll try to make the capacity smaller than it, not greater). + maximum_desired_capacity = MAX2(maximum_desired_capacity, min_heap_size); if (PrintGC && Verbose) { - const double free_percentage = ((double)free_after_gc) / capacity(); + const double free_percentage = + (double) free_after_gc / (double) capacity_after_gc; gclog_or_tty->print_cr("Computing new size after full GC "); gclog_or_tty->print_cr(" " " minimum_free_percentage: %6.2f", @@ -1078,45 +1105,47 @@ " capacity: %6.1fK" " minimum_desired_capacity: %6.1fK" " maximum_desired_capacity: %6.1fK", - capacity() / (double) K, - minimum_desired_capacity / (double) K, - maximum_desired_capacity / (double) K); + (double) capacity_after_gc / (double) K, + (double) minimum_desired_capacity / (double) K, + (double) maximum_desired_capacity / (double) K); gclog_or_tty->print_cr(" " - " free_after_gc : %6.1fK" - " used_after_gc : %6.1fK", - free_after_gc / (double) K, - used_after_gc / (double) K); + " free_after_gc: %6.1fK" + " used_after_gc: %6.1fK", + (double) free_after_gc / (double) K, + (double) used_after_gc / (double) K); gclog_or_tty->print_cr(" " " free_percentage: %6.2f", free_percentage); } - if (capacity() < minimum_desired_capacity) { + if (capacity_after_gc < minimum_desired_capacity) { // Don't expand unless it's significant size_t expand_bytes = minimum_desired_capacity - capacity_after_gc; expand(expand_bytes); if (PrintGC && Verbose) { - gclog_or_tty->print_cr(" expanding:" + gclog_or_tty->print_cr(" " + " expanding:" + " max_heap_size: %6.1fK" " minimum_desired_capacity: %6.1fK" " expand_bytes: %6.1fK", - minimum_desired_capacity / (double) K, - expand_bytes / (double) K); + (double) max_heap_size / (double) K, + (double) minimum_desired_capacity / (double) K, + (double) expand_bytes / (double) K); } // No expansion, now see if we want to shrink - } else if (capacity() > maximum_desired_capacity) { + } else if (capacity_after_gc > maximum_desired_capacity) { // Capacity too large, compute shrinking size size_t shrink_bytes = capacity_after_gc - maximum_desired_capacity; shrink(shrink_bytes); if (PrintGC && Verbose) { gclog_or_tty->print_cr(" " " shrinking:" - " initSize: %.1fK" - " maximum_desired_capacity: %.1fK", - collector_policy()->initial_heap_byte_size() / (double) K, - maximum_desired_capacity / (double) K); - gclog_or_tty->print_cr(" " - " shrink_bytes: %.1fK", - shrink_bytes / (double) K); + " min_heap_size: %6.1fK" + " maximum_desired_capacity: %6.1fK" + " shrink_bytes: %6.1fK", + (double) min_heap_size / (double) K, + (double) maximum_desired_capacity / (double) K, + (double) shrink_bytes / (double) K); } } } @@ -2165,9 +2194,12 @@ } } -HeapWord* G1CollectedHeap::allocate_new_tlab(size_t size) { +HeapWord* G1CollectedHeap::allocate_new_tlab(size_t word_size) { + assert(!isHumongous(word_size), + err_msg("a TLAB should not be of humongous size, " + "word_size = "SIZE_FORMAT, word_size)); bool dummy; - return G1CollectedHeap::mem_allocate(size, false, true, &dummy); + return G1CollectedHeap::mem_allocate(word_size, false, true, &dummy); } bool G1CollectedHeap::allocs_are_zero_filled() { @@ -3576,7 +3608,7 @@ if (!r->evacuation_failed()) { r->set_evacuation_failed(true); if (G1PrintHeapRegions) { - gclog_or_tty->print("evacuation failed in heap region "PTR_FORMAT" " + gclog_or_tty->print("overflow in heap region "PTR_FORMAT" " "["PTR_FORMAT","PTR_FORMAT")\n", r, r->bottom(), r->end()); } @@ -3610,6 +3642,10 @@ HeapWord* G1CollectedHeap::par_allocate_during_gc(GCAllocPurpose purpose, size_t word_size) { + assert(!isHumongous(word_size), + err_msg("we should not be seeing humongous allocation requests " + "during GC, word_size = "SIZE_FORMAT, word_size)); + HeapRegion* alloc_region = _gc_alloc_regions[purpose]; // let the caller handle alloc failure if (alloc_region == NULL) return NULL; @@ -3642,6 +3678,10 @@ HeapRegion* alloc_region, bool par, size_t word_size) { + assert(!isHumongous(word_size), + err_msg("we should not be seeing humongous allocation requests " + "during GC, word_size = "SIZE_FORMAT, word_size)); + HeapWord* block = NULL; // In the parallel case, a previous thread to obtain the lock may have // already assigned a new gc_alloc_region. @@ -4281,7 +4321,7 @@ if (evacuation_failed()) { remove_self_forwarding_pointers(); if (PrintGCDetails) { - gclog_or_tty->print(" (evacuation failed)"); + gclog_or_tty->print(" (to-space overflow)"); } else if (PrintGC) { gclog_or_tty->print("--"); }
--- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Tue Aug 17 22:52:50 2010 -0700 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Thu Aug 19 14:08:58 2010 -0400 @@ -1032,7 +1032,7 @@ virtual bool supports_tlab_allocation() const; virtual size_t tlab_capacity(Thread* thr) const; virtual size_t unsafe_max_tlab_alloc(Thread* thr) const; - virtual HeapWord* allocate_new_tlab(size_t size); + virtual HeapWord* allocate_new_tlab(size_t word_size); // Can a compiler initialize a new object without store barriers? // This permission only extends from the creation of a new object
--- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp Tue Aug 17 22:52:50 2010 -0700 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp Thu Aug 19 14:08:58 2010 -0400 @@ -57,8 +57,9 @@ assert( SafepointSynchronize::is_at_safepoint() || Heap_lock->owned_by_self(), "pre-condition of the call" ); - if (_cur_alloc_region != NULL) { - + // All humongous allocation requests should go through the slow path in + // attempt_allocation_slow(). + if (!isHumongous(word_size) && _cur_alloc_region != NULL) { // If this allocation causes a region to become non empty, // then we need to update our free_regions count. @@ -69,13 +70,14 @@ } else { res = _cur_alloc_region->allocate(word_size); } - } - if (res != NULL) { - if (!SafepointSynchronize::is_at_safepoint()) { - assert( Heap_lock->owned_by_self(), "invariant" ); - Heap_lock->unlock(); + + if (res != NULL) { + if (!SafepointSynchronize::is_at_safepoint()) { + assert( Heap_lock->owned_by_self(), "invariant" ); + Heap_lock->unlock(); + } + return res; } - return res; } // attempt_allocation_slow will also unlock the heap lock when appropriate. return attempt_allocation_slow(word_size, permit_collection_pause);
--- a/src/share/vm/gc_implementation/g1/heapRegion.cpp Tue Aug 17 22:52:50 2010 -0700 +++ b/src/share/vm/gc_implementation/g1/heapRegion.cpp Thu Aug 19 14:08:58 2010 -0400 @@ -790,8 +790,18 @@ int objs = 0; int blocks = 0; VerifyLiveClosure vl_cl(g1, use_prev_marking); + bool is_humongous = isHumongous(); + size_t object_num = 0; while (p < top()) { size_t size = oop(p)->size(); + if (is_humongous != g1->isHumongous(size)) { + gclog_or_tty->print_cr("obj "PTR_FORMAT" is of %shumongous size (" + SIZE_FORMAT" words) in a %shumongous region", + p, g1->isHumongous(size) ? "" : "non-", + size, is_humongous ? "" : "non-"); + *failures = true; + } + object_num += 1; if (blocks == BLOCK_SAMPLE_INTERVAL) { HeapWord* res = block_start_const(p + (size/2)); if (p != res) { @@ -857,6 +867,13 @@ } } + if (is_humongous && object_num > 1) { + gclog_or_tty->print_cr("region ["PTR_FORMAT","PTR_FORMAT"] is humongous " + "but has "SIZE_FORMAT", objects", + bottom(), end(), object_num); + *failures = true; + } + if (p != top()) { gclog_or_tty->print_cr("end of last object "PTR_FORMAT" " "does not match top "PTR_FORMAT, p, top());
--- a/src/share/vm/gc_implementation/includeDB_gc_concurrentMarkSweep Tue Aug 17 22:52:50 2010 -0700 +++ b/src/share/vm/gc_implementation/includeDB_gc_concurrentMarkSweep Thu Aug 19 14:08:58 2010 -0400 @@ -1,5 +1,5 @@ // -// Copyright (c) 2004, 2009, Oracle and/or its affiliates. All rights reserved. +// Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // // This code is free software; you can redistribute it and/or modify it @@ -34,6 +34,8 @@ binaryTreeDictionary.hpp freeBlockDictionary.hpp binaryTreeDictionary.hpp freeList.hpp +blockOffsetTable.inline.hpp concurrentMarkSweepGeneration.hpp + cmsAdaptiveSizePolicy.cpp cmsAdaptiveSizePolicy.hpp cmsAdaptiveSizePolicy.cpp defNewGeneration.hpp cmsAdaptiveSizePolicy.cpp gcStats.hpp @@ -85,7 +87,7 @@ cmsOopClosures.inline.hpp cmsOopClosures.hpp cmsOopClosures.inline.hpp concurrentMarkSweepGeneration.hpp -cmsPermGen.cpp blockOffsetTable.hpp +cmsPermGen.cpp blockOffsetTable.inline.hpp cmsPermGen.cpp cSpaceCounters.hpp cmsPermGen.cpp cmsPermGen.hpp cmsPermGen.cpp collectedHeap.inline.hpp @@ -121,6 +123,7 @@ compactibleFreeListSpace.cpp vmThread.hpp compactibleFreeListSpace.hpp binaryTreeDictionary.hpp +compactibleFreeListSpace.hpp blockOffsetTable.inline.hpp compactibleFreeListSpace.hpp freeList.hpp compactibleFreeListSpace.hpp promotionInfo.hpp compactibleFreeListSpace.hpp space.hpp
--- a/src/share/vm/includeDB_core Tue Aug 17 22:52:50 2010 -0700 +++ b/src/share/vm/includeDB_core Thu Aug 19 14:08:58 2010 -0400 @@ -225,7 +225,6 @@ arrayOop.cpp symbolOop.hpp arrayOop.hpp oop.hpp -arrayOop.hpp universe.hpp arrayOop.hpp universe.inline.hpp assembler.cpp assembler.hpp @@ -236,7 +235,6 @@ assembler.cpp os.hpp assembler.hpp allocation.hpp -assembler.hpp allocation.inline.hpp assembler.hpp debug.hpp assembler.hpp growableArray.hpp assembler.hpp oopRecorder.hpp @@ -330,7 +328,7 @@ blockOffsetTable.cpp iterator.hpp blockOffsetTable.cpp java.hpp blockOffsetTable.cpp oop.inline.hpp -blockOffsetTable.cpp space.hpp +blockOffsetTable.cpp space.inline.hpp blockOffsetTable.cpp universe.hpp blockOffsetTable.hpp globalDefinitions.hpp @@ -338,6 +336,7 @@ blockOffsetTable.hpp virtualspace.hpp blockOffsetTable.inline.hpp blockOffsetTable.hpp +blockOffsetTable.inline.hpp safepoint.hpp blockOffsetTable.inline.hpp space.hpp bytecode.cpp bytecode.hpp @@ -1807,7 +1806,7 @@ generateOopMap.hpp universe.inline.hpp generation.cpp allocation.inline.hpp -generation.cpp blockOffsetTable.hpp +generation.cpp blockOffsetTable.inline.hpp generation.cpp cardTableRS.hpp generation.cpp collectedHeap.inline.hpp generation.cpp copy.hpp @@ -3436,7 +3435,7 @@ perfMemory_<os_family>.cpp resourceArea.hpp perfMemory_<os_family>.cpp vmSymbols.hpp -permGen.cpp blockOffsetTable.hpp +permGen.cpp blockOffsetTable.inline.hpp permGen.cpp cSpaceCounters.hpp permGen.cpp collectedHeap.inline.hpp permGen.cpp compactPermGen.hpp @@ -3805,7 +3804,7 @@ sizes.hpp allocation.hpp sizes.hpp globalDefinitions.hpp -space.cpp blockOffsetTable.hpp +space.cpp blockOffsetTable.inline.hpp space.cpp copy.hpp space.cpp defNewGeneration.hpp space.cpp genCollectedHeap.hpp @@ -3835,7 +3834,6 @@ space.hpp watermark.hpp space.hpp workgroup.hpp -space.inline.hpp blockOffsetTable.inline.hpp space.inline.hpp collectedHeap.hpp space.inline.hpp safepoint.hpp space.inline.hpp space.hpp
--- a/src/share/vm/memory/allocation.cpp Tue Aug 17 22:52:50 2010 -0700 +++ b/src/share/vm/memory/allocation.cpp Thu Aug 19 14:08:58 2010 -0400 @@ -58,7 +58,7 @@ void ResourceObj::operator delete(void* p) { assert(((ResourceObj *)p)->allocated_on_C_heap(), "delete only allowed for C_HEAP objects"); - DEBUG_ONLY(((ResourceObj *)p)->_allocation = badHeapOopVal;) + DEBUG_ONLY(((ResourceObj *)p)->_allocation = (uintptr_t) badHeapOopVal;) FreeHeap(p); } @@ -104,7 +104,7 @@ ResourceObj::~ResourceObj() { // allocated_on_C_heap() also checks that encoded (in _allocation) address == this. if (!allocated_on_C_heap()) { // ResourceObj::delete() zaps _allocation for C_heap. - _allocation = badHeapOopVal; // zap type + _allocation = (uintptr_t) badHeapOopVal; // zap type } } #endif // ASSERT
--- a/src/share/vm/memory/blockOffsetTable.cpp Tue Aug 17 22:52:50 2010 -0700 +++ b/src/share/vm/memory/blockOffsetTable.cpp Thu Aug 19 14:08:58 2010 -0400 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2006, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -103,13 +103,13 @@ ////////////////////////////////////////////////////////////////////// BlockOffsetArray::BlockOffsetArray(BlockOffsetSharedArray* array, - MemRegion mr, bool init_to_zero) : + MemRegion mr, bool init_to_zero_) : BlockOffsetTable(mr.start(), mr.end()), - _array(array), - _init_to_zero(init_to_zero) + _array(array) { assert(_bottom <= _end, "arguments out of order"); - if (!_init_to_zero) { + set_init_to_zero(init_to_zero_); + if (!init_to_zero_) { // initialize cards to point back to mr.start() set_remainder_to_point_to_start(mr.start() + N_words, mr.end()); _array->set_offset_array(0, 0); // set first card to 0 @@ -121,8 +121,9 @@ // a right-open interval: [start, end) void BlockOffsetArray:: -set_remainder_to_point_to_start(HeapWord* start, HeapWord* end) { +set_remainder_to_point_to_start(HeapWord* start, HeapWord* end, bool reducing) { + check_reducing_assertion(reducing); if (start >= end) { // The start address is equal to the end address (or to // the right of the end address) so there are not cards @@ -167,7 +168,7 @@ size_t end_card = _array->index_for(end-1); assert(start ==_array->address_for_index(start_card), "Precondition"); assert(end ==_array->address_for_index(end_card)+N_words, "Precondition"); - set_remainder_to_point_to_start_incl(start_card, end_card); // closed interval + set_remainder_to_point_to_start_incl(start_card, end_card, reducing); // closed interval } @@ -175,7 +176,9 @@ // a closed, inclusive interval: [start_card, end_card], cf set_remainder_to_point_to_start() // above. void -BlockOffsetArray::set_remainder_to_point_to_start_incl(size_t start_card, size_t end_card) { +BlockOffsetArray::set_remainder_to_point_to_start_incl(size_t start_card, size_t end_card, bool reducing) { + + check_reducing_assertion(reducing); if (start_card > end_card) { return; } @@ -191,11 +194,11 @@ size_t reach = start_card - 1 + (power_to_cards_back(i+1) - 1); offset = N_words + i; if (reach >= end_card) { - _array->set_offset_array(start_card_for_region, end_card, offset); + _array->set_offset_array(start_card_for_region, end_card, offset, reducing); start_card_for_region = reach + 1; break; } - _array->set_offset_array(start_card_for_region, reach, offset); + _array->set_offset_array(start_card_for_region, reach, offset, reducing); start_card_for_region = reach + 1; } assert(start_card_for_region > end_card, "Sanity check"); @@ -211,8 +214,10 @@ return; } guarantee(_array->offset_array(start_card) == N_words, "Wrong value in second card"); + u_char last_entry = N_words; for (size_t c = start_card + 1; c <= end_card; c++ /* yeah! */) { u_char entry = _array->offset_array(c); + guarantee(entry >= last_entry, "Monotonicity"); if (c - start_card > power_to_cards_back(1)) { guarantee(entry > N_words, "Should be in logarithmic region"); } @@ -220,11 +225,13 @@ size_t landing_card = c - backskip; guarantee(landing_card >= (start_card - 1), "Inv"); if (landing_card >= start_card) { - guarantee(_array->offset_array(landing_card) <= entry, "monotonicity"); + guarantee(_array->offset_array(landing_card) <= entry, "Monotonicity"); } else { - guarantee(landing_card == start_card - 1, "Tautology"); + guarantee(landing_card == (start_card - 1), "Tautology"); + // Note that N_words is the maximum offset value guarantee(_array->offset_array(landing_card) <= N_words, "Offset value"); } + last_entry = entry; // remember for monotonicity test } } @@ -243,7 +250,7 @@ void BlockOffsetArray::do_block_internal(HeapWord* blk_start, HeapWord* blk_end, - Action action) { + Action action, bool reducing) { assert(Universe::heap()->is_in_reserved(blk_start), "reference must be into the heap"); assert(Universe::heap()->is_in_reserved(blk_end-1), @@ -275,18 +282,18 @@ switch (action) { case Action_mark: { if (init_to_zero()) { - _array->set_offset_array(start_index, boundary, blk_start); + _array->set_offset_array(start_index, boundary, blk_start, reducing); break; } // Else fall through to the next case } case Action_single: { - _array->set_offset_array(start_index, boundary, blk_start); + _array->set_offset_array(start_index, boundary, blk_start, reducing); // We have finished marking the "offset card". We need to now // mark the subsequent cards that this blk spans. if (start_index < end_index) { HeapWord* rem_st = _array->address_for_index(start_index) + N_words; HeapWord* rem_end = _array->address_for_index(end_index) + N_words; - set_remainder_to_point_to_start(rem_st, rem_end); + set_remainder_to_point_to_start(rem_st, rem_end, reducing); } break; } @@ -395,7 +402,7 @@ // Indices for starts of prefix block and suffix block. size_t pref_index = _array->index_for(pref_addr); if (_array->address_for_index(pref_index) != pref_addr) { - // pref_addr deos not begin pref_index + // pref_addr does not begin pref_index pref_index++; } @@ -430,18 +437,18 @@ if (num_suff_cards > 0) { HeapWord* boundary = _array->address_for_index(suff_index); // Set the offset card for suffix block - _array->set_offset_array(suff_index, boundary, suff_addr); + _array->set_offset_array(suff_index, boundary, suff_addr, true /* reducing */); // Change any further cards that need changing in the suffix if (num_pref_cards > 0) { if (num_pref_cards >= num_suff_cards) { // Unilaterally fix all of the suffix cards: closed card // index interval in args below. - set_remainder_to_point_to_start_incl(suff_index + 1, end_index - 1); + set_remainder_to_point_to_start_incl(suff_index + 1, end_index - 1, true /* reducing */); } else { // Unilaterally fix the first (num_pref_cards - 1) following // the "offset card" in the suffix block. set_remainder_to_point_to_start_incl(suff_index + 1, - suff_index + num_pref_cards - 1); + suff_index + num_pref_cards - 1, true /* reducing */); // Fix the appropriate cards in the remainder of the // suffix block -- these are the last num_pref_cards // cards in each power block of the "new" range plumbed @@ -461,7 +468,7 @@ // is non-null. if (left_index <= right_index) { _array->set_offset_array(left_index, right_index, - N_words + i - 1); + N_words + i - 1, true /* reducing */); } else { more = false; // we are done } @@ -482,7 +489,7 @@ more = false; } assert(left_index <= right_index, "Error"); - _array->set_offset_array(left_index, right_index, N_words + i - 1); + _array->set_offset_array(left_index, right_index, N_words + i - 1, true /* reducing */); i++; } } @@ -501,14 +508,13 @@ // any cards subsequent to the first one. void BlockOffsetArrayNonContigSpace::mark_block(HeapWord* blk_start, - HeapWord* blk_end) { - do_block_internal(blk_start, blk_end, Action_mark); + HeapWord* blk_end, bool reducing) { + do_block_internal(blk_start, blk_end, Action_mark, reducing); } HeapWord* BlockOffsetArrayNonContigSpace::block_start_unsafe( const void* addr) const { assert(_array->offset_array(0) == 0, "objects can't cross covered areas"); - assert(_bottom <= addr && addr < _end, "addr must be covered by this Array"); // Must read this exactly once because it can be modified by parallel @@ -542,9 +548,10 @@ debug_only(HeapWord* last = q); // for debugging q = n; n += _sp->block_size(n); + assert(n > q, err_msg("Looping at: " INTPTR_FORMAT, n)); } - assert(q <= addr, "wrong order for current and arg"); - assert(addr <= n, "wrong order for arg and next"); + assert(q <= addr, err_msg("wrong order for current (" INTPTR_FORMAT ") <= arg (" INTPTR_FORMAT ")", q, addr)); + assert(addr <= n, err_msg("wrong order for arg (" INTPTR_FORMAT ") <= next (" INTPTR_FORMAT ")", addr, n)); return q; } @@ -727,9 +734,8 @@ _next_offset_index = end_index + 1; // Calculate _next_offset_threshold this way because end_index // may be the last valid index in the covered region. - _next_offset_threshold = _array->address_for_index(end_index) + - N_words; - assert(_next_offset_threshold >= blk_end, "Incorrent offset threshold"); + _next_offset_threshold = _array->address_for_index(end_index) + N_words; + assert(_next_offset_threshold >= blk_end, "Incorrect offset threshold"); #ifdef ASSERT // The offset can be 0 if the block starts on a boundary. That
--- a/src/share/vm/memory/blockOffsetTable.hpp Tue Aug 17 22:52:50 2010 -0700 +++ b/src/share/vm/memory/blockOffsetTable.hpp Thu Aug 19 14:08:58 2010 -0400 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2009, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -107,6 +107,8 @@ N_words = 1 << LogN_words }; + bool _init_to_zero; + // The reserved region covered by the shared array. MemRegion _reserved; @@ -125,17 +127,28 @@ assert(index < _vs.committed_size(), "index out of range"); return _offset_array[index]; } - void set_offset_array(size_t index, u_char offset) { + // An assertion-checking helper method for the set_offset_array() methods below. + void check_reducing_assertion(bool reducing); + + void set_offset_array(size_t index, u_char offset, bool reducing = false) { + check_reducing_assertion(reducing); assert(index < _vs.committed_size(), "index out of range"); + assert(!reducing || _offset_array[index] >= offset, "Not reducing"); _offset_array[index] = offset; } - void set_offset_array(size_t index, HeapWord* high, HeapWord* low) { + + void set_offset_array(size_t index, HeapWord* high, HeapWord* low, bool reducing = false) { + check_reducing_assertion(reducing); assert(index < _vs.committed_size(), "index out of range"); assert(high >= low, "addresses out of order"); assert(pointer_delta(high, low) <= N_words, "offset too large"); + assert(!reducing || _offset_array[index] >= (u_char)pointer_delta(high, low), + "Not reducing"); _offset_array[index] = (u_char)pointer_delta(high, low); } - void set_offset_array(HeapWord* left, HeapWord* right, u_char offset) { + + void set_offset_array(HeapWord* left, HeapWord* right, u_char offset, bool reducing = false) { + check_reducing_assertion(reducing); assert(index_for(right - 1) < _vs.committed_size(), "right address out of range"); assert(left < right, "Heap addresses out of order"); @@ -150,12 +163,14 @@ size_t i = index_for(left); const size_t end = i + num_cards; for (; i < end; i++) { + assert(!reducing || _offset_array[i] >= offset, "Not reducing"); _offset_array[i] = offset; } } } - void set_offset_array(size_t left, size_t right, u_char offset) { + void set_offset_array(size_t left, size_t right, u_char offset, bool reducing = false) { + check_reducing_assertion(reducing); assert(right < _vs.committed_size(), "right address out of range"); assert(left <= right, "indexes out of order"); size_t num_cards = right - left + 1; @@ -169,6 +184,7 @@ size_t i = left; const size_t end = i + num_cards; for (; i < end; i++) { + assert(!reducing || _offset_array[i] >= offset, "Not reducing"); _offset_array[i] = offset; } } @@ -212,6 +228,11 @@ void set_bottom(HeapWord* new_bottom); + // Whether entries should be initialized to zero. Used currently only for + // error checking. + void set_init_to_zero(bool val) { _init_to_zero = val; } + bool init_to_zero() { return _init_to_zero; } + // Updates all the BlockOffsetArray's sharing this shared array to // reflect the current "top"'s of their spaces. void update_offset_arrays(); // Not yet implemented! @@ -285,17 +306,23 @@ // initialized to point backwards to the beginning of the covered region. bool _init_to_zero; + // An assertion-checking helper method for the set_remainder*() methods below. + void check_reducing_assertion(bool reducing) { _array->check_reducing_assertion(reducing); } + // Sets the entries // corresponding to the cards starting at "start" and ending at "end" // to point back to the card before "start": the interval [start, end) - // is right-open. - void set_remainder_to_point_to_start(HeapWord* start, HeapWord* end); + // is right-open. The last parameter, reducing, indicates whether the + // updates to individual entries always reduce the entry from a higher + // to a lower value. (For example this would hold true during a temporal + // regime during which only block splits were updating the BOT. + void set_remainder_to_point_to_start(HeapWord* start, HeapWord* end, bool reducing = false); // Same as above, except that the args here are a card _index_ interval // that is closed: [start_index, end_index] - void set_remainder_to_point_to_start_incl(size_t start, size_t end); + void set_remainder_to_point_to_start_incl(size_t start, size_t end, bool reducing = false); // A helper function for BOT adjustment/verification work - void do_block_internal(HeapWord* blk_start, HeapWord* blk_end, Action action); + void do_block_internal(HeapWord* blk_start, HeapWord* blk_end, Action action, bool reducing = false); public: // The space may not have its bottom and top set yet, which is why the @@ -303,7 +330,7 @@ // elements of the array are initialized to zero. Otherwise, they are // initialized to point backwards to the beginning. BlockOffsetArray(BlockOffsetSharedArray* array, MemRegion mr, - bool init_to_zero); + bool init_to_zero_); // Note: this ought to be part of the constructor, but that would require // "this" to be passed as a parameter to a member constructor for @@ -358,6 +385,12 @@ // If true, initialize array slots with no allocated blocks to zero. // Otherwise, make them point back to the front. bool init_to_zero() { return _init_to_zero; } + // Corresponding setter + void set_init_to_zero(bool val) { + _init_to_zero = val; + assert(_array != NULL, "_array should be non-NULL"); + _array->set_init_to_zero(val); + } // Debugging // Return the index of the last entry in the "active" region. @@ -424,16 +457,16 @@ // of BOT is touched. It is assumed (and verified in the // non-product VM) that the remaining cards of the block // are correct. - void mark_block(HeapWord* blk_start, HeapWord* blk_end); - void mark_block(HeapWord* blk, size_t size) { - mark_block(blk, blk + size); + void mark_block(HeapWord* blk_start, HeapWord* blk_end, bool reducing = false); + void mark_block(HeapWord* blk, size_t size, bool reducing = false) { + mark_block(blk, blk + size, reducing); } // Adjust _unallocated_block to indicate that a particular // block has been newly allocated or freed. It is assumed (and // verified in the non-product VM) that the BOT is correct for // the given block. - void allocated(HeapWord* blk_start, HeapWord* blk_end) { + void allocated(HeapWord* blk_start, HeapWord* blk_end, bool reducing = false) { // Verify that the BOT shows [blk, blk + blk_size) to be one block. verify_single_block(blk_start, blk_end); if (BlockOffsetArrayUseUnallocatedBlock) { @@ -441,14 +474,12 @@ } } - void allocated(HeapWord* blk, size_t size) { - allocated(blk, blk + size); + void allocated(HeapWord* blk, size_t size, bool reducing = false) { + allocated(blk, blk + size, reducing); } void freed(HeapWord* blk_start, HeapWord* blk_end); - void freed(HeapWord* blk, size_t size) { - freed(blk, blk + size); - } + void freed(HeapWord* blk, size_t size); HeapWord* block_start_unsafe(const void* addr) const; @@ -456,7 +487,6 @@ // start of the block that contains the given address. HeapWord* block_start_careful(const void* addr) const; - // Verification & debugging: ensure that the offset table reflects // the fact that the block [blk_start, blk_end) or [blk, blk + size) // is a single block of storage. NOTE: can't const this because of
--- a/src/share/vm/memory/blockOffsetTable.inline.hpp Tue Aug 17 22:52:50 2010 -0700 +++ b/src/share/vm/memory/blockOffsetTable.inline.hpp Thu Aug 19 14:08:58 2010 -0400 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2002, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -55,10 +55,22 @@ return result; } +inline void BlockOffsetSharedArray::check_reducing_assertion(bool reducing) { + assert(reducing || !SafepointSynchronize::is_at_safepoint() || init_to_zero() || + Thread::current()->is_VM_thread() || + Thread::current()->is_ConcurrentGC_thread() || + ((!Thread::current()->is_ConcurrentGC_thread()) && + ParGCRareEvent_lock->owned_by_self()), "Crack"); +} ////////////////////////////////////////////////////////////////////////// // BlockOffsetArrayNonContigSpace inlines ////////////////////////////////////////////////////////////////////////// +inline void BlockOffsetArrayNonContigSpace::freed(HeapWord* blk, + size_t size) { + freed(blk, blk + size); +} + inline void BlockOffsetArrayNonContigSpace::freed(HeapWord* blk_start, HeapWord* blk_end) { // Verify that the BOT shows [blk_start, blk_end) to be one block.
--- a/src/share/vm/runtime/globals.hpp Tue Aug 17 22:52:50 2010 -0700 +++ b/src/share/vm/runtime/globals.hpp Thu Aug 19 14:08:58 2010 -0400 @@ -1712,7 +1712,7 @@ develop(bool, VerifyBlockOffsetArray, false, \ "Do (expensive!) block offset array verification") \ \ - product(bool, BlockOffsetArrayUseUnallocatedBlock, trueInDebug, \ + product(bool, BlockOffsetArrayUseUnallocatedBlock, false, \ "Maintain _unallocated_block in BlockOffsetArray" \ " (currently applicable only to CMS collector)") \ \