Mercurial > hg > release > icedtea7-forest-2.5 > hotspot
changeset 5019:b7d44793cd26 hs24.60-b02
Merge
| author | amurillo |
|---|---|
| date | Wed, 23 Oct 2013 10:15:50 -0700 |
| parents | eeef89c7279e (current diff) 4911caf5a68d (diff) |
| children | a03f20e7b0fc |
| files | make/hotspot_version src/os/bsd/vm/os_bsd.cpp |
| diffstat | 24 files changed, 1278 insertions(+), 181 deletions(-) [+] |
line wrap: on
line diff
--- a/make/hotspot_version Mon Oct 14 23:15:43 2013 -0700 +++ b/make/hotspot_version Wed Oct 23 10:15:50 2013 -0700 @@ -35,7 +35,7 @@ HS_MAJOR_VER=24 HS_MINOR_VER=60 -HS_BUILD_NUMBER=01 +HS_BUILD_NUMBER=02 JDK_MAJOR_VER=1 JDK_MINOR_VER=7
--- a/src/os/bsd/vm/os_bsd.cpp Mon Oct 14 23:15:43 2013 -0700 +++ b/src/os/bsd/vm/os_bsd.cpp Wed Oct 23 10:15:50 2013 -0700 @@ -3434,7 +3434,9 @@ #endif #endif -char* os::reserve_memory_special(size_t bytes, char* req_addr, bool exec) { +char* os::reserve_memory_special(size_t bytes, size_t alignment, char* req_addr, bool exec) { + fatal("This code is not used or maintained."); + // "exec" is passed in but not used. Creating the shared image for // the code cache doesn't have an SHM_X executable permission to check. assert(UseLargePages && UseSHM, "only for SHM large pages"); @@ -6105,3 +6107,9 @@ return n; } +#ifndef PRODUCT +void TestReserveMemorySpecial_test() { + // No tests available for this platform +} +#endif +
--- a/src/os/linux/vm/globals_linux.hpp Mon Oct 14 23:15:43 2013 -0700 +++ b/src/os/linux/vm/globals_linux.hpp Wed Oct 23 10:15:50 2013 -0700 @@ -40,6 +40,9 @@ product(bool, UseHugeTLBFS, false, \ "Use MAP_HUGETLB for large pages") \ \ + product(bool, UseTransparentHugePages, false, \ + "Use MADV_HUGEPAGE for large pages") \ + \ product(bool, LoadExecStackDllInVMThread, true, \ "Load DLLs with executable-stack attribute in the VM Thread") \ \ @@ -50,7 +53,7 @@ // Defines Linux-specific default values. The flags are available on all // platforms, but they may have different default values on other platforms. // -define_pd_global(bool, UseLargePages, true); +define_pd_global(bool, UseLargePages, false); define_pd_global(bool, UseLargePagesIndividualAllocation, false); define_pd_global(bool, UseOSErrorReporting, false); define_pd_global(bool, UseThreadPriorities, true) ;
--- a/src/os/linux/vm/os_linux.cpp Mon Oct 14 23:15:43 2013 -0700 +++ b/src/os/linux/vm/os_linux.cpp Wed Oct 23 10:15:50 2013 -0700 @@ -2752,35 +2752,7 @@ int os::Linux::commit_memory_impl(char* addr, size_t size, size_t alignment_hint, bool exec) { - int err; - if (UseHugeTLBFS && alignment_hint > (size_t)vm_page_size()) { - int prot = exec ? PROT_READ|PROT_WRITE|PROT_EXEC : PROT_READ|PROT_WRITE; - uintptr_t res = - (uintptr_t) ::mmap(addr, size, prot, - MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_HUGETLB, - -1, 0); - if (res != (uintptr_t) MAP_FAILED) { - if (UseNUMAInterleaving) { - numa_make_global(addr, size); - } - return 0; - } - - err = errno; // save errno from mmap() call above - - if (!recoverable_mmap_error(err)) { - // However, it is not clear that this loss of our reserved mapping - // happens with large pages on Linux or that we cannot recover - // from the loss. For now, we just issue a warning and we don't - // call vm_exit_out_of_memory(). This issue is being tracked by - // JBS-8007074. - warn_fail_commit_memory(addr, size, alignment_hint, exec, err); -// vm_exit_out_of_memory(size, "committing reserved memory."); - } - // Fall through and try to use small pages - } - - err = os::Linux::commit_memory_impl(addr, size, exec); + int err = os::Linux::commit_memory_impl(addr, size, exec); if (err == 0) { realign_memory(addr, size, alignment_hint); } @@ -2805,7 +2777,7 @@ } void os::pd_realign_memory(char *addr, size_t bytes, size_t alignment_hint) { - if (UseHugeTLBFS && alignment_hint > (size_t)vm_page_size()) { + if (UseTransparentHugePages && alignment_hint > (size_t)vm_page_size()) { // We don't check the return value: madvise(MADV_HUGEPAGE) may not // be supported or the memory may already be backed by huge pages. ::madvise(addr, bytes, MADV_HUGEPAGE); @@ -2818,7 +2790,7 @@ // uncommitted at all. We don't do anything in this case to avoid creating a segment with // small pages on top of the SHM segment. This method always works for small pages, so we // allow that in any case. - if (alignment_hint <= (size_t)os::vm_page_size() || !UseSHM) { + if (alignment_hint <= (size_t)os::vm_page_size() || can_commit_large_page_memory()) { commit_memory(addr, bytes, alignment_hint, !ExecMem); } } @@ -3187,11 +3159,31 @@ return linux_mprotect(addr, size, PROT_READ|PROT_WRITE); } +bool os::Linux::transparent_huge_pages_sanity_check(bool warn, size_t page_size) { + bool result = false; + void *p = mmap(NULL, page_size * 2, PROT_READ|PROT_WRITE, + MAP_ANONYMOUS|MAP_PRIVATE, + -1, 0); + if (p != MAP_FAILED) { + void *aligned_p = align_ptr_up(p, page_size); + + result = madvise(aligned_p, page_size, MADV_HUGEPAGE) == 0; + + munmap(p, page_size * 2); + } + + if (warn && !result) { + warning("TransparentHugePages is not supported by the operating system."); + } + + return result; +} + bool os::Linux::hugetlbfs_sanity_check(bool warn, size_t page_size) { bool result = false; - void *p = mmap (NULL, page_size, PROT_READ|PROT_WRITE, - MAP_ANONYMOUS|MAP_PRIVATE|MAP_HUGETLB, - -1, 0); + void *p = mmap(NULL, page_size, PROT_READ|PROT_WRITE, + MAP_ANONYMOUS|MAP_PRIVATE|MAP_HUGETLB, + -1, 0); if (p != MAP_FAILED) { // We don't know if this really is a huge page or not. @@ -3212,12 +3204,10 @@ } fclose(fp); } - munmap (p, page_size); - if (result) - return true; - } - - if (warn) { + munmap(p, page_size); + } + + if (warn && !result) { warning("HugeTLBFS is not supported by the operating system."); } @@ -3265,82 +3255,126 @@ static size_t _large_page_size = 0; -void os::large_page_init() { - if (!UseLargePages) { - UseHugeTLBFS = false; - UseSHM = false; - return; - } - - if (FLAG_IS_DEFAULT(UseHugeTLBFS) && FLAG_IS_DEFAULT(UseSHM)) { - // If UseLargePages is specified on the command line try both methods, - // if it's default, then try only HugeTLBFS. - if (FLAG_IS_DEFAULT(UseLargePages)) { - UseHugeTLBFS = true; - } else { - UseHugeTLBFS = UseSHM = true; - } - } - - if (LargePageSizeInBytes) { - _large_page_size = LargePageSizeInBytes; - } else { - // large_page_size on Linux is used to round up heap size. x86 uses either - // 2M or 4M page, depending on whether PAE (Physical Address Extensions) - // mode is enabled. AMD64/EM64T uses 2M page in 64bit mode. IA64 can use - // page as large as 256M. - // - // Here we try to figure out page size by parsing /proc/meminfo and looking - // for a line with the following format: - // Hugepagesize: 2048 kB - // - // If we can't determine the value (e.g. /proc is not mounted, or the text - // format has been changed), we'll use the largest page size supported by - // the processor. +size_t os::Linux::find_large_page_size() { + size_t large_page_size = 0; + + // large_page_size on Linux is used to round up heap size. x86 uses either + // 2M or 4M page, depending on whether PAE (Physical Address Extensions) + // mode is enabled. AMD64/EM64T uses 2M page in 64bit mode. IA64 can use + // page as large as 256M. + // + // Here we try to figure out page size by parsing /proc/meminfo and looking + // for a line with the following format: + // Hugepagesize: 2048 kB + // + // If we can't determine the value (e.g. /proc is not mounted, or the text + // format has been changed), we'll use the largest page size supported by + // the processor. #ifndef ZERO - _large_page_size = IA32_ONLY(4 * M) AMD64_ONLY(2 * M) IA64_ONLY(256 * M) SPARC_ONLY(4 * M) - ARM_ONLY(2 * M) PPC_ONLY(4 * M); + large_page_size = IA32_ONLY(4 * M) AMD64_ONLY(2 * M) IA64_ONLY(256 * M) SPARC_ONLY(4 * M) + ARM_ONLY(2 * M) PPC_ONLY(4 * M); #endif // ZERO - FILE *fp = fopen("/proc/meminfo", "r"); - if (fp) { - while (!feof(fp)) { - int x = 0; - char buf[16]; - if (fscanf(fp, "Hugepagesize: %d", &x) == 1) { - if (x && fgets(buf, sizeof(buf), fp) && strcmp(buf, " kB\n") == 0) { - _large_page_size = x * K; - break; - } - } else { - // skip to next line - for (;;) { - int ch = fgetc(fp); - if (ch == EOF || ch == (int)'\n') break; - } + FILE *fp = fopen("/proc/meminfo", "r"); + if (fp) { + while (!feof(fp)) { + int x = 0; + char buf[16]; + if (fscanf(fp, "Hugepagesize: %d", &x) == 1) { + if (x && fgets(buf, sizeof(buf), fp) && strcmp(buf, " kB\n") == 0) { + large_page_size = x * K; + break; + } + } else { + // skip to next line + for (;;) { + int ch = fgetc(fp); + if (ch == EOF || ch == (int)'\n') break; } } - fclose(fp); } - } - - // print a warning if any large page related flag is specified on command line - bool warn_on_failure = !FLAG_IS_DEFAULT(UseHugeTLBFS); - + fclose(fp); + } + + if (!FLAG_IS_DEFAULT(LargePageSizeInBytes) && LargePageSizeInBytes != large_page_size) { + warning("Setting LargePageSizeInBytes has no effect on this OS. Large page size is " + SIZE_FORMAT "%s.", byte_size_in_proper_unit(large_page_size), + proper_unit_for_byte_size(large_page_size)); + } + + return large_page_size; +} + +size_t os::Linux::setup_large_page_size() { + _large_page_size = Linux::find_large_page_size(); const size_t default_page_size = (size_t)Linux::page_size(); if (_large_page_size > default_page_size) { _page_sizes[0] = _large_page_size; _page_sizes[1] = default_page_size; _page_sizes[2] = 0; } - UseHugeTLBFS = UseHugeTLBFS && - Linux::hugetlbfs_sanity_check(warn_on_failure, _large_page_size); - - if (UseHugeTLBFS) + + return _large_page_size; +} + +bool os::Linux::setup_large_page_type(size_t page_size) { + if (FLAG_IS_DEFAULT(UseHugeTLBFS) && + FLAG_IS_DEFAULT(UseSHM) && + FLAG_IS_DEFAULT(UseTransparentHugePages)) { + + // The type of large pages has not been specified by the user. + + // Try UseHugeTLBFS and then UseSHM. + UseHugeTLBFS = UseSHM = true; + + // Don't try UseTransparentHugePages since there are known + // performance issues with it turned on. This might change in the future. + UseTransparentHugePages = false; + } + + if (UseTransparentHugePages) { + bool warn_on_failure = !FLAG_IS_DEFAULT(UseTransparentHugePages); + if (transparent_huge_pages_sanity_check(warn_on_failure, page_size)) { + UseHugeTLBFS = false; + UseSHM = false; + return true; + } + UseTransparentHugePages = false; + } + + if (UseHugeTLBFS) { + bool warn_on_failure = !FLAG_IS_DEFAULT(UseHugeTLBFS); + if (hugetlbfs_sanity_check(warn_on_failure, page_size)) { + UseSHM = false; + return true; + } + UseHugeTLBFS = false; + } + + return UseSHM; +} + +void os::large_page_init() { + if (!UseLargePages && + !UseTransparentHugePages && + !UseHugeTLBFS && + !UseSHM) { + // Not using large pages. + return; + } + + if (!FLAG_IS_DEFAULT(UseLargePages) && !UseLargePages) { + // The user explicitly turned off large pages. + // Ignore the rest of the large pages flags. + UseTransparentHugePages = false; + UseHugeTLBFS = false; UseSHM = false; - - UseLargePages = UseHugeTLBFS || UseSHM; + return; + } + + size_t large_page_size = Linux::setup_large_page_size(); + UseLargePages = Linux::setup_large_page_type(large_page_size); set_coredump_filter(); } @@ -3349,16 +3383,22 @@ #define SHM_HUGETLB 04000 #endif -char* os::reserve_memory_special(size_t bytes, char* req_addr, bool exec) { +char* os::Linux::reserve_memory_special_shm(size_t bytes, size_t alignment, char* req_addr, bool exec) { // "exec" is passed in but not used. Creating the shared image for // the code cache doesn't have an SHM_X executable permission to check. assert(UseLargePages && UseSHM, "only for SHM large pages"); + assert(is_ptr_aligned(req_addr, os::large_page_size()), "Unaligned address"); + + if (!is_size_aligned(bytes, os::large_page_size()) || alignment > os::large_page_size()) { + return NULL; // Fallback to small pages. + } key_t key = IPC_PRIVATE; char *addr; bool warn_on_failure = UseLargePages && (!FLAG_IS_DEFAULT(UseLargePages) || + !FLAG_IS_DEFAULT(UseSHM) || !FLAG_IS_DEFAULT(LargePageSizeInBytes) ); char msg[128]; @@ -3406,42 +3446,220 @@ return NULL; } - if ((addr != NULL) && UseNUMAInterleaving) { - numa_make_global(addr, bytes); - } - - // The memory is committed - MemTracker::record_virtual_memory_reserve_and_commit((address)addr, bytes, mtNone, CALLER_PC); + return addr; +} + +static void warn_on_large_pages_failure(char* req_addr, size_t bytes, int error) { + assert(error == ENOMEM, "Only expect to fail if no memory is available"); + + bool warn_on_failure = UseLargePages && + (!FLAG_IS_DEFAULT(UseLargePages) || + !FLAG_IS_DEFAULT(UseHugeTLBFS) || + !FLAG_IS_DEFAULT(LargePageSizeInBytes)); + + if (warn_on_failure) { + char msg[128]; + jio_snprintf(msg, sizeof(msg), "Failed to reserve large pages memory req_addr: " + PTR_FORMAT " bytes: " SIZE_FORMAT " (errno = %d).", req_addr, bytes, error); + warning(msg); + } +} + +char* os::Linux::reserve_memory_special_huge_tlbfs_only(size_t bytes, char* req_addr, bool exec) { + assert(UseLargePages && UseHugeTLBFS, "only for Huge TLBFS large pages"); + assert(is_size_aligned(bytes, os::large_page_size()), "Unaligned size"); + assert(is_ptr_aligned(req_addr, os::large_page_size()), "Unaligned address"); + + int prot = exec ? PROT_READ|PROT_WRITE|PROT_EXEC : PROT_READ|PROT_WRITE; + char* addr = (char*)::mmap(req_addr, bytes, prot, + MAP_PRIVATE|MAP_ANONYMOUS|MAP_HUGETLB, + -1, 0); + + if (addr == MAP_FAILED) { + warn_on_large_pages_failure(req_addr, bytes, errno); + return NULL; + } + + assert(is_ptr_aligned(addr, os::large_page_size()), "Must be"); return addr; } +char* os::Linux::reserve_memory_special_huge_tlbfs_mixed(size_t bytes, size_t alignment, char* req_addr, bool exec) { + size_t large_page_size = os::large_page_size(); + + assert(bytes >= large_page_size, "Shouldn't allocate large pages for small sizes"); + + // Allocate small pages. + + char* start; + if (req_addr != NULL) { + assert(is_ptr_aligned(req_addr, alignment), "Must be"); + assert(is_size_aligned(bytes, alignment), "Must be"); + start = os::reserve_memory(bytes, req_addr); + assert(start == NULL || start == req_addr, "Must be"); + } else { + start = os::reserve_memory_aligned(bytes, alignment); + } + + if (start == NULL) { + return NULL; + } + + assert(is_ptr_aligned(start, alignment), "Must be"); + + // os::reserve_memory_special will record this memory area. + // Need to release it here to prevent overlapping reservations. + MemTracker::record_virtual_memory_release((address)start, bytes); + + char* end = start + bytes; + + // Find the regions of the allocated chunk that can be promoted to large pages. + char* lp_start = (char*)align_ptr_up(start, large_page_size); + char* lp_end = (char*)align_ptr_down(end, large_page_size); + + size_t lp_bytes = lp_end - lp_start; + + assert(is_size_aligned(lp_bytes, large_page_size), "Must be"); + + if (lp_bytes == 0) { + // The mapped region doesn't even span the start and the end of a large page. + // Fall back to allocate a non-special area. + ::munmap(start, end - start); + return NULL; + } + + int prot = exec ? PROT_READ|PROT_WRITE|PROT_EXEC : PROT_READ|PROT_WRITE; + + + void* result; + + if (start != lp_start) { + result = ::mmap(start, lp_start - start, prot, + MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, + -1, 0); + if (result == MAP_FAILED) { + ::munmap(lp_start, end - lp_start); + return NULL; + } + } + + result = ::mmap(lp_start, lp_bytes, prot, + MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED|MAP_HUGETLB, + -1, 0); + if (result == MAP_FAILED) { + warn_on_large_pages_failure(req_addr, bytes, errno); + // If the mmap above fails, the large pages region will be unmapped and we + // have regions before and after with small pages. Release these regions. + // + // | mapped | unmapped | mapped | + // ^ ^ ^ ^ + // start lp_start lp_end end + // + ::munmap(start, lp_start - start); + ::munmap(lp_end, end - lp_end); + return NULL; + } + + if (lp_end != end) { + result = ::mmap(lp_end, end - lp_end, prot, + MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, + -1, 0); + if (result == MAP_FAILED) { + ::munmap(start, lp_end - start); + return NULL; + } + } + + return start; +} + +char* os::Linux::reserve_memory_special_huge_tlbfs(size_t bytes, size_t alignment, char* req_addr, bool exec) { + assert(UseLargePages && UseHugeTLBFS, "only for Huge TLBFS large pages"); + assert(is_ptr_aligned(req_addr, alignment), "Must be"); + assert(is_power_of_2(alignment), "Must be"); + assert(is_power_of_2(os::large_page_size()), "Must be"); + assert(bytes >= os::large_page_size(), "Shouldn't allocate large pages for small sizes"); + + if (is_size_aligned(bytes, os::large_page_size()) && alignment <= os::large_page_size()) { + return reserve_memory_special_huge_tlbfs_only(bytes, req_addr, exec); + } else { + return reserve_memory_special_huge_tlbfs_mixed(bytes, alignment, req_addr, exec); + } +} + +char* os::reserve_memory_special(size_t bytes, size_t alignment, char* req_addr, bool exec) { + assert(UseLargePages, "only for large pages"); + + char* addr; + if (UseSHM) { + addr = os::Linux::reserve_memory_special_shm(bytes, alignment, req_addr, exec); + } else { + assert(UseHugeTLBFS, "must be"); + addr = os::Linux::reserve_memory_special_huge_tlbfs(bytes, alignment, req_addr, exec); + } + + if (addr != NULL) { + if (UseNUMAInterleaving) { + numa_make_global(addr, bytes); + } + + // The memory is committed + MemTracker::record_virtual_memory_reserve_and_commit((address)addr, bytes, mtNone, CALLER_PC); + } + + return addr; +} + +bool os::Linux::release_memory_special_shm(char* base, size_t bytes) { + // detaching the SHM segment will also delete it, see reserve_memory_special_shm() + return shmdt(base) == 0; +} + +bool os::Linux::release_memory_special_huge_tlbfs(char* base, size_t bytes) { + return pd_release_memory(base, bytes); +} + bool os::release_memory_special(char* base, size_t bytes) { + assert(UseLargePages, "only for large pages"); + MemTracker::Tracker tkr = MemTracker::get_virtual_memory_release_tracker(); - // detaching the SHM segment will also delete it, see reserve_memory_special() - int rslt = shmdt(base); - if (rslt == 0) { + + bool res; + if (UseSHM) { + res = os::Linux::release_memory_special_shm(base, bytes); + } else { + assert(UseHugeTLBFS, "must be"); + res = os::Linux::release_memory_special_huge_tlbfs(base, bytes); + } + + if (res) { tkr.record((address)base, bytes); - return true; } else { tkr.discard(); - return false; - } -} + } + + return res; +} + size_t os::large_page_size() { return _large_page_size; } -// HugeTLBFS allows application to commit large page memory on demand; -// with SysV SHM the entire memory region must be allocated as shared +// With SysV SHM the entire memory region must be allocated as shared // memory. +// HugeTLBFS allows application to commit large page memory on demand. +// However, when committing memory with HugeTLBFS fails, the region +// that was supposed to be committed will lose the old reservation +// and allow other threads to steal that memory region. Because of this +// behavior we can't commit HugeTLBFS memory. bool os::can_commit_large_page_memory() { - return UseHugeTLBFS; + return UseTransparentHugePages; } bool os::can_execute_large_page_memory() { - return UseHugeTLBFS; + return UseTransparentHugePages || UseHugeTLBFS; } // Reserve memory at an arbitrary address, only if that area is @@ -4615,21 +4833,23 @@ UseNUMA = false; } } - // With SHM large pages we cannot uncommit a page, so there's not way + // With SHM and HugeTLBFS large pages we cannot uncommit a page, so there's no way // we can make the adaptive lgrp chunk resizing work. If the user specified - // both UseNUMA and UseLargePages (or UseSHM) on the command line - warn and + // both UseNUMA and UseLargePages (or UseSHM/UseHugeTLBFS) on the command line - warn and // disable adaptive resizing. - if (UseNUMA && UseLargePages && UseSHM) { - if (!FLAG_IS_DEFAULT(UseNUMA)) { - if (FLAG_IS_DEFAULT(UseLargePages) && FLAG_IS_DEFAULT(UseSHM)) { + if (UseNUMA && UseLargePages && !can_commit_large_page_memory()) { + if (FLAG_IS_DEFAULT(UseNUMA)) { + UseNUMA = false; + } else { + if (FLAG_IS_DEFAULT(UseLargePages) && + FLAG_IS_DEFAULT(UseSHM) && + FLAG_IS_DEFAULT(UseHugeTLBFS)) { UseLargePages = false; } else { - warning("UseNUMA is not fully compatible with SHM large pages, disabling adaptive resizing"); + warning("UseNUMA is not fully compatible with SHM/HugeTLBFS large pages, disabling adaptive resizing"); UseAdaptiveSizePolicy = false; UseAdaptiveNUMAChunkSizing = false; } - } else { - UseNUMA = false; } } if (!UseNUMA && ForceNUMA) { @@ -5982,3 +6202,149 @@ } #endif // JAVASE_EMBEDDED + + +/////////////// Unit tests /////////////// + +#ifndef PRODUCT + +#define test_log(...) \ + do {\ + if (VerboseInternalVMTests) { \ + tty->print_cr(__VA_ARGS__); \ + tty->flush(); \ + }\ + } while (false) + +class TestReserveMemorySpecial : AllStatic { + public: + static void small_page_write(void* addr, size_t size) { + size_t page_size = os::vm_page_size(); + + char* end = (char*)addr + size; + for (char* p = (char*)addr; p < end; p += page_size) { + *p = 1; + } + } + + static void test_reserve_memory_special_huge_tlbfs_only(size_t size) { + if (!UseHugeTLBFS) { + return; + } + + test_log("test_reserve_memory_special_huge_tlbfs_only(" SIZE_FORMAT ")", size); + + char* addr = os::Linux::reserve_memory_special_huge_tlbfs_only(size, NULL, false); + + if (addr != NULL) { + small_page_write(addr, size); + + os::Linux::release_memory_special_huge_tlbfs(addr, size); + } + } + + static void test_reserve_memory_special_huge_tlbfs_only() { + if (!UseHugeTLBFS) { + return; + } + + size_t lp = os::large_page_size(); + + for (size_t size = lp; size <= lp * 10; size += lp) { + test_reserve_memory_special_huge_tlbfs_only(size); + } + } + + static void test_reserve_memory_special_huge_tlbfs_mixed(size_t size, size_t alignment) { + if (!UseHugeTLBFS) { + return; + } + + test_log("test_reserve_memory_special_huge_tlbfs_mixed(" SIZE_FORMAT ", " SIZE_FORMAT ")", + size, alignment); + + assert(size >= os::large_page_size(), "Incorrect input to test"); + + char* addr = os::Linux::reserve_memory_special_huge_tlbfs_mixed(size, alignment, NULL, false); + + if (addr != NULL) { + small_page_write(addr, size); + + os::Linux::release_memory_special_huge_tlbfs(addr, size); + } + } + + static void test_reserve_memory_special_huge_tlbfs_mixed_all_alignments(size_t size) { + size_t lp = os::large_page_size(); + size_t ag = os::vm_allocation_granularity(); + + for (size_t alignment = ag; is_size_aligned(size, alignment); alignment *= 2) { + test_reserve_memory_special_huge_tlbfs_mixed(size, alignment); + } + } + + static void test_reserve_memory_special_huge_tlbfs_mixed() { + size_t lp = os::large_page_size(); + size_t ag = os::vm_allocation_granularity(); + + test_reserve_memory_special_huge_tlbfs_mixed_all_alignments(lp); + test_reserve_memory_special_huge_tlbfs_mixed_all_alignments(lp + ag); + test_reserve_memory_special_huge_tlbfs_mixed_all_alignments(lp + lp / 2); + test_reserve_memory_special_huge_tlbfs_mixed_all_alignments(lp * 2); + test_reserve_memory_special_huge_tlbfs_mixed_all_alignments(lp * 2 + ag); + test_reserve_memory_special_huge_tlbfs_mixed_all_alignments(lp * 2 - ag); + test_reserve_memory_special_huge_tlbfs_mixed_all_alignments(lp * 2 + lp / 2); + test_reserve_memory_special_huge_tlbfs_mixed_all_alignments(lp * 10); + test_reserve_memory_special_huge_tlbfs_mixed_all_alignments(lp * 10 + lp / 2); + } + + static void test_reserve_memory_special_huge_tlbfs() { + if (!UseHugeTLBFS) { + return; + } + + test_reserve_memory_special_huge_tlbfs_only(); + test_reserve_memory_special_huge_tlbfs_mixed(); + } + + static void test_reserve_memory_special_shm(size_t size, size_t alignment) { + if (!UseSHM) { + return; + } + + test_log("test_reserve_memory_special_shm(" SIZE_FORMAT ", " SIZE_FORMAT ")", size, alignment); + + char* addr = os::Linux::reserve_memory_special_shm(size, alignment, NULL, false); + + if (addr != NULL) { + assert(is_ptr_aligned(addr, alignment), "Check"); + assert(is_ptr_aligned(addr, os::large_page_size()), "Check"); + + small_page_write(addr, size); + + os::Linux::release_memory_special_shm(addr, size); + } + } + + static void test_reserve_memory_special_shm() { + size_t lp = os::large_page_size(); + size_t ag = os::vm_allocation_granularity(); + + for (size_t size = ag; size < lp * 3; size += ag) { + for (size_t alignment = ag; is_size_aligned(size, alignment); alignment *= 2) { + test_reserve_memory_special_shm(size, alignment); + } + } + } + + static void test() { + test_reserve_memory_special_huge_tlbfs(); + test_reserve_memory_special_shm(); + } +}; + +void TestReserveMemorySpecial_test() { + TestReserveMemorySpecial::test(); +} + +#endif
--- a/src/os/linux/vm/os_linux.hpp Mon Oct 14 23:15:43 2013 -0700 +++ b/src/os/linux/vm/os_linux.hpp Wed Oct 23 10:15:50 2013 -0700 @@ -32,6 +32,7 @@ class Linux { friend class os; + friend class TestReserveMemorySpecial; // For signal-chaining #define MAXSIGNUM 32 @@ -92,8 +93,21 @@ static void rebuild_cpu_to_node_map(); static GrowableArray<int>* cpu_to_node() { return _cpu_to_node; } + static size_t find_large_page_size(); + static size_t setup_large_page_size(); + + static bool setup_large_page_type(size_t page_size); + static bool transparent_huge_pages_sanity_check(bool warn, size_t pages_size); static bool hugetlbfs_sanity_check(bool warn, size_t page_size); + static char* reserve_memory_special_shm(size_t bytes, size_t alignment, char* req_addr, bool exec); + static char* reserve_memory_special_huge_tlbfs(size_t bytes, size_t alignment, char* req_addr, bool exec); + static char* reserve_memory_special_huge_tlbfs_only(size_t bytes, char* req_addr, bool exec); + static char* reserve_memory_special_huge_tlbfs_mixed(size_t bytes, size_t alignment, char* req_addr, bool exec); + + static bool release_memory_special_shm(char* base, size_t bytes); + static bool release_memory_special_huge_tlbfs(char* base, size_t bytes); + static void print_full_memory_info(outputStream* st); static void print_distro_info(outputStream* st); static void print_libversion_info(outputStream* st);
--- a/src/os/solaris/vm/os_solaris.cpp Mon Oct 14 23:15:43 2013 -0700 +++ b/src/os/solaris/vm/os_solaris.cpp Wed Oct 23 10:15:50 2013 -0700 @@ -3530,11 +3530,15 @@ return true; } -char* os::reserve_memory_special(size_t size, char* addr, bool exec) { +char* os::reserve_memory_special(size_t size, size_t alignment, char* addr, bool exec) { // "exec" is passed in but not used. Creating the shared image for // the code cache doesn't have an SHM_X executable permission to check. assert(UseLargePages && UseISM, "only for ISM large pages"); + if (!is_size_aligned(size, os::large_page_size()) || alignment > os::large_page_size()) { + return NULL; // Fallback to small pages. + } + char* retAddr = NULL; int shmid; key_t ismKey; @@ -6862,3 +6866,9 @@ return strlen(buffer); } + +#ifndef PRODUCT +void TestReserveMemorySpecial_test() { + // No tests available for this platform +} +#endif
--- a/src/os/windows/vm/os_windows.cpp Mon Oct 14 23:15:43 2013 -0700 +++ b/src/os/windows/vm/os_windows.cpp Wed Oct 23 10:15:50 2013 -0700 @@ -3079,7 +3079,12 @@ return true; } -char* os::reserve_memory_special(size_t bytes, char* addr, bool exec) { +char* os::reserve_memory_special(size_t bytes, size_t alignment, char* addr, bool exec) { + assert(UseLargePages, "only for large pages"); + + if (!is_size_aligned(bytes, os::large_page_size()) || alignment > os::large_page_size()) { + return NULL; // Fallback to small pages. + } const DWORD prot = exec ? PAGE_EXECUTE_READWRITE : PAGE_READWRITE; const DWORD flags = MEM_RESERVE | MEM_COMMIT | MEM_LARGE_PAGES; @@ -5584,3 +5589,9 @@ } #endif + +#ifndef PRODUCT +void TestReserveMemorySpecial_test() { + // No tests available for this platform +} +#endif
--- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Mon Oct 14 23:15:43 2013 -0700 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Wed Oct 23 10:15:50 2013 -0700 @@ -2002,10 +2002,12 @@ size_t init_byte_size = collector_policy()->initial_heap_byte_size(); size_t max_byte_size = collector_policy()->max_heap_byte_size(); + size_t heap_alignment = collector_policy()->max_alignment(); // Ensure that the sizes are properly aligned. Universe::check_alignment(init_byte_size, HeapRegion::GrainBytes, "g1 heap"); Universe::check_alignment(max_byte_size, HeapRegion::GrainBytes, "g1 heap"); + Universe::check_alignment(max_byte_size, heap_alignment, "g1 heap"); _cg1r = new ConcurrentG1Refine(this); @@ -2029,14 +2031,14 @@ size_t total_reserved = 0; total_reserved = add_and_check_overflow(total_reserved, max_byte_size); - size_t pg_max_size = (size_t) align_size_up(pgs->max_size(), HeapRegion::GrainBytes); + size_t pg_max_size = (size_t) align_size_up(pgs->max_size(), heap_alignment); total_reserved = add_and_check_overflow(total_reserved, pg_max_size); Universe::check_alignment(total_reserved, HeapRegion::GrainBytes, "g1 heap and perm"); - char* addr = Universe::preferred_heap_base(total_reserved, Universe::UnscaledNarrowOop); - - ReservedHeapSpace heap_rs(total_reserved, HeapRegion::GrainBytes, + char* addr = Universe::preferred_heap_base(total_reserved, heap_alignment, Universe::UnscaledNarrowOop); + + ReservedHeapSpace heap_rs(total_reserved, heap_alignment, UseLargePages, addr); if (UseCompressedOops) { @@ -2044,17 +2046,17 @@ // Failed to reserve at specified address - the requested memory // region is taken already, for example, by 'java' launcher. // Try again to reserver heap higher. - addr = Universe::preferred_heap_base(total_reserved, Universe::ZeroBasedNarrowOop); - - ReservedHeapSpace heap_rs0(total_reserved, HeapRegion::GrainBytes, + addr = Universe::preferred_heap_base(total_reserved, heap_alignment, Universe::ZeroBasedNarrowOop); + + ReservedHeapSpace heap_rs0(total_reserved, heap_alignment, UseLargePages, addr); if (addr != NULL && !heap_rs0.is_reserved()) { // Failed to reserve at specified address again - give up. - addr = Universe::preferred_heap_base(total_reserved, Universe::HeapBasedNarrowOop); + addr = Universe::preferred_heap_base(total_reserved, heap_alignment, Universe::HeapBasedNarrowOop); assert(addr == NULL, ""); - ReservedHeapSpace heap_rs1(total_reserved, HeapRegion::GrainBytes, + ReservedHeapSpace heap_rs1(total_reserved, heap_alignment, UseLargePages, addr); heap_rs = heap_rs1; } else {
--- a/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp Mon Oct 14 23:15:43 2013 -0700 +++ b/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp Wed Oct 23 10:15:50 2013 -0700 @@ -310,7 +310,8 @@ void G1CollectorPolicy::initialize_flags() { set_min_alignment(HeapRegion::GrainBytes); size_t card_table_alignment = GenRemSet::max_alignment_constraint(rem_set_name()); - set_max_alignment(MAX2(card_table_alignment, min_alignment())); + size_t page_size = UseLargePages ? os::large_page_size() : os::vm_page_size(); + set_max_alignment(MAX3(card_table_alignment, min_alignment(), page_size)); if (SurvivorRatio < 1) { vm_exit_during_initialization("Invalid survivor ratio specified"); }
--- a/src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.cpp Mon Oct 14 23:15:43 2013 -0700 +++ b/src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.cpp Wed Oct 23 10:15:50 2013 -0700 @@ -90,14 +90,16 @@ og_min_size, og_max_size, yg_min_size, yg_max_size); - // The ReservedSpace ctor used below requires that the page size for the perm - // gen is <= the page size for the rest of the heap (young + old gens). const size_t og_page_sz = os::page_size_for_region(yg_min_size + og_min_size, yg_max_size + og_max_size, 8); - const size_t pg_page_sz = MIN2(os::page_size_for_region(pg_min_size, - pg_max_size, 16), - og_page_sz); + + // Use the same page size for both perm gen and old gen, + // to allow large pages to be allocated when the heap is reserved + // for the implementations that can't 'commit' large pages. + // NEEDS_CLEANUP. ReservedHeapSpace/ReservedSpace that takes both + // a prefix and a suffix alignment can now be removed. + const size_t pg_page_sz = og_page_sz; const size_t pg_align = set_alignment(_perm_gen_alignment, pg_page_sz); const size_t og_align = set_alignment(_old_gen_alignment, og_page_sz); @@ -138,12 +140,9 @@ total_reserved = add_and_check_overflow(total_reserved, og_max_size); total_reserved = add_and_check_overflow(total_reserved, yg_max_size); - char* addr = Universe::preferred_heap_base(total_reserved, Universe::UnscaledNarrowOop); + assert(is_size_aligned(total_reserved, og_align), "Must be"); - // The main part of the heap (old gen + young gen) can often use a larger page - // size than is needed or wanted for the perm gen. Use the "compound - // alignment" ReservedSpace ctor to avoid having to use the same page size for - // all gens. + char* addr = Universe::preferred_heap_base(total_reserved, og_align, Universe::UnscaledNarrowOop); ReservedHeapSpace heap_rs(pg_max_size, pg_align, og_max_size + yg_max_size, og_align, addr); @@ -153,12 +152,12 @@ // Failed to reserve at specified address - the requested memory // region is taken already, for example, by 'java' launcher. // Try again to reserver heap higher. - addr = Universe::preferred_heap_base(total_reserved, Universe::ZeroBasedNarrowOop); + addr = Universe::preferred_heap_base(total_reserved, og_align, Universe::ZeroBasedNarrowOop); ReservedHeapSpace heap_rs0(pg_max_size, pg_align, og_max_size + yg_max_size, og_align, addr); if (addr != NULL && !heap_rs0.is_reserved()) { // Failed to reserve at specified address again - give up. - addr = Universe::preferred_heap_base(total_reserved, Universe::HeapBasedNarrowOop); + addr = Universe::preferred_heap_base(total_reserved, og_align, Universe::HeapBasedNarrowOop); assert(addr == NULL, ""); ReservedHeapSpace heap_rs1(pg_max_size, pg_align, og_max_size + yg_max_size, og_align, addr);
--- a/src/share/vm/memory/collectorPolicy.cpp Mon Oct 14 23:15:43 2013 -0700 +++ b/src/share/vm/memory/collectorPolicy.cpp Wed Oct 23 10:15:50 2013 -0700 @@ -230,6 +230,8 @@ alignment = lcm(os::large_page_size(), alignment); } + assert(alignment >= min_alignment(), "Must be"); + return alignment; }
--- a/src/share/vm/memory/genCollectedHeap.cpp Mon Oct 14 23:15:43 2013 -0700 +++ b/src/share/vm/memory/genCollectedHeap.cpp Wed Oct 23 10:15:50 2013 -0700 @@ -99,17 +99,19 @@ guarantee(HeapWordSize == wordSize, "HeapWordSize must equal wordSize"); // The heap must be at least as aligned as generations. - size_t alignment = Generation::GenGrain; + size_t gen_alignment = Generation::GenGrain; _gen_specs = gen_policy()->generations(); PermanentGenerationSpec *perm_gen_spec = collector_policy()->permanent_generation(); + size_t heap_alignment = collector_policy()->max_alignment(); + // Make sure the sizes are all aligned. for (i = 0; i < _n_gens; i++) { - _gen_specs[i]->align(alignment); + _gen_specs[i]->align(gen_alignment); } - perm_gen_spec->align(alignment); + perm_gen_spec->align(heap_alignment); // If we are dumping the heap, then allocate a wasted block of address // space in order to push the heap to a lower address. This extra @@ -130,9 +132,9 @@ char* heap_address; size_t total_reserved = 0; int n_covered_regions = 0; - ReservedSpace heap_rs(0); + ReservedSpace heap_rs; - heap_address = allocate(alignment, perm_gen_spec, &total_reserved, + heap_address = allocate(heap_alignment, perm_gen_spec, &total_reserved, &n_covered_regions, &heap_rs); if (UseSharedSpaces) { @@ -142,7 +144,7 @@ } FileMapInfo* mapinfo = FileMapInfo::current_info(); mapinfo->fail_continue("Unable to reserve shared region."); - allocate(alignment, perm_gen_spec, &total_reserved, &n_covered_regions, + allocate(heap_alignment, perm_gen_spec, &total_reserved, &n_covered_regions, &heap_rs); } } @@ -207,19 +209,21 @@ const size_t pageSize = UseLargePages ? os::large_page_size() : os::vm_page_size(); + assert(alignment % pageSize == 0, "Must be"); + for (int i = 0; i < _n_gens; i++) { total_reserved = add_and_check_overflow(total_reserved, _gen_specs[i]->max_size()); n_covered_regions += _gen_specs[i]->n_covered_regions(); } - assert(total_reserved % pageSize == 0, - err_msg("Gen size; total_reserved=" SIZE_FORMAT ", pageSize=" - SIZE_FORMAT, total_reserved, pageSize)); + assert(total_reserved % alignment == 0, + err_msg("Gen size; total_reserved=" SIZE_FORMAT ", alignment=" + SIZE_FORMAT, total_reserved, alignment)); total_reserved = add_and_check_overflow(total_reserved, perm_gen_spec->max_size()); - assert(total_reserved % pageSize == 0, - err_msg("Perm size; total_reserved=" SIZE_FORMAT ", pageSize=" + assert(total_reserved % alignment == 0, + err_msg("Perm size; total_reserved=" SIZE_FORMAT ", alignment=" SIZE_FORMAT ", perm gen max=" SIZE_FORMAT, total_reserved, - pageSize, perm_gen_spec->max_size())); + alignment, perm_gen_spec->max_size())); n_covered_regions += perm_gen_spec->n_covered_regions(); @@ -229,7 +233,9 @@ total_reserved = add_and_check_overflow(total_reserved, misc); if (UseLargePages) { + assert(misc == 0, "CDS does not support Large Pages"); assert(total_reserved != 0, "total_reserved cannot be 0"); + assert(is_size_aligned(total_reserved, os::large_page_size()), "Must be"); total_reserved = round_up_and_check_overflow(total_reserved, os::large_page_size()); } @@ -250,7 +256,7 @@ } else { heap_address = NULL; // any address will do. if (UseCompressedOops) { - heap_address = Universe::preferred_heap_base(total_reserved, Universe::UnscaledNarrowOop); + heap_address = Universe::preferred_heap_base(total_reserved, alignment, Universe::UnscaledNarrowOop); *_total_reserved = total_reserved; *_n_covered_regions = n_covered_regions; *heap_rs = ReservedHeapSpace(total_reserved, alignment, @@ -260,13 +266,13 @@ // Failed to reserve at specified address - the requested memory // region is taken already, for example, by 'java' launcher. // Try again to reserver heap higher. - heap_address = Universe::preferred_heap_base(total_reserved, Universe::ZeroBasedNarrowOop); + heap_address = Universe::preferred_heap_base(total_reserved, alignment, Universe::ZeroBasedNarrowOop); *heap_rs = ReservedHeapSpace(total_reserved, alignment, UseLargePages, heap_address); if (heap_address != NULL && !heap_rs->is_reserved()) { // Failed to reserve at specified address again - give up. - heap_address = Universe::preferred_heap_base(total_reserved, Universe::HeapBasedNarrowOop); + heap_address = Universe::preferred_heap_base(total_reserved, alignment, Universe::HeapBasedNarrowOop); assert(heap_address == NULL, ""); *heap_rs = ReservedHeapSpace(total_reserved, alignment, UseLargePages, heap_address);
--- a/src/share/vm/memory/universe.cpp Mon Oct 14 23:15:43 2013 -0700 +++ b/src/share/vm/memory/universe.cpp Wed Oct 23 10:15:50 2013 -0700 @@ -826,17 +826,23 @@ // 32Gb // OopEncodingHeapMax == NarrowOopHeapMax << LogMinObjAlignmentInBytes; -char* Universe::preferred_heap_base(size_t heap_size, NARROW_OOP_MODE mode) { +char* Universe::preferred_heap_base(size_t heap_size, size_t alignment, NARROW_OOP_MODE mode) { + assert(is_size_aligned((size_t)OopEncodingHeapMax, alignment), "Must be"); + assert(is_size_aligned((size_t)NarrowOopHeapMax, alignment), "Must be"); + assert(is_size_aligned(heap_size, alignment), "Must be"); + + uintx heap_base_min_address_aligned = align_size_up(HeapBaseMinAddress, alignment); + size_t base = 0; #ifdef _LP64 if (UseCompressedOops) { assert(mode == UnscaledNarrowOop || mode == ZeroBasedNarrowOop || mode == HeapBasedNarrowOop, "mode is invalid"); - const size_t total_size = heap_size + HeapBaseMinAddress; + const size_t total_size = heap_size + heap_base_min_address_aligned; // Return specified base for the first request. if (!FLAG_IS_DEFAULT(HeapBaseMinAddress) && (mode == UnscaledNarrowOop)) { - base = HeapBaseMinAddress; + base = heap_base_min_address_aligned; } else if (total_size <= OopEncodingHeapMax && (mode != HeapBasedNarrowOop)) { if (total_size <= NarrowOopHeapMax && (mode == UnscaledNarrowOop) && (Universe::narrow_oop_shift() == 0)) { @@ -882,6 +888,8 @@ } } #endif + + assert(is_ptr_aligned((char*)base, alignment), "Must be"); return (char*)base; // also return NULL (don't care) for 32-bit VM }
--- a/src/share/vm/memory/universe.hpp Mon Oct 14 23:15:43 2013 -0700 +++ b/src/share/vm/memory/universe.hpp Wed Oct 23 10:15:50 2013 -0700 @@ -380,7 +380,7 @@ static NARROW_OOP_MODE narrow_oop_mode(); - static char* preferred_heap_base(size_t heap_size, NARROW_OOP_MODE mode); + static char* preferred_heap_base(size_t heap_size, size_t alignment, NARROW_OOP_MODE mode); // Historic gc information static size_t get_heap_capacity_at_last_gc() { return _heap_capacity_at_last_gc; }
--- a/src/share/vm/prims/jni.cpp Mon Oct 14 23:15:43 2013 -0700 +++ b/src/share/vm/prims/jni.cpp Wed Oct 23 10:15:50 2013 -0700 @@ -5042,9 +5042,15 @@ tty->print_cr("Running test: " #unit_test_function_call); \ unit_test_function_call +// Forward declaration +void TestReservedSpace_test(); +void TestReserveMemorySpecial_test(); + void execute_internal_vm_tests() { if (ExecuteInternalVMTests) { tty->print_cr("Running internal VM tests"); + run_unit_test(TestReservedSpace_test()); + run_unit_test(TestReserveMemorySpecial_test()); run_unit_test(GCTimerAllTest::all()); run_unit_test(arrayOopDesc::test_max_array_length()); run_unit_test(CollectedHeap::test_is_in());
--- a/src/share/vm/runtime/globals.hpp Mon Oct 14 23:15:43 2013 -0700 +++ b/src/share/vm/runtime/globals.hpp Wed Oct 23 10:15:50 2013 -0700 @@ -1917,6 +1917,9 @@ notproduct(bool, ExecuteInternalVMTests, false, \ "Enable execution of internal VM tests.") \ \ + notproduct(bool, VerboseInternalVMTests, false, \ + "Turn on logging for internal VM tests.") \ + \ product_pd(bool, UseTLAB, "Use thread-local object allocation") \ \ product_pd(bool, ResizeTLAB, \
--- a/src/share/vm/runtime/os.hpp Mon Oct 14 23:15:43 2013 -0700 +++ b/src/share/vm/runtime/os.hpp Wed Oct 23 10:15:50 2013 -0700 @@ -330,8 +330,8 @@ static char* non_memory_address_word(); // reserve, commit and pin the entire memory region - static char* reserve_memory_special(size_t size, char* addr = NULL, - bool executable = false); + static char* reserve_memory_special(size_t size, size_t alignment, + char* addr, bool executable); static bool release_memory_special(char* addr, size_t bytes); static void large_page_init(); static size_t large_page_size();
--- a/src/share/vm/runtime/virtualspace.cpp Mon Oct 14 23:15:43 2013 -0700 +++ b/src/share/vm/runtime/virtualspace.cpp Wed Oct 23 10:15:50 2013 -0700 @@ -42,8 +42,19 @@ // ReservedSpace + +// Dummy constructor +ReservedSpace::ReservedSpace() : _base(NULL), _size(0), _noaccess_prefix(0), + _alignment(0), _special(false), _executable(false) { +} + ReservedSpace::ReservedSpace(size_t size) { - initialize(size, 0, false, NULL, 0, false); + size_t page_size = os::page_size_for_region(size, size, 1); + bool large_pages = page_size != (size_t)os::vm_page_size(); + // Don't force the alignment to be large page aligned, + // since that will waste memory. + size_t alignment = os::vm_allocation_granularity(); + initialize(size, alignment, large_pages, NULL, 0, false); } ReservedSpace::ReservedSpace(size_t size, size_t alignment, @@ -323,16 +334,18 @@ if (special) { - base = os::reserve_memory_special(size, requested_address, executable); + base = os::reserve_memory_special(size, alignment, requested_address, executable); if (base != NULL) { if (failed_to_reserve_as_requested(base, requested_address, size, true)) { // OS ignored requested address. Try different address. return; } - // Check alignment constraints + // Check alignment constraints. assert((uintptr_t) base % alignment == 0, - "Large pages returned a non-aligned address"); + err_msg("Large pages returned a non-aligned address, base: " + PTR_FORMAT " alignment: " PTR_FORMAT, + base, (void*)(uintptr_t)alignment)); _special = true; } else { // failed; try to reserve regular memory below @@ -928,4 +941,188 @@ tty->print_cr(" - [low_b, high_b]: [" INTPTR_FORMAT ", " INTPTR_FORMAT "]", low_boundary(), high_boundary()); } + +/////////////// Unit tests /////////////// + +#ifndef PRODUCT + +#define test_log(...) \ + do {\ + if (VerboseInternalVMTests) { \ + tty->print_cr(__VA_ARGS__); \ + tty->flush(); \ + }\ + } while (false) + +class TestReservedSpace : AllStatic { + public: + static void small_page_write(void* addr, size_t size) { + size_t page_size = os::vm_page_size(); + + char* end = (char*)addr + size; + for (char* p = (char*)addr; p < end; p += page_size) { + *p = 1; + } + } + + static void release_memory_for_test(ReservedSpace rs) { + if (rs.special()) { + guarantee(os::release_memory_special(rs.base(), rs.size()), "Shouldn't fail"); + } else { + guarantee(os::release_memory(rs.base(), rs.size()), "Shouldn't fail"); + } + } + + static void test_reserved_space1(size_t size, size_t alignment) { + test_log("test_reserved_space1(%p)", (void*) (uintptr_t) size); + + assert(is_size_aligned(size, alignment), "Incorrect input parameters"); + + ReservedSpace rs(size, // size + alignment, // alignment + UseLargePages, // large + NULL, // requested_address + 0); // noacces_prefix + + test_log(" rs.special() == %d", rs.special()); + + assert(rs.base() != NULL, "Must be"); + assert(rs.size() == size, "Must be"); + + assert(is_ptr_aligned(rs.base(), alignment), "aligned sizes should always give aligned addresses"); + assert(is_size_aligned(rs.size(), alignment), "aligned sizes should always give aligned addresses"); + + if (rs.special()) { + small_page_write(rs.base(), size); + } + + release_memory_for_test(rs); + } + + static void test_reserved_space2(size_t size) { + test_log("test_reserved_space2(%p)", (void*)(uintptr_t)size); + + assert(is_size_aligned(size, os::vm_allocation_granularity()), "Must be at least AG aligned"); + + ReservedSpace rs(size); + + test_log(" rs.special() == %d", rs.special()); + + assert(rs.base() != NULL, "Must be"); + assert(rs.size() == size, "Must be"); + + if (rs.special()) { + small_page_write(rs.base(), size); + } + + release_memory_for_test(rs); + } + + static void test_reserved_space3(size_t size, size_t alignment, bool maybe_large) { + test_log("test_reserved_space3(%p, %p, %d)", + (void*)(uintptr_t)size, (void*)(uintptr_t)alignment, maybe_large); + + assert(is_size_aligned(size, os::vm_allocation_granularity()), "Must be at least AG aligned"); + assert(is_size_aligned(size, alignment), "Must be at least aligned against alignment"); + + bool large = maybe_large && UseLargePages && size >= os::large_page_size(); + + ReservedSpace rs(size, alignment, large, false); + + test_log(" rs.special() == %d", rs.special()); + + assert(rs.base() != NULL, "Must be"); + assert(rs.size() == size, "Must be"); + + if (rs.special()) { + small_page_write(rs.base(), size); + } + + release_memory_for_test(rs); + } + + + static void test_reserved_space1() { + size_t size = 2 * 1024 * 1024; + size_t ag = os::vm_allocation_granularity(); + + test_reserved_space1(size, ag); + test_reserved_space1(size * 2, ag); + test_reserved_space1(size * 10, ag); + } + + static void test_reserved_space2() { + size_t size = 2 * 1024 * 1024; + size_t ag = os::vm_allocation_granularity(); + + test_reserved_space2(size * 1); + test_reserved_space2(size * 2); + test_reserved_space2(size * 10); + test_reserved_space2(ag); + test_reserved_space2(size - ag); + test_reserved_space2(size); + test_reserved_space2(size + ag); + test_reserved_space2(size * 2); + test_reserved_space2(size * 2 - ag); + test_reserved_space2(size * 2 + ag); + test_reserved_space2(size * 3); + test_reserved_space2(size * 3 - ag); + test_reserved_space2(size * 3 + ag); + test_reserved_space2(size * 10); + test_reserved_space2(size * 10 + size / 2); + } + + static void test_reserved_space3() { + size_t ag = os::vm_allocation_granularity(); + + test_reserved_space3(ag, ag , false); + test_reserved_space3(ag * 2, ag , false); + test_reserved_space3(ag * 3, ag , false); + test_reserved_space3(ag * 2, ag * 2, false); + test_reserved_space3(ag * 4, ag * 2, false); + test_reserved_space3(ag * 8, ag * 2, false); + test_reserved_space3(ag * 4, ag * 4, false); + test_reserved_space3(ag * 8, ag * 4, false); + test_reserved_space3(ag * 16, ag * 4, false); + + if (UseLargePages) { + size_t lp = os::large_page_size(); + + // Without large pages + test_reserved_space3(lp, ag * 4, false); + test_reserved_space3(lp * 2, ag * 4, false); + test_reserved_space3(lp * 4, ag * 4, false); + test_reserved_space3(lp, lp , false); + test_reserved_space3(lp * 2, lp , false); + test_reserved_space3(lp * 3, lp , false); + test_reserved_space3(lp * 2, lp * 2, false); + test_reserved_space3(lp * 4, lp * 2, false); + test_reserved_space3(lp * 8, lp * 2, false); + + // With large pages + test_reserved_space3(lp, ag * 4 , true); + test_reserved_space3(lp * 2, ag * 4, true); + test_reserved_space3(lp * 4, ag * 4, true); + test_reserved_space3(lp, lp , true); + test_reserved_space3(lp * 2, lp , true); + test_reserved_space3(lp * 3, lp , true); + test_reserved_space3(lp * 2, lp * 2, true); + test_reserved_space3(lp * 4, lp * 2, true); + test_reserved_space3(lp * 8, lp * 2, true); + } + } + + static void test_reserved_space() { + test_reserved_space1(); + test_reserved_space2(); + test_reserved_space3(); + } +}; + +void TestReservedSpace_test() { + TestReservedSpace::test_reserved_space(); +} + +#endif // PRODUCT + #endif
--- a/src/share/vm/runtime/virtualspace.hpp Mon Oct 14 23:15:43 2013 -0700 +++ b/src/share/vm/runtime/virtualspace.hpp Wed Oct 23 10:15:50 2013 -0700 @@ -90,6 +90,7 @@ public: // Constructor + ReservedSpace(); ReservedSpace(size_t size); ReservedSpace(size_t size, size_t alignment, bool large, char* requested_address = NULL,
--- a/src/share/vm/services/memTracker.hpp Mon Oct 14 23:15:43 2013 -0700 +++ b/src/share/vm/services/memTracker.hpp Wed Oct 23 10:15:50 2013 -0700 @@ -302,6 +302,13 @@ } } + static inline void record_virtual_memory_release(address addr, size_t size, + Thread* thread = NULL) { + if (is_on()) { + Tracker tkr(Tracker::Release, thread); + tkr.record(addr, size); + } + } // record memory type on virtual memory base address static inline void record_virtual_memory_type(address base, MEMFLAGS flags,
--- a/src/share/vm/utilities/globalDefinitions.hpp Mon Oct 14 23:15:43 2013 -0700 +++ b/src/share/vm/utilities/globalDefinitions.hpp Wed Oct 23 10:15:50 2013 -0700 @@ -373,6 +373,14 @@ #define align_size_up_(size, alignment) (((size) + ((alignment) - 1)) & ~((alignment) - 1)) +inline bool is_size_aligned(size_t size, size_t alignment) { + return align_size_up_(size, alignment) == size; +} + +inline bool is_ptr_aligned(void* ptr, size_t alignment) { + return align_size_up_((intptr_t)ptr, (intptr_t)alignment) == (intptr_t)ptr; +} + inline intptr_t align_size_up(intptr_t size, intptr_t alignment) { return align_size_up_(size, alignment); } @@ -383,6 +391,14 @@ return align_size_down_(size, alignment); } +inline void* align_ptr_up(void* ptr, size_t alignment) { + return (void*)align_size_up((intptr_t)ptr, (intptr_t)alignment); +} + +inline void* align_ptr_down(void* ptr, size_t alignment) { + return (void*)align_size_down((intptr_t)ptr, (intptr_t)alignment); +} + // Align objects by rounding up their size, in HeapWord units. #define align_object_size_(size) align_size_up_(size, MinObjAlignment)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/runtime/memory/LargePages/TestLargePagesFlags.java Wed Oct 23 10:15:50 2013 -0700 @@ -0,0 +1,389 @@ +/* + * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* @test TestLargePagesFlags + * @summary Tests how large pages are choosen depending on the given large pages flag combinations. + * @library /testlibrary + * @run main TestLargePagesFlags + */ + +import com.oracle.java.testlibrary.OutputAnalyzer; +import com.oracle.java.testlibrary.Platform; +import com.oracle.java.testlibrary.ProcessTools; +import java.util.ArrayList; + +public class TestLargePagesFlags { + + public static void main(String [] args) throws Exception { + if (!Platform.isLinux()) { + System.out.println("Skipping. TestLargePagesFlags has only been implemented for Linux."); + return; + } + + testUseTransparentHugePages(); + testUseHugeTLBFS(); + testUseSHM(); + testCombinations(); + } + + public static void testUseTransparentHugePages() throws Exception { + if (!canUse(UseTransparentHugePages(true))) { + System.out.println("Skipping testUseTransparentHugePages"); + return; + } + + // -XX:-UseLargePages overrides all other flags. + new FlagTester() + .use(UseLargePages(false), + UseTransparentHugePages(true)) + .expect( + UseLargePages(false), + UseTransparentHugePages(false), + UseHugeTLBFS(false), + UseSHM(false)); + + // Explicitly turn on UseTransparentHugePages. + new FlagTester() + .use(UseTransparentHugePages(true)) + .expect( + UseLargePages(true), + UseTransparentHugePages(true), + UseHugeTLBFS(false), + UseSHM(false)); + + new FlagTester() + .use(UseLargePages(true), + UseTransparentHugePages(true)) + .expect( + UseLargePages(true), + UseTransparentHugePages(true), + UseHugeTLBFS(false), + UseSHM(false)); + + // Setting a specific large pages flag will turn + // off heuristics to choose large pages type. + new FlagTester() + .use(UseLargePages(true), + UseTransparentHugePages(false)) + .expect( + UseLargePages(false), + UseTransparentHugePages(false), + UseHugeTLBFS(false), + UseSHM(false)); + + // Don't turn on UseTransparentHugePages + // unless the user explicitly asks for them. + new FlagTester() + .use(UseLargePages(true)) + .expect( + UseTransparentHugePages(false)); + } + + public static void testUseHugeTLBFS() throws Exception { + if (!canUse(UseHugeTLBFS(true))) { + System.out.println("Skipping testUseHugeTLBFS"); + return; + } + + // -XX:-UseLargePages overrides all other flags. + new FlagTester() + .use(UseLargePages(false), + UseHugeTLBFS(true)) + .expect( + UseLargePages(false), + UseTransparentHugePages(false), + UseHugeTLBFS(false), + UseSHM(false)); + + // Explicitly turn on UseHugeTLBFS. + new FlagTester() + .use(UseHugeTLBFS(true)) + .expect( + UseLargePages(true), + UseTransparentHugePages(false), + UseHugeTLBFS(true), + UseSHM(false)); + + new FlagTester() + .use(UseLargePages(true), + UseHugeTLBFS(true)) + .expect( + UseLargePages(true), + UseTransparentHugePages(false), + UseHugeTLBFS(true), + UseSHM(false)); + + // Setting a specific large pages flag will turn + // off heuristics to choose large pages type. + new FlagTester() + .use(UseLargePages(true), + UseHugeTLBFS(false)) + .expect( + UseLargePages(false), + UseTransparentHugePages(false), + UseHugeTLBFS(false), + UseSHM(false)); + + // Using UseLargePages will default to UseHugeTLBFS large pages. + new FlagTester() + .use(UseLargePages(true)) + .expect( + UseLargePages(true), + UseTransparentHugePages(false), + UseHugeTLBFS(true), + UseSHM(false)); + } + + public static void testUseSHM() throws Exception { + if (!canUse(UseSHM(true))) { + System.out.println("Skipping testUseSHM"); + return; + } + + // -XX:-UseLargePages overrides all other flags. + new FlagTester() + .use(UseLargePages(false), + UseSHM(true)) + .expect( + UseLargePages(false), + UseTransparentHugePages(false), + UseHugeTLBFS(false), + UseSHM(false)); + + // Explicitly turn on UseSHM. + new FlagTester() + .use(UseSHM(true)) + .expect( + UseLargePages(true), + UseTransparentHugePages(false), + UseHugeTLBFS(false), + UseSHM(true)) ; + + new FlagTester() + .use(UseLargePages(true), + UseSHM(true)) + .expect( + UseLargePages(true), + UseTransparentHugePages(false), + UseHugeTLBFS(false), + UseSHM(true)) ; + + // Setting a specific large pages flag will turn + // off heuristics to choose large pages type. + new FlagTester() + .use(UseLargePages(true), + UseSHM(false)) + .expect( + UseLargePages(false), + UseTransparentHugePages(false), + UseHugeTLBFS(false), + UseSHM(false)); + + // Setting UseLargePages can allow the system to choose + // UseHugeTLBFS instead of UseSHM, but never UseTransparentHugePages. + new FlagTester() + .use(UseLargePages(true)) + .expect( + UseLargePages(true), + UseTransparentHugePages(false)); + } + + public static void testCombinations() throws Exception { + if (!canUse(UseSHM(true)) || !canUse(UseHugeTLBFS(true))) { + System.out.println("Skipping testUseHugeTLBFSAndUseSHMCombination"); + return; + } + + // UseHugeTLBFS takes precedence over SHM. + + new FlagTester() + .use(UseLargePages(true), + UseHugeTLBFS(true), + UseSHM(true)) + .expect( + UseLargePages(true), + UseTransparentHugePages(false), + UseHugeTLBFS(true), + UseSHM(false)); + + new FlagTester() + .use(UseLargePages(true), + UseHugeTLBFS(false), + UseSHM(true)) + .expect( + UseLargePages(true), + UseTransparentHugePages(false), + UseHugeTLBFS(false), + UseSHM(true)); + + new FlagTester() + .use(UseLargePages(true), + UseHugeTLBFS(true), + UseSHM(false)) + .expect( + UseLargePages(true), + UseTransparentHugePages(false), + UseHugeTLBFS(true), + UseSHM(false)); + + new FlagTester() + .use(UseLargePages(true), + UseHugeTLBFS(false), + UseSHM(false)) + .expect( + UseLargePages(false), + UseTransparentHugePages(false), + UseHugeTLBFS(false), + UseSHM(false)); + + + if (!canUse(UseTransparentHugePages(true))) { + return; + } + + // UseTransparentHugePages takes precedence. + + new FlagTester() + .use(UseLargePages(true), + UseTransparentHugePages(true), + UseHugeTLBFS(true), + UseSHM(true)) + .expect( + UseLargePages(true), + UseTransparentHugePages(true), + UseHugeTLBFS(false), + UseSHM(false)); + + new FlagTester() + .use(UseTransparentHugePages(true), + UseHugeTLBFS(true), + UseSHM(true)) + .expect( + UseLargePages(true), + UseTransparentHugePages(true), + UseHugeTLBFS(false), + UseSHM(false)); + } + + private static class FlagTester { + private Flag [] useFlags; + + public FlagTester use(Flag... useFlags) { + this.useFlags = useFlags; + return this; + } + + public void expect(Flag... expectedFlags) throws Exception { + if (useFlags == null) { + throw new IllegalStateException("Must run use() before expect()"); + } + + OutputAnalyzer output = executeNewJVM(useFlags); + + for (Flag flag : expectedFlags) { + System.out.println("Looking for: " + flag.flagString()); + String strValue = output.firstMatch(".* " + flag.name() + " .* :?= (\\S+).*", 1); + + if (strValue == null) { + throw new RuntimeException("Flag " + flag.name() + " couldn't be found"); + } + + if (!flag.value().equals(strValue)) { + throw new RuntimeException("Wrong value for: " + flag.name() + + " expected: " + flag.value() + + " got: " + strValue); + } + } + + output.shouldHaveExitValue(0); + } + } + + private static OutputAnalyzer executeNewJVM(Flag... flags) throws Exception { + ArrayList<String> args = new ArrayList<>(); + for (Flag flag : flags) { + args.add(flag.flagString()); + } + args.add("-XX:+PrintFlagsFinal"); + args.add("-version"); + + ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(args.toArray(new String[args.size()])); + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + + return output; + } + + private static boolean canUse(Flag flag) { + try { + new FlagTester().use(flag).expect(flag); + } catch (Exception e) { + return false; + } + + return true; + } + + private static Flag UseLargePages(boolean value) { + return new BooleanFlag("UseLargePages", value); + } + + private static Flag UseTransparentHugePages(boolean value) { + return new BooleanFlag("UseTransparentHugePages", value); + } + + private static Flag UseHugeTLBFS(boolean value) { + return new BooleanFlag("UseHugeTLBFS", value); + } + + private static Flag UseSHM(boolean value) { + return new BooleanFlag("UseSHM", value); + } + + private static class BooleanFlag implements Flag { + private String name; + private boolean value; + + BooleanFlag(String name, boolean value) { + this.name = name; + this.value = value; + } + + public String flagString() { + return "-XX:" + (value ? "+" : "-") + name; + } + + public String name() { + return name; + } + + public String value() { + return Boolean.toString(value); + } + } + + private static interface Flag { + public String flagString(); + public String name(); + public String value(); + } +}
--- a/test/testlibrary/OutputAnalyzerTest.java Mon Oct 14 23:15:43 2013 -0700 +++ b/test/testlibrary/OutputAnalyzerTest.java Wed Oct 23 10:15:50 2013 -0700 @@ -172,5 +172,22 @@ } catch (RuntimeException e) { // expected } + + { + String aaaa = "aaaa"; + String result = output.firstMatch(aaaa); + if (!aaaa.equals(result)) { + throw new Exception("firstMatch(String) faild to match. Expected: " + aaaa + " got: " + result); + } + } + + { + String aa = "aa"; + String aa_grouped_aa = aa + "(" + aa + ")"; + String result = output.firstMatch(aa_grouped_aa, 1); + if (!aa.equals(result)) { + throw new Exception("firstMatch(String, int) failed to match. Expected: " + aa + " got: " + result); + } + } } }
--- a/test/testlibrary/com/oracle/java/testlibrary/OutputAnalyzer.java Mon Oct 14 23:15:43 2013 -0700 +++ b/test/testlibrary/com/oracle/java/testlibrary/OutputAnalyzer.java Wed Oct 23 10:15:50 2013 -0700 @@ -241,7 +241,38 @@ } /** - * Verifiy the exit value of the process + * Get the captured group of the first string matching the pattern. + * stderr is searched before stdout. + * + * @param pattern The multi-line pattern to match + * @param group The group to capture + * @return The matched string or null if no match was found + */ + public String firstMatch(String pattern, int group) { + Matcher stderrMatcher = Pattern.compile(pattern, Pattern.MULTILINE).matcher(stderr); + Matcher stdoutMatcher = Pattern.compile(pattern, Pattern.MULTILINE).matcher(stdout); + if (stderrMatcher.find()) { + return stderrMatcher.group(group); + } + if (stdoutMatcher.find()) { + return stdoutMatcher.group(group); + } + return null; + } + + /** + * Get the first string matching the pattern. + * stderr is searched before stdout. + * + * @param pattern The multi-line pattern to match + * @return The matched string or null if no match was found + */ + public String firstMatch(String pattern) { + return firstMatch(pattern, 0); + } + + /** + * Verify the exit value of the process * * @param expectedExitValue Expected exit value from process * @throws RuntimeException If the exit value from the process did not match the expected value