# HG changeset patch
# User kamg
# Date 1296505046 18000
# Node ID a6f5011d46a97d3e710aaed5c8ea85af04236c28
# Parent  2c8e1acf0433db897eb3bc8f6e1276b2c84769b7
6878713: Verifier heap corruption, relating to backward jsrs
Summary: Added overflow detection in arena Amalloc methods
Reviewed-by: coleenp, phh

diff --git a/src/share/vm/memory/allocation.cpp b/src/share/vm/memory/allocation.cpp
--- openjdk/hotspot/src/share/vm/memory/allocation.cpp
+++ openjdk/hotspot/src/share/vm/memory/allocation.cpp
@@ -377,6 +377,9 @@ size_t Arena::used() const {
   return sum;                   // Return total consumed space.
 }
 
+void Arena::signal_out_of_memory(size_t sz, const char* whence) const {
+  vm_exit_out_of_memory(sz, whence);
+}
 
 // Grow a new Chunk
 void* Arena::grow( size_t x ) {
@@ -386,8 +389,9 @@ void* Arena::grow( size_t x ) {
   Chunk *k = _chunk;            // Get filled-up chunk address
   _chunk = new (len) Chunk(len);
 
-  if (_chunk == NULL)
-      vm_exit_out_of_memory(len * Chunk::aligned_overhead_size(), "Arena::grow");
+  if (_chunk == NULL) {
+    signal_out_of_memory(len * Chunk::aligned_overhead_size(), "Arena::grow");
+  }
 
   if (k) k->set_next(_chunk);   // Append new chunk to end of linked list
   else _first = _chunk;
@@ -484,6 +488,7 @@ void* Arena::malloc(size_t size) {
 // for debugging with UseMallocOnly
 void* Arena::internal_malloc_4(size_t x) {
   assert( (x&(sizeof(char*)-1)) == 0, "misaligned size" );
+  check_for_overflow(x, "Arena::internal_malloc_4");
   if (_hwm + x > _max) {
     return grow(x);
   } else {
diff --git a/src/share/vm/memory/allocation.hpp b/src/share/vm/memory/allocation.hpp
--- openjdk/hotspot/src/share/vm/memory/allocation.hpp
+++ openjdk/hotspot/src/share/vm/memory/allocation.hpp
@@ -194,6 +194,15 @@ protected:
   friend class AllocStats;
   debug_only(void* malloc(size_t size);)
   debug_only(void* internal_malloc_4(size_t x);)
+
+  void signal_out_of_memory(size_t request, const char* whence) const;
+
+  void check_for_overflow(size_t request, const char* whence) const {
+    if (UINTPTR_MAX - request < (uintptr_t)_hwm) {
+      signal_out_of_memory(request, whence);
+    }
+  }
+
  public:
   Arena();
   Arena(size_t init_size);
@@ -207,6 +216,7 @@ protected:
     assert(is_power_of_2(ARENA_AMALLOC_ALIGNMENT) , "should be a power of 2");
     x = ARENA_ALIGN(x);
     debug_only(if (UseMallocOnly) return malloc(x);)
+    check_for_overflow(x, "Arena::Amalloc");
     NOT_PRODUCT(_bytes_allocated += x);
     if (_hwm + x > _max) {
       return grow(x);
@@ -220,6 +230,7 @@ protected:
   void *Amalloc_4(size_t x) {
     assert( (x&(sizeof(char*)-1)) == 0, "misaligned size" );
     debug_only(if (UseMallocOnly) return malloc(x);)
+    check_for_overflow(x, "Arena::Amalloc_4");
     NOT_PRODUCT(_bytes_allocated += x);
     if (_hwm + x > _max) {
       return grow(x);
@@ -240,6 +251,7 @@ protected:
     size_t delta = (((size_t)_hwm + DALIGN_M1) & ~DALIGN_M1) - (size_t)_hwm;
     x += delta;
 #endif
+    check_for_overflow(x, "Arena::Amalloc_D");
     NOT_PRODUCT(_bytes_allocated += x);
     if (_hwm + x > _max) {
       return grow(x); // grow() returns a result aligned >= 8 bytes.
diff --git a/src/share/vm/utilities/globalDefinitions_gcc.hpp b/src/share/vm/utilities/globalDefinitions_gcc.hpp
--- openjdk/hotspot/src/share/vm/utilities/globalDefinitions_gcc.hpp
+++ openjdk/hotspot/src/share/vm/utilities/globalDefinitions_gcc.hpp
@@ -72,6 +72,7 @@
 # endif
 
 #ifdef LINUX
+#define __STDC_LIMIT_MACROS
 #include <inttypes.h>
 #include <signal.h>
 #include <ucontext.h>
diff --git a/src/share/vm/utilities/globalDefinitions_sparcWorks.hpp b/src/share/vm/utilities/globalDefinitions_sparcWorks.hpp
--- openjdk/hotspot/src/share/vm/utilities/globalDefinitions_sparcWorks.hpp
+++ openjdk/hotspot/src/share/vm/utilities/globalDefinitions_sparcWorks.hpp
@@ -141,6 +141,17 @@ typedef unsigned int            uintptr_
 // If this gets an error, figure out a symbol XXX that implies the
 // prior definition of intptr_t, and add "&& !defined(XXX)" above.
 #endif
+#endif
+
+// On solaris 8, UINTPTR_MAX is defined as empty.  
+// Everywhere else it's an actual value.
+#if UINTPTR_MAX - 1 == -1
+#undef UINTPTR_MAX
+#ifdef _LP64
+#define UINTPTR_MAX UINT64_MAX
+#else
+#define UINTPTR_MAX UINT32_MAX
+#endif /* ifdef _LP64 */
 #endif
 
 // Additional Java basic types
diff --git a/src/share/vm/utilities/globalDefinitions_visCPP.hpp b/src/share/vm/utilities/globalDefinitions_visCPP.hpp
--- openjdk/hotspot/src/share/vm/utilities/globalDefinitions_visCPP.hpp
+++ openjdk/hotspot/src/share/vm/utilities/globalDefinitions_visCPP.hpp
@@ -36,6 +36,7 @@
 # include <stdio.h> // for va_list
 # include <time.h>
 # include <fcntl.h>
+# include <limits.h>
 // Need this on windows to get the math constants (e.g., M_PI).
 #define _USE_MATH_DEFINES
 # include <math.h>
@@ -92,6 +93,14 @@ typedef signed   __int64 ssize_t;
 #else
 typedef signed   int intptr_t;
 typedef signed   int ssize_t;
+#endif
+
+#ifndef UINTPTR_MAX
+#ifdef _WIN64
+#define UINTPTR_MAX _UI64_MAX
+#else
+#define UINTPTR_MAX _UI32_MAX
+#endif
 #endif
 
 //----------------------------------------------------------------------------------------------------