# HG changeset patch
# User kamg
# Date 1300992148 14400
# Node ID f6b8cfca1b530e9f7fd9a0c95eeb239afdb53177
# Parent  4863fa64ae5f5c96c36c68c5c2bb765e23a5d697
7020373: JSR rewriting can overflow memory address size variables
Summary: Abort if incoming classfile's parameters would cause overflows
Reviewed-by: coleenp, dcubed, never

diff --git a/src/share/vm/oops/generateOopMap.cpp b/src/share/vm/oops/generateOopMap.cpp
--- openjdk/hotspot/src/share/vm/oops/generateOopMap.cpp
+++ openjdk/hotspot/src/share/vm/oops/generateOopMap.cpp
@@ -956,10 +956,21 @@ void GenerateOopMap::init_basic_blocks()
   // initialize the CellTypeState-related information.
   init_state();
 
-  // We allocate space for all state-vectors for all basicblocks in one huge chuck.
-  // Then in the next part of the code, we set a pointer in each _basic_block that
-  // points to each piece.
-  CellTypeState *basicBlockState = NEW_RESOURCE_ARRAY(CellTypeState, bbNo * _state_len);
+  // We allocate space for all state-vectors for all basicblocks in one huge
+  // chunk.  Then in the next part of the code, we set a pointer in each
+  // _basic_block that points to each piece.
+
+  // The product of bbNo and _state_len can get large if there are lots of
+  // basic blocks and stack/locals/monitors.  Need to check to make sure
+  // we don't overflow the capacity of a pointer.
+  if ((unsigned)bbNo > UINTPTR_MAX / sizeof(CellTypeState) / _state_len) {
+    report_error("The amount of memory required to analyze this method "
+                 "exceeds addressable range");
+    return;
+  }
+
+  CellTypeState *basicBlockState =
+      NEW_RESOURCE_ARRAY(CellTypeState, bbNo * _state_len);
   memset(basicBlockState, 0, bbNo * _state_len * sizeof(CellTypeState));
 
   // Make a pass over the basicblocks and assign their state vectors.