--- a/src/share/native/sun/java2d/loops/TransformHelper.c	Fri Jun 10 13:22:37 2011 -0700
+++ b/src/share/native/sun/java2d/loops/TransformHelper.c	Fri Jun 10 14:15:50 2011 -0700
@@ -284,7 +284,7 @@
     TransformHelperFunc *pHelperFunc;
     TransformInterpFunc *pInterpFunc;
     jdouble xorig, yorig;
-    jint numedges;
+    jlong numedges;
     jint *pEdges;
     jint edgebuf[2 + MAXEDGES * 2];
     union {
@@ -379,19 +379,44 @@
     }
     Region_IntersectBounds(&clipInfo, &dstInfo.bounds);
 
-    numedges = (dstInfo.bounds.y2 - dstInfo.bounds.y1);
-    if (numedges > MAXEDGES) {
-        pEdges = malloc((2 + 2 * numedges) * sizeof (*pEdges));
-        if (pEdges == NULL) {
-            SurfaceData_InvokeUnlock(env, dstOps, &dstInfo);
-            SurfaceData_InvokeUnlock(env, srcOps, &srcInfo);
-            /* edgeArray should already contain zeros for min/maxy */
-            return;
-        }
+    numedges = (((jlong) dstInfo.bounds.y2) - ((jlong) dstInfo.bounds.y1));
+    if (numedges <= 0) {
+        pEdges = NULL;
+    } else if (!JNU_IsNull(env, edgeArray)) {
+        /*
+         * Ideally Java should allocate an array large enough, but if
+         * we ever have a miscommunication about the number of edge
+         * lines, or if the Java array calculation should overflow to
+         * a positive number and succeed in allocating an array that
+         * is too small, we need to verify that it can still hold the
+         * number of integers that we plan to store to be safe.
+         */
+        jsize edgesize = (*env)->GetArrayLength(env, edgeArray);
+        /* (edgesize/2 - 1) should avoid any overflow or underflow. */
+        pEdges = (((edgesize / 2) - 1) >= numedges)
+            ? (*env)->GetPrimitiveArrayCritical(env, edgeArray, NULL)
+            : NULL;
+    } else if (numedges > MAXEDGES) {
+        /* numedges variable (jlong) can be at most ((1<<32)-1) */
+        /* memsize can overflow a jint, but not a jlong */
+        jlong memsize = ((numedges * 2) + 2) * sizeof(*pEdges);
+        pEdges = (memsize == ((size_t) memsize))
+            ? malloc((size_t) memsize)
+            : NULL;
     } else {
         pEdges = edgebuf;
     }
 
+    if (pEdges == NULL) {
+        if (numedges > 0) {
+            JNU_ThrowInternalError(env, "Unable to allocate edge list");
+        }
+        SurfaceData_InvokeUnlock(env, dstOps, &dstInfo);
+        SurfaceData_InvokeUnlock(env, srcOps, &srcInfo);
+        /* edgeArray should already contain zeros for min/maxy */
+        return;
+    }
+
     Transform_GetInfo(env, itxform, &itxInfo);
 
     if (!Region_IsEmpty(&clipInfo)) {
@@ -500,14 +525,13 @@
     } else {
         pEdges[0] = pEdges[1] = 0;
     }
+    if (!JNU_IsNull(env, edgeArray)) {
+        (*env)->ReleasePrimitiveArrayCritical(env, edgeArray, pEdges, 0);
+    } else if (pEdges != edgebuf) {
+        free(pEdges);
+    }
     SurfaceData_InvokeUnlock(env, dstOps, &dstInfo);
     SurfaceData_InvokeUnlock(env, srcOps, &srcInfo);
-    if (!JNU_IsNull(env, edgeArray)) {
-        (*env)->SetIntArrayRegion(env, edgeArray, 0, 2+numedges*2, pEdges);
-    }
-    if (pEdges != edgebuf) {
-        free(pEdges);
-    }
 }
 
 static void

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/sun/java2d/loops/TransformOverflow.java	Fri Jun 10 14:15:50 2011 -0700
@@ -0,0 +1,30 @@
+/**
+ * @test %W% %E%
+ * @bug 7023640
+ * @summary Checks for malloc overflow when we transform an image
+ *          into a really tall destination
+ * @run main/othervm/timeout=5000 -Xmx1g TransformOverflow
+ */
+
+import java.awt.image.BufferedImage;
+import java.awt.Graphics2D;
+
+public class TransformOverflow {
+    public static void main(String argv[]) {
+        test(1, 0x20000000);
+        System.out.println("done");
+    }
+
+    public static void test(int w, int h) {
+        BufferedImage bimg =
+            new BufferedImage(w, h, BufferedImage.TYPE_BYTE_GRAY);
+        Graphics2D g2d = bimg.createGraphics();
+        g2d.scale(1.0, Math.PI);
+        g2d.shear(0.0, 1.0);
+        try {
+            g2d.drawImage(bimg, 0, 0, null);
+        } catch (Throwable t) {
+            t.printStackTrace();
+        }
+    }
+}