Mercurial > hg > release > icedtea7-forest-2.4 > jdk

changeset 7324:b4fc2a92ae21

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
PR1781: NSS PKCS11 provider fails to handle multipart AES encryption
author andrew
date Wed, 21 May 2014 15:25:53 +0100
parents c8c15663cef6
children a982f3aa40ed
files src/share/classes/sun/security/pkcs11/P11Cipher.java
diffstat 1 files changed, 235 insertions(+), 149 deletions(-) [+]
line wrap: on
line diff
--- a/src/share/classes/sun/security/pkcs11/P11Cipher.java	Thu Apr 17 17:20:13 2014 +0100
+++ b/src/share/classes/sun/security/pkcs11/P11Cipher.java	Wed May 21 15:25:53 2014 +0100
@@ -69,7 +69,7 @@
     private static interface Padding {
         // ENC: format the specified buffer with padding bytes and return the
         // actual padding length
-        int setPaddingBytes(byte[] paddingBuffer, int padLen);
+        int setPaddingBytes(byte[] paddingBuffer, int offset, int padLen);
 
         // DEC: return the length of trailing padding bytes given the specified
         // padded data
@@ -90,8 +90,8 @@
             this.blockSize = blockSize;
         }
 
-        public int setPaddingBytes(byte[] paddingBuffer, int padLen) {
-            Arrays.fill(paddingBuffer, 0, padLen, (byte) (padLen & 0x007f));
+        public int setPaddingBytes(byte[] paddingBuffer, int offset, int padLen) {
+            Arrays.fill(paddingBuffer, offset, offset + padLen, (byte) (padLen & 0x007f));
             return padLen;
         }
 
@@ -160,10 +160,16 @@
     // original IV, if in MODE_CBC or MODE_CTR
     private byte[] iv;
 
-    // number of bytes buffered internally by the native mechanism and padBuffer
-    // if we do the padding
+    // number of bytes buffered by the blockBuffer
     private int bytesBuffered;
 
+    // number of bytes buffered internally
+    private int bytesBufferedInt;
+
+    // bytes buffered from an incomplete block
+    private byte[] blockBuffer;
+    private int blockBufferLen;
+
     P11Cipher(Token token, String algorithm, long mechanism)
             throws PKCS11Exception, NoSuchAlgorithmException {
         super();
@@ -194,6 +200,9 @@
             // should not happen
             throw new ProviderException(nspe);
         }
+
+	if (blockSize > 0)
+	    blockBuffer = new byte[blockSize];
     }
 
     protected void engineSetMode(String mode) throws NoSuchAlgorithmException {
@@ -403,7 +412,8 @@
                 token.p11.C_DecryptFinal(session.id(), 0, buffer, 0, bufLen);
             }
         } catch (PKCS11Exception e) {
-            throw new ProviderException("Cancel failed", e);
+	    if (e.getErrorCode() != CKR_OPERATION_NOT_INITIALIZED)
+		throw new ProviderException("Cancel failed", e);
         } finally {
             reset();
         }
@@ -435,7 +445,9 @@
             throw ex;
         }
         bytesBuffered = 0;
+	bytesBufferedInt = 0;
         padBufferLen = 0;
+	blockBufferLen = 0;
         initialized = true;
     }
 
@@ -445,7 +457,7 @@
             return 0;
         }
 
-        int result = inLen + bytesBuffered;
+        int result = inLen + bytesBuffered + bytesBufferedInt;
         if (blockSize != 0) {
             // minus the number of bytes in the last incomplete block.
             result -= (result & (blockSize - 1));
@@ -459,7 +471,7 @@
             return 0;
         }
 
-        int result = inLen + bytesBuffered;
+        int result = inLen + bytesBuffered + bytesBufferedInt;
         if (blockSize != 0 && encrypt && paddingType != PAD_NONE) {
             // add the number of bytes to make the last block complete.
             result += (blockSize - (result & (blockSize - 1)));
@@ -471,7 +483,9 @@
     private void reset() {
         initialized = false;
         bytesBuffered = 0;
+	bytesBufferedInt = 0;
         padBufferLen = 0;
+	blockBufferLen = 0;
         if (session != null) {
             session = token.releaseSession(session);
         }
@@ -546,50 +560,65 @@
         }
         try {
             ensureInitialized();
-            int k = 0;
-            if (encrypt) {
-                k = token.p11.C_EncryptUpdate(session.id(), 0, in, inOfs, inLen,
-                        0, out, outOfs, outLen);
-            } else {
-                int newPadBufferLen = 0;
-                if (paddingObj != null) {
-                    if (padBufferLen != 0) {
-                        // NSS throws up when called with data not in multiple
-                        // of blocks. Try to work around this by holding the
-                        // extra data in padBuffer.
-                        if (padBufferLen != padBuffer.length) {
-                            int bufCapacity = padBuffer.length - padBufferLen;
-                            if (inLen > bufCapacity) {
-                                bufferInputBytes(in, inOfs, bufCapacity);
-                                inOfs += bufCapacity;
-                                inLen -= bufCapacity;
-                            } else {
-                                bufferInputBytes(in, inOfs, inLen);
-                                return 0;
-                            }
-                        }
-                        k = token.p11.C_DecryptUpdate(session.id(),
-                                0, padBuffer, 0, padBufferLen,
-                                0, out, outOfs, outLen);
-                        padBufferLen = 0;
-                    }
-                    newPadBufferLen = inLen & (blockSize - 1);
-                    if (newPadBufferLen == 0) {
-                        newPadBufferLen = padBuffer.length;
-                    }
-                    inLen -= newPadBufferLen;
-                }
-                if (inLen > 0) {
-                    k += token.p11.C_DecryptUpdate(session.id(), 0, in, inOfs,
-                            inLen, 0, out, (outOfs + k), (outLen - k));
-                }
-                // update 'padBuffer' if using our own padding impl.
-                if (paddingObj != null) {
-                    bufferInputBytes(in, inOfs + inLen, newPadBufferLen);
-                }
-            }
-            bytesBuffered += (inLen - k);
-            return k;
+            int bufRes = 0;
+	    int inRes = 0;
+	    int newBlockBufferLen = 0;
+
+	    // NSS throws up when called with data not in multiple
+	    // of blocks. Try to work around this by holding the
+	    // extra data in blockBuffer.
+	    if (blockBufferLen != 0) {
+		if (blockBufferLen != blockBuffer.length) {
+		    int bufCapacity = blockBuffer.length - blockBufferLen;
+		    if (inLen >= bufCapacity) {
+			bufferInputBytes(in, inOfs, bufCapacity);
+			inOfs += bufCapacity;
+			inLen -= bufCapacity;
+		    } else {
+			bufferInputBytes(in, inOfs, inLen);
+			return 0;
+		    }
+		}
+		if (encrypt) {
+		    bufRes = token.p11.C_EncryptUpdate(session.id(), 0, blockBuffer, 0,
+						       blockBufferLen, 0, out, outOfs,
+						       outLen);
+		} else {
+		    bufRes = token.p11.C_DecryptUpdate(session.id(), 0, blockBuffer, 0,
+						       blockBufferLen, 0, out, outOfs,
+						       outLen);
+		}
+		bytesBufferedInt += (blockBufferLen - bufRes);
+		blockBufferLen = 0;
+		bytesBuffered = 0;
+	    }
+
+	    if (inLen == 0)
+		return bufRes;
+
+	    if (blockBuffer != null) {
+		newBlockBufferLen = inLen & (blockSize - 1);
+		if (!encrypt && paddingObj != null && newBlockBufferLen == 0)
+		    // Hold the last block in the buffer if we need to unpad
+		    newBlockBufferLen = blockBuffer.length;
+		inLen -= newBlockBufferLen;
+		bufferInputBytes(in, inOfs + inLen, newBlockBufferLen);
+	    }
+
+	    if (inLen > 0) {
+		if (encrypt) {
+                    inRes = token.p11.C_EncryptUpdate(session.id(), 0, in, inOfs,
+						      inLen, 0, out, (outOfs + bufRes),
+						      (outLen - bufRes));
+		} else {
+                    inRes = token.p11.C_DecryptUpdate(session.id(), 0, in, inOfs,
+						      inLen, 0, out, (outOfs + bufRes),
+						      (outLen - bufRes));
+		}
+		bytesBufferedInt += (inLen - inRes);
+	    }
+
+            return inRes + bufRes;
         } catch (PKCS11Exception e) {
             if (e.getErrorCode() == CKR_BUFFER_TOO_SMALL) {
                 throw (ShortBufferException)
@@ -642,70 +671,79 @@
                 }
             }
 
-            int k = 0;
-            if (encrypt) {
-                if (inAddr == 0 && inArray == null) {
-                    inArray = new byte[inLen];
-                    inBuffer.get(inArray);
-                } else {
-                    inBuffer.position(origPos + inLen);
-                }
-                k = token.p11.C_EncryptUpdate(session.id(),
-                        inAddr, inArray, inOfs, inLen,
-                        outAddr, outArray, outOfs, outLen);
-            } else {
-                int newPadBufferLen = 0;
-                if (paddingObj != null) {
-                    if (padBufferLen != 0) {
-                        // NSS throws up when called with data not in multiple
-                        // of blocks. Try to work around this by holding the
-                        // extra data in padBuffer.
-                        if (padBufferLen != padBuffer.length) {
-                            int bufCapacity = padBuffer.length - padBufferLen;
-                            if (inLen > bufCapacity) {
-                                bufferInputBytes(inBuffer, bufCapacity);
-                                inOfs += bufCapacity;
-                                inLen -= bufCapacity;
-                            } else {
-                                bufferInputBytes(inBuffer, inLen);
-                                return 0;
-                            }
-                        }
-                        k = token.p11.C_DecryptUpdate(session.id(), 0,
-                                padBuffer, 0, padBufferLen, outAddr, outArray,
-                                outOfs, outLen);
-                        padBufferLen = 0;
-                    }
-                    newPadBufferLen = inLen & (blockSize - 1);
-                    if (newPadBufferLen == 0) {
-                        newPadBufferLen = padBuffer.length;
-                    }
-                    inLen -= newPadBufferLen;
-                }
-                if (inLen > 0) {
-                    if (inAddr == 0 && inArray == null) {
-                        inArray = new byte[inLen];
-                        inBuffer.get(inArray);
-                    } else {
-                        inBuffer.position(inBuffer.position() + inLen);
-                    }
-                    k += token.p11.C_DecryptUpdate(session.id(), inAddr,
-                            inArray, inOfs, inLen, outAddr, outArray,
-                            (outOfs + k), (outLen - k));
-                }
-                // update 'padBuffer' if using our own padding impl.
-                if (paddingObj != null && newPadBufferLen != 0) {
-                    bufferInputBytes(inBuffer, newPadBufferLen);
-                }
-            }
-            bytesBuffered += (inLen - k);
+            int bufRes = 0;
+	    int inRes = 0;
+	    int newBlockBufferLen = 0;
+
+	    // NSS throws up when called with data not in multiple
+	    // of blocks. Try to work around this by holding the
+	    // extra data in blockBuffer.
+	    if (blockBufferLen != 0) {
+		if (blockBufferLen != blockBuffer.length) {
+		    int bufCapacity = blockBuffer.length - blockBufferLen;
+		    if (inLen >= bufCapacity) {
+			bufferInputBytes(inBuffer, bufCapacity);
+			inOfs += bufCapacity;
+			inLen -= bufCapacity;
+		    } else {
+			bufferInputBytes(inBuffer, inLen);
+			return 0;
+		    }
+		}
+		if (encrypt) {
+		    bufRes = token.p11.C_EncryptUpdate(session.id(), 0, blockBuffer, 0,
+						       blockBufferLen, outAddr, outArray, outOfs,
+						       outLen);
+		} else {
+		    bufRes = token.p11.C_DecryptUpdate(session.id(), 0, blockBuffer, 0,
+						       blockBufferLen, outAddr, outArray, outOfs,
+						       outLen);
+		}
+		bytesBufferedInt += (blockBufferLen - bufRes);
+		blockBufferLen = 0;
+		bytesBuffered = 0;
+	    }
+
+	    if (inLen == 0)
+		return bufRes;
+
+	    if (blockBuffer != null) {
+		newBlockBufferLen = inLen & (blockSize - 1);
+		if (!encrypt && paddingObj != null && newBlockBufferLen == 0)
+		    // Hold the last block in the buffer if we need to unpad
+		    newBlockBufferLen = blockBuffer.length;
+		inLen -= newBlockBufferLen;
+		bufferInputBytes(inBuffer, newBlockBufferLen);
+	    }
+
+	    if (inAddr == 0 && inArray == null) {
+		inArray = new byte[inLen];
+		inBuffer.get(inArray);
+	    } else {
+		inBuffer.position(inBuffer.position() + inLen);
+	    }
+
+	    if (inLen > 0) {
+		if (encrypt) {
+                    inRes = token.p11.C_EncryptUpdate(session.id(), inAddr, inArray, inOfs,
+						      inLen, outAddr, outArray, (outOfs + bufRes),
+						      (outLen - bufRes));
+		} else {
+                    inRes = token.p11.C_DecryptUpdate(session.id(), inAddr, inArray, inOfs,
+						      inLen, outAddr, outArray, (outOfs + bufRes),
+						      (outLen - bufRes));
+		}
+		bytesBufferedInt += (inLen - inRes);
+	    }
+
+	    int total = inRes + bufRes;
             if (!(outBuffer instanceof DirectBuffer) &&
                     !outBuffer.hasArray()) {
-                outBuffer.put(outArray, outOfs, k);
+                outBuffer.put(outArray, outOfs, total);
             } else {
-                outBuffer.position(outBuffer.position() + k);
+                outBuffer.position(outBuffer.position() + total);
             }
-            return k;
+	    return total;
         } catch (PKCS11Exception e) {
             // Reset input buffer to its original position for
             inBuffer.position(origPos);
@@ -722,45 +760,69 @@
             throws ShortBufferException, IllegalBlockSizeException,
             BadPaddingException {
         int requiredOutLen = doFinalLength(0);
+	boolean updating = false;
+	PKCS11Exception err = null;
+
         if (outLen < requiredOutLen) {
             throw new ShortBufferException();
         }
         try {
             ensureInitialized();
             int k = 0;
-            if (encrypt) {
-                if (paddingObj != null) {
-                    int actualPadLen = paddingObj.setPaddingBytes(padBuffer,
-                            requiredOutLen - bytesBuffered);
-                    k = token.p11.C_EncryptUpdate(session.id(),
-                            0, padBuffer, 0, actualPadLen,
-                            0, out, outOfs, outLen);
-                }
+	    if (encrypt) {
+		// Do we need to pad?
+		if (paddingObj != null) {
+		    int actualPadLen = paddingObj.setPaddingBytes(blockBuffer,
+			blockBufferLen, blockSize - blockBufferLen);
+		    blockBufferLen = blockSize;
+		}
+		updating = true;
+		k = token.p11.C_EncryptUpdate(session.id(),
+					      0, blockBuffer, 0, blockBufferLen,
+					      0, out, outOfs, outLen);
+		updating = false;
                 k += token.p11.C_EncryptFinal(session.id(),
                         0, out, (outOfs + k), (outLen - k));
             } else {
+		if (blockBufferLen != 0) {
+		    if (paddingObj == null)
+			k = token.p11.C_DecryptUpdate(session.id(), 0,
+						      blockBuffer, 0, blockBufferLen, 0,
+						      out, outOfs, outLen);
+		    else
+			k = token.p11.C_DecryptUpdate(session.id(), 0,
+						      blockBuffer, 0, blockBufferLen, 0,
+						      padBuffer, 0, padBuffer.length);
+		}
                 if (paddingObj != null) {
-                    if (padBufferLen != 0) {
-                        k = token.p11.C_DecryptUpdate(session.id(), 0,
-                                padBuffer, 0, padBufferLen, 0, padBuffer, 0,
-                                padBuffer.length);
-                    }
                     k += token.p11.C_DecryptFinal(session.id(), 0, padBuffer, k,
                             padBuffer.length - k);
                     int actualPadLen = paddingObj.unpad(padBuffer, k);
                     k -= actualPadLen;
                     System.arraycopy(padBuffer, 0, out, outOfs, k);
                 } else {
-                    k = token.p11.C_DecryptFinal(session.id(), 0, out, outOfs,
-                            outLen);
+                    k += token.p11.C_DecryptFinal(session.id(), 0, out, (outOfs + k),
+						  (outLen - k));
                 }
             }
             return k;
         } catch (PKCS11Exception e) {
+	    err = e;
             handleException(e);
             throw new ProviderException("doFinal() failed", e);
         } finally {
-            reset();
+            if (err != null) {
+		if (err.getErrorCode() != CKR_BUFFER_TOO_SMALL) {
+		    if (updating)
+			// Work around NSS not cancelling the
+			// operation on an error in update
+			cancelOperation();
+		    else
+			reset();
+		}
+	    } else {
+		reset();
+	    }
         }
     }
 
@@ -769,6 +831,9 @@
             BadPaddingException {
         int outLen = outBuffer.remaining();
         int requiredOutLen = doFinalLength(0);
+	boolean updating = false;
+	PKCS11Exception err = null;
+
         if (outLen < requiredOutLen) {
             throw new ShortBufferException();
         }
@@ -793,24 +858,32 @@
 
             int k = 0;
 
-            if (encrypt) {
-                if (paddingObj != null) {
-                    int actualPadLen = paddingObj.setPaddingBytes(padBuffer,
-                            requiredOutLen - bytesBuffered);
-                    k = token.p11.C_EncryptUpdate(session.id(),
-                            0, padBuffer, 0, actualPadLen,
-                            outAddr, outArray, outOfs, outLen);
-                }
+	    if (encrypt) {
+		// Do we need to pad?
+		if (paddingObj != null) {
+		    int actualPadLen = paddingObj.setPaddingBytes(blockBuffer,
+			blockBufferLen, blockSize - blockBufferLen);
+		    blockBufferLen = blockSize;
+		}
+		updating = true;
+		k = token.p11.C_EncryptUpdate(session.id(),
+						  0, blockBuffer, 0, blockBufferLen,
+						  outAddr, outArray, outOfs, outLen);
+		updating = false;
                 k += token.p11.C_EncryptFinal(session.id(),
                         outAddr, outArray, (outOfs + k), (outLen - k));
-            } else {
+	    } else {
+		if (blockBufferLen != 0) {
+		    k = token.p11.C_DecryptUpdate(session.id(), 0,
+						  blockBuffer, 0, blockBufferLen, outAddr,
+						  outArray, outOfs, outLen);
+		} else {
+			k = token.p11.C_DecryptUpdate(session.id(), 0,
+						      blockBuffer, 0, blockBufferLen, 0,
+						      padBuffer, 0, padBuffer.length);
+		}
+
                 if (paddingObj != null) {
-                    if (padBufferLen != 0) {
-                        k = token.p11.C_DecryptUpdate(session.id(),
-                                0, padBuffer, 0, padBufferLen,
-                                0, padBuffer, 0, padBuffer.length);
-                        padBufferLen = 0;
-                    }
                     k += token.p11.C_DecryptFinal(session.id(),
                             0, padBuffer, k, padBuffer.length - k);
                     int actualPadLen = paddingObj.unpad(padBuffer, k);
@@ -819,7 +892,8 @@
                     outOfs = 0;
                 } else {
                     k = token.p11.C_DecryptFinal(session.id(),
-                            outAddr, outArray, outOfs, outLen);
+						 outAddr, outArray,
+						 (outOfs + k), (outLen - k));
                 }
             }
             if ((!encrypt && paddingObj != null) ||
@@ -831,10 +905,22 @@
             }
             return k;
         } catch (PKCS11Exception e) {
+	    err = e;
             handleException(e);
             throw new ProviderException("doFinal() failed", e);
         } finally {
-            reset();
+            if (err != null) {
+		if (err.getErrorCode() != CKR_BUFFER_TOO_SMALL) {
+		    if (updating)
+			// Work around NSS not cancelling the
+			// operation on an error in update
+			cancelOperation();
+		    else
+			reset();
+		}
+	    } else {
+		reset();
+	    }
         }
     }
 
@@ -875,14 +961,14 @@
     }
 
     private final void bufferInputBytes(byte[] in, int inOfs, int len) {
-        System.arraycopy(in, inOfs, padBuffer, padBufferLen, len);
-        padBufferLen += len;
+        System.arraycopy(in, inOfs, blockBuffer, blockBufferLen, len);
+        blockBufferLen += len;
         bytesBuffered += len;
     }
 
     private final void bufferInputBytes(ByteBuffer inBuffer, int len) {
-        inBuffer.get(padBuffer, padBufferLen, len);
-        padBufferLen += len;
+        inBuffer.get(blockBuffer, blockBufferLen, len);
+        blockBufferLen += len;
         bytesBuffered += len;
     }
 }