Mercurial > hg > icedtea9-forest > jdk
changeset 4887:417d91754849
7096080: UTF8 update and new CESU-8 charset
7082884: Incorrect UTF8 conversion for sequence ED 31
7082883: Incorrect UTF8 conversion for sequence fc 80 80 8f bf bf
Summary: Updated UTF8 and added CESU-8 to following the latest Standard
Reviewed-by: alanb
| author | sherman |
|---|---|
| date | Mon, 07 Nov 2011 13:46:02 -0800 |
| parents | ead9dabe8c75 |
| children | 0bb498332894 f410b91caf45 |
| files | make/java/nio/FILES_java.gmk src/share/classes/sun/nio/cs/CESU_8.java src/share/classes/sun/nio/cs/UTF_8.java src/share/classes/sun/nio/cs/standard-charsets test/java/nio/charset/coders/Errors.java test/sun/nio/cs/TestStringCoding.java test/sun/nio/cs/TestStringCodingUTF8.java test/sun/nio/cs/TestUTF8.java |
| diffstat | 8 files changed, 904 insertions(+), 111 deletions(-) [+] |
line wrap: on
line diff
--- a/make/java/nio/FILES_java.gmk Sat Nov 05 00:00:43 2011 -0700 +++ b/make/java/nio/FILES_java.gmk Mon Nov 07 13:46:02 2011 -0800 @@ -232,6 +232,7 @@ sun/nio/cs/UTF_16BE.java \ sun/nio/cs/UTF_16LE.java \ sun/nio/cs/UTF_8.java \ + sun/nio/cs/CESU_8.java \ sun/nio/cs/Unicode.java \ sun/nio/cs/UnicodeDecoder.java \ sun/nio/cs/UnicodeEncoder.java \
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/classes/sun/nio/cs/CESU_8.java Mon Nov 07 13:46:02 2011 -0800 @@ -0,0 +1,604 @@ +/* + * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package sun.nio.cs; + +import java.nio.Buffer; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CoderResult; +import java.nio.charset.CodingErrorAction; + +/* Legal CESU-8 Byte Sequences + * + * # Code Points Bits Bit/Byte pattern + * 1 7 0xxxxxxx + * U+0000..U+007F 00..7F + * + * 2 11 110xxxxx 10xxxxxx + * U+0080..U+07FF C2..DF 80..BF + * + * 3 16 1110xxxx 10xxxxxx 10xxxxxx + * U+0800..U+0FFF E0 A0..BF 80..BF + * U+1000..U+FFFF E1..EF 80..BF 80..BF + * + */ + +class CESU_8 extends Unicode +{ + public CESU_8() { + super("CESU-8", StandardCharsets.aliases_CESU_8); + } + + public String historicalName() { + return "CESU8"; + } + + public CharsetDecoder newDecoder() { + return new Decoder(this); + } + + public CharsetEncoder newEncoder() { + return new Encoder(this); + } + + private static final void updatePositions(Buffer src, int sp, + Buffer dst, int dp) { + src.position(sp - src.arrayOffset()); + dst.position(dp - dst.arrayOffset()); + } + + private static class Decoder extends CharsetDecoder + implements ArrayDecoder { + private Decoder(Charset cs) { + super(cs, 1.0f, 1.0f); + } + + private static boolean isNotContinuation(int b) { + return (b & 0xc0) != 0x80; + } + + // [E0] [A0..BF] [80..BF] + // [E1..EF] [80..BF] [80..BF] + private static boolean isMalformed3(int b1, int b2, int b3) { + return (b1 == (byte)0xe0 && (b2 & 0xe0) == 0x80) || + (b2 & 0xc0) != 0x80 || (b3 & 0xc0) != 0x80; + } + + // only used when there is only one byte left in src buffer + private static boolean isMalformed3_2(int b1, int b2) { + return (b1 == (byte)0xe0 && (b2 & 0xe0) == 0x80) || + (b2 & 0xc0) != 0x80; + } + + + // [F0] [90..BF] [80..BF] [80..BF] + // [F1..F3] [80..BF] [80..BF] [80..BF] + // [F4] [80..8F] [80..BF] [80..BF] + // only check 80-be range here, the [0xf0,0x80...] and [0xf4,0x90-...] + // will be checked by Character.isSupplementaryCodePoint(uc) + private static boolean isMalformed4(int b2, int b3, int b4) { + return (b2 & 0xc0) != 0x80 || (b3 & 0xc0) != 0x80 || + (b4 & 0xc0) != 0x80; + } + + // only used when there is less than 4 bytes left in src buffer + private static boolean isMalformed4_2(int b1, int b2) { + return (b1 == 0xf0 && b2 == 0x90) || + (b2 & 0xc0) != 0x80; + } + + private static boolean isMalformed4_3(int b3) { + return (b3 & 0xc0) != 0x80; + } + + private static CoderResult malformedN(ByteBuffer src, int nb) { + switch (nb) { + case 1: + case 2: // always 1 + return CoderResult.malformedForLength(1); + case 3: + int b1 = src.get(); + int b2 = src.get(); // no need to lookup b3 + return CoderResult.malformedForLength( + ((b1 == (byte)0xe0 && (b2 & 0xe0) == 0x80) || + isNotContinuation(b2)) ? 1 : 2); + case 4: // we don't care the speed here + b1 = src.get() & 0xff; + b2 = src.get() & 0xff; + if (b1 > 0xf4 || + (b1 == 0xf0 && (b2 < 0x90 || b2 > 0xbf)) || + (b1 == 0xf4 && (b2 & 0xf0) != 0x80) || + isNotContinuation(b2)) + return CoderResult.malformedForLength(1); + if (isNotContinuation(src.get())) + return CoderResult.malformedForLength(2); + return CoderResult.malformedForLength(3); + default: + assert false; + return null; + } + } + + private static CoderResult malformed(ByteBuffer src, int sp, + CharBuffer dst, int dp, + int nb) + { + src.position(sp - src.arrayOffset()); + CoderResult cr = malformedN(src, nb); + updatePositions(src, sp, dst, dp); + return cr; + } + + + private static CoderResult malformed(ByteBuffer src, + int mark, int nb) + { + src.position(mark); + CoderResult cr = malformedN(src, nb); + src.position(mark); + return cr; + } + + private static CoderResult malformedForLength(ByteBuffer src, + int sp, + CharBuffer dst, + int dp, + int malformedNB) + { + updatePositions(src, sp, dst, dp); + return CoderResult.malformedForLength(malformedNB); + } + + private static CoderResult malformedForLength(ByteBuffer src, + int mark, + int malformedNB) + { + src.position(mark); + return CoderResult.malformedForLength(malformedNB); + } + + + private static CoderResult xflow(Buffer src, int sp, int sl, + Buffer dst, int dp, int nb) { + updatePositions(src, sp, dst, dp); + return (nb == 0 || sl - sp < nb) + ? CoderResult.UNDERFLOW : CoderResult.OVERFLOW; + } + + private static CoderResult xflow(Buffer src, int mark, int nb) { + src.position(mark); + return (nb == 0 || src.remaining() < nb) + ? CoderResult.UNDERFLOW : CoderResult.OVERFLOW; + } + + private CoderResult decodeArrayLoop(ByteBuffer src, + CharBuffer dst) + { + // This method is optimized for ASCII input. + byte[] sa = src.array(); + int sp = src.arrayOffset() + src.position(); + int sl = src.arrayOffset() + src.limit(); + + char[] da = dst.array(); + int dp = dst.arrayOffset() + dst.position(); + int dl = dst.arrayOffset() + dst.limit(); + int dlASCII = dp + Math.min(sl - sp, dl - dp); + + // ASCII only loop + while (dp < dlASCII && sa[sp] >= 0) + da[dp++] = (char) sa[sp++]; + while (sp < sl) { + int b1 = sa[sp]; + if (b1 >= 0) { + // 1 byte, 7 bits: 0xxxxxxx + if (dp >= dl) + return xflow(src, sp, sl, dst, dp, 1); + da[dp++] = (char) b1; + sp++; + } else if ((b1 >> 5) == -2 && (b1 & 0x1e) != 0) { + // 2 bytes, 11 bits: 110xxxxx 10xxxxxx + if (sl - sp < 2 || dp >= dl) + return xflow(src, sp, sl, dst, dp, 2); + int b2 = sa[sp + 1]; + if (isNotContinuation(b2)) + return malformedForLength(src, sp, dst, dp, 1); + da[dp++] = (char) (((b1 << 6) ^ b2) + ^ + (((byte) 0xC0 << 6) ^ + ((byte) 0x80 << 0))); + sp += 2; + } else if ((b1 >> 4) == -2) { + // 3 bytes, 16 bits: 1110xxxx 10xxxxxx 10xxxxxx + int srcRemaining = sl - sp; + if (srcRemaining < 3 || dp >= dl) { + if (srcRemaining > 1 && isMalformed3_2(b1, sa[sp + 1])) + return malformedForLength(src, sp, dst, dp, 1); + return xflow(src, sp, sl, dst, dp, 3); + } + int b2 = sa[sp + 1]; + int b3 = sa[sp + 2]; + if (isMalformed3(b1, b2, b3)) + return malformed(src, sp, dst, dp, 3); + da[dp++] = (char) + ((b1 << 12) ^ + (b2 << 6) ^ + (b3 ^ + (((byte) 0xE0 << 12) ^ + ((byte) 0x80 << 6) ^ + ((byte) 0x80 << 0)))); + sp += 3; + } else { + return malformed(src, sp, dst, dp, 1); + } + } + return xflow(src, sp, sl, dst, dp, 0); + } + + private CoderResult decodeBufferLoop(ByteBuffer src, + CharBuffer dst) + { + int mark = src.position(); + int limit = src.limit(); + while (mark < limit) { + int b1 = src.get(); + if (b1 >= 0) { + // 1 byte, 7 bits: 0xxxxxxx + if (dst.remaining() < 1) + return xflow(src, mark, 1); // overflow + dst.put((char) b1); + mark++; + } else if ((b1 >> 5) == -2 && (b1 & 0x1e) != 0) { + // 2 bytes, 11 bits: 110xxxxx 10xxxxxx + if (limit - mark < 2|| dst.remaining() < 1) + return xflow(src, mark, 2); + int b2 = src.get(); + if (isNotContinuation(b2)) + return malformedForLength(src, mark, 1); + dst.put((char) (((b1 << 6) ^ b2) + ^ + (((byte) 0xC0 << 6) ^ + ((byte) 0x80 << 0)))); + mark += 2; + } else if ((b1 >> 4) == -2) { + // 3 bytes, 16 bits: 1110xxxx 10xxxxxx 10xxxxxx + int srcRemaining = limit - mark; + if (srcRemaining < 3 || dst.remaining() < 1) { + if (srcRemaining > 1 && isMalformed3_2(b1, src.get())) + return malformedForLength(src, mark, 1); + return xflow(src, mark, 3); + } + int b2 = src.get(); + int b3 = src.get(); + if (isMalformed3(b1, b2, b3)) + return malformed(src, mark, 3); + dst.put((char) + ((b1 << 12) ^ + (b2 << 6) ^ + (b3 ^ + (((byte) 0xE0 << 12) ^ + ((byte) 0x80 << 6) ^ + ((byte) 0x80 << 0))))); + mark += 3; + } else { + return malformed(src, mark, 1); + } + } + return xflow(src, mark, 0); + } + + protected CoderResult decodeLoop(ByteBuffer src, + CharBuffer dst) + { + if (src.hasArray() && dst.hasArray()) + return decodeArrayLoop(src, dst); + else + return decodeBufferLoop(src, dst); + } + + private static ByteBuffer getByteBuffer(ByteBuffer bb, byte[] ba, int sp) + { + if (bb == null) + bb = ByteBuffer.wrap(ba); + bb.position(sp); + return bb; + } + + // returns -1 if there is/are malformed byte(s) and the + // "action" for malformed input is not REPLACE. + public int decode(byte[] sa, int sp, int len, char[] da) { + final int sl = sp + len; + int dp = 0; + int dlASCII = Math.min(len, da.length); + ByteBuffer bb = null; // only necessary if malformed + + // ASCII only optimized loop + while (dp < dlASCII && sa[sp] >= 0) + da[dp++] = (char) sa[sp++]; + + while (sp < sl) { + int b1 = sa[sp++]; + if (b1 >= 0) { + // 1 byte, 7 bits: 0xxxxxxx + da[dp++] = (char) b1; + } else if ((b1 >> 5) == -2 && (b1 & 0x1e) != 0) { + // 2 bytes, 11 bits: 110xxxxx 10xxxxxx + if (sp < sl) { + int b2 = sa[sp++]; + if (isNotContinuation(b2)) { + if (malformedInputAction() != CodingErrorAction.REPLACE) + return -1; + da[dp++] = replacement().charAt(0); + sp--; // malformedN(bb, 2) always returns 1 + } else { + da[dp++] = (char) (((b1 << 6) ^ b2)^ + (((byte) 0xC0 << 6) ^ + ((byte) 0x80 << 0))); + } + continue; + } + if (malformedInputAction() != CodingErrorAction.REPLACE) + return -1; + da[dp++] = replacement().charAt(0); + return dp; + } else if ((b1 >> 4) == -2) { + // 3 bytes, 16 bits: 1110xxxx 10xxxxxx 10xxxxxx + if (sp + 1 < sl) { + int b2 = sa[sp++]; + int b3 = sa[sp++]; + if (isMalformed3(b1, b2, b3)) { + if (malformedInputAction() != CodingErrorAction.REPLACE) + return -1; + da[dp++] = replacement().charAt(0); + sp -=3; + bb = getByteBuffer(bb, sa, sp); + sp += malformedN(bb, 3).length(); + } else { + da[dp++] = (char)((b1 << 12) ^ + (b2 << 6) ^ + (b3 ^ + (((byte) 0xE0 << 12) ^ + ((byte) 0x80 << 6) ^ + ((byte) 0x80 << 0)))); + } + continue; + } + if (malformedInputAction() != CodingErrorAction.REPLACE) + return -1; + if (sp < sl && isMalformed3_2(b1, sa[sp])) { + da[dp++] = replacement().charAt(0); + continue; + + } + da[dp++] = replacement().charAt(0); + return dp; + } else { + if (malformedInputAction() != CodingErrorAction.REPLACE) + return -1; + da[dp++] = replacement().charAt(0); + } + } + return dp; + } + } + + private static class Encoder extends CharsetEncoder + implements ArrayEncoder { + + private Encoder(Charset cs) { + super(cs, 1.1f, 3.0f); + } + + public boolean canEncode(char c) { + return !Character.isSurrogate(c); + } + + public boolean isLegalReplacement(byte[] repl) { + return ((repl.length == 1 && repl[0] >= 0) || + super.isLegalReplacement(repl)); + } + + private static CoderResult overflow(CharBuffer src, int sp, + ByteBuffer dst, int dp) { + updatePositions(src, sp, dst, dp); + return CoderResult.OVERFLOW; + } + + private static CoderResult overflow(CharBuffer src, int mark) { + src.position(mark); + return CoderResult.OVERFLOW; + } + + private static void to3Bytes(byte[] da, int dp, char c) { + da[dp] = (byte)(0xe0 | ((c >> 12))); + da[dp + 1] = (byte)(0x80 | ((c >> 6) & 0x3f)); + da[dp + 2] = (byte)(0x80 | (c & 0x3f)); + } + + private static void to3Bytes(ByteBuffer dst, char c) { + dst.put((byte)(0xe0 | ((c >> 12)))); + dst.put((byte)(0x80 | ((c >> 6) & 0x3f))); + dst.put((byte)(0x80 | (c & 0x3f))); + } + + private Surrogate.Parser sgp; + private char[] c2; + private CoderResult encodeArrayLoop(CharBuffer src, + ByteBuffer dst) + { + char[] sa = src.array(); + int sp = src.arrayOffset() + src.position(); + int sl = src.arrayOffset() + src.limit(); + + byte[] da = dst.array(); + int dp = dst.arrayOffset() + dst.position(); + int dl = dst.arrayOffset() + dst.limit(); + int dlASCII = dp + Math.min(sl - sp, dl - dp); + + // ASCII only loop + while (dp < dlASCII && sa[sp] < '\u0080') + da[dp++] = (byte) sa[sp++]; + while (sp < sl) { + char c = sa[sp]; + if (c < 0x80) { + // Have at most seven bits + if (dp >= dl) + return overflow(src, sp, dst, dp); + da[dp++] = (byte)c; + } else if (c < 0x800) { + // 2 bytes, 11 bits + if (dl - dp < 2) + return overflow(src, sp, dst, dp); + da[dp++] = (byte)(0xc0 | (c >> 6)); + da[dp++] = (byte)(0x80 | (c & 0x3f)); + } else if (Character.isSurrogate(c)) { + // Have a surrogate pair + if (sgp == null) + sgp = new Surrogate.Parser(); + int uc = sgp.parse(c, sa, sp, sl); + if (uc < 0) { + updatePositions(src, sp, dst, dp); + return sgp.error(); + } + if (dl - dp < 6) + return overflow(src, sp, dst, dp); + to3Bytes(da, dp, Character.highSurrogate(uc)); + dp += 3; + to3Bytes(da, dp, Character.lowSurrogate(uc)); + dp += 3; + sp++; // 2 chars + } else { + // 3 bytes, 16 bits + if (dl - dp < 3) + return overflow(src, sp, dst, dp); + to3Bytes(da, dp, c); + dp += 3; + } + sp++; + } + updatePositions(src, sp, dst, dp); + return CoderResult.UNDERFLOW; + } + + private CoderResult encodeBufferLoop(CharBuffer src, + ByteBuffer dst) + { + int mark = src.position(); + while (src.hasRemaining()) { + char c = src.get(); + if (c < 0x80) { + // Have at most seven bits + if (!dst.hasRemaining()) + return overflow(src, mark); + dst.put((byte)c); + } else if (c < 0x800) { + // 2 bytes, 11 bits + if (dst.remaining() < 2) + return overflow(src, mark); + dst.put((byte)(0xc0 | (c >> 6))); + dst.put((byte)(0x80 | (c & 0x3f))); + } else if (Character.isSurrogate(c)) { + // Have a surrogate pair + if (sgp == null) + sgp = new Surrogate.Parser(); + int uc = sgp.parse(c, src); + if (uc < 0) { + src.position(mark); + return sgp.error(); + } + if (dst.remaining() < 6) + return overflow(src, mark); + to3Bytes(dst, Character.highSurrogate(uc)); + to3Bytes(dst, Character.lowSurrogate(uc)); + mark++; // 2 chars + } else { + // 3 bytes, 16 bits + if (dst.remaining() < 3) + return overflow(src, mark); + to3Bytes(dst, c); + } + mark++; + } + src.position(mark); + return CoderResult.UNDERFLOW; + } + + protected final CoderResult encodeLoop(CharBuffer src, + ByteBuffer dst) + { + if (src.hasArray() && dst.hasArray()) + return encodeArrayLoop(src, dst); + else + return encodeBufferLoop(src, dst); + } + + // returns -1 if there is malformed char(s) and the + // "action" for malformed input is not REPLACE. + public int encode(char[] sa, int sp, int len, byte[] da) { + int sl = sp + len; + int dp = 0; + int dlASCII = dp + Math.min(len, da.length); + + // ASCII only optimized loop + while (dp < dlASCII && sa[sp] < '\u0080') + da[dp++] = (byte) sa[sp++]; + + while (sp < sl) { + char c = sa[sp++]; + if (c < 0x80) { + // Have at most seven bits + da[dp++] = (byte)c; + } else if (c < 0x800) { + // 2 bytes, 11 bits + da[dp++] = (byte)(0xc0 | (c >> 6)); + da[dp++] = (byte)(0x80 | (c & 0x3f)); + } else if (Character.isSurrogate(c)) { + if (sgp == null) + sgp = new Surrogate.Parser(); + int uc = sgp.parse(c, sa, sp - 1, sl); + if (uc < 0) { + if (malformedInputAction() != CodingErrorAction.REPLACE) + return -1; + da[dp++] = replacement()[0]; + } else { + to3Bytes(da, dp, Character.highSurrogate(uc)); + dp += 3; + to3Bytes(da, dp, Character.lowSurrogate(uc)); + dp += 3; + sp++; // 2 chars + } + } else { + // 3 bytes, 16 bits + to3Bytes(da, dp, c); + dp += 3; + } + } + return dp; + } + } +}
--- a/src/share/classes/sun/nio/cs/UTF_8.java Sat Nov 05 00:00:43 2011 -0700 +++ b/src/share/classes/sun/nio/cs/UTF_8.java Mon Nov 07 13:46:02 2011 -0800 @@ -72,8 +72,8 @@ return new Encoder(this); } - static final void updatePositions(Buffer src, int sp, - Buffer dst, int dp) { + private static final void updatePositions(Buffer src, int sp, + Buffer dst, int dp) { src.position(sp - src.arrayOffset()); dst.position(dp - dst.arrayOffset()); } @@ -88,11 +88,6 @@ return (b & 0xc0) != 0x80; } - // [C2..DF] [80..BF] - private static boolean isMalformed2(int b1, int b2) { - return (b1 & 0x1e) == 0x0 || (b2 & 0xc0) != 0x80; - } - // [E0] [A0..BF] [80..BF] // [E1..EF] [80..BF] [80..BF] private static boolean isMalformed3(int b1, int b2, int b3) { @@ -100,6 +95,12 @@ (b2 & 0xc0) != 0x80 || (b3 & 0xc0) != 0x80; } + // only used when there is only one byte left in src buffer + private static boolean isMalformed3_2(int b1, int b2) { + return (b1 == (byte)0xe0 && (b2 & 0xe0) == 0x80) || + (b2 & 0xc0) != 0x80; + } + // [F0] [90..BF] [80..BF] [80..BF] // [F1..F3] [80..BF] [80..BF] [80..BF] // [F4] [80..8F] [80..BF] [80..BF] @@ -110,6 +111,16 @@ (b4 & 0xc0) != 0x80; } + // only used when there is less than 4 bytes left in src buffer + private static boolean isMalformed4_2(int b1, int b2) { + return (b1 == 0xf0 && b2 == 0x90) || + (b2 & 0xc0) != 0x80; + } + + private static boolean isMalformed4_3(int b3) { + return (b3 & 0xc0) != 0x80; + } + private static CoderResult lookupN(ByteBuffer src, int n) { for (int i = 1; i < n; i++) { @@ -122,28 +133,14 @@ private static CoderResult malformedN(ByteBuffer src, int nb) { switch (nb) { case 1: - int b1 = src.get(); - if ((b1 >> 2) == -2) { - // 5 bytes 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx - if (src.remaining() < 4) - return CoderResult.UNDERFLOW; - return lookupN(src, 5); - } - if ((b1 >> 1) == -2) { - // 6 bytes 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx - if (src.remaining() < 5) - return CoderResult.UNDERFLOW; - return lookupN(src, 6); - } - return CoderResult.malformedForLength(1); case 2: // always 1 return CoderResult.malformedForLength(1); case 3: - b1 = src.get(); + int b1 = src.get(); int b2 = src.get(); // no need to lookup b3 return CoderResult.malformedForLength( ((b1 == (byte)0xe0 && (b2 & 0xe0) == 0x80) || - isNotContinuation(b2))?1:2); + isNotContinuation(b2)) ? 1 : 2); case 4: // we don't care the speed here b1 = src.get() & 0xff; b2 = src.get() & 0xff; @@ -171,6 +168,7 @@ return cr; } + private static CoderResult malformed(ByteBuffer src, int mark, int nb) { @@ -180,18 +178,36 @@ return cr; } + private static CoderResult malformedForLength(ByteBuffer src, + int sp, + CharBuffer dst, + int dp, + int malformedNB) + { + updatePositions(src, sp, dst, dp); + return CoderResult.malformedForLength(malformedNB); + } + + private static CoderResult malformedForLength(ByteBuffer src, + int mark, + int malformedNB) + { + src.position(mark); + return CoderResult.malformedForLength(malformedNB); + } + + private static CoderResult xflow(Buffer src, int sp, int sl, Buffer dst, int dp, int nb) { updatePositions(src, sp, dst, dp); return (nb == 0 || sl - sp < nb) - ?CoderResult.UNDERFLOW:CoderResult.OVERFLOW; + ? CoderResult.UNDERFLOW : CoderResult.OVERFLOW; } private static CoderResult xflow(Buffer src, int mark, int nb) { - CoderResult cr = (nb == 0 || src.remaining() < (nb - 1)) - ?CoderResult.UNDERFLOW:CoderResult.OVERFLOW; src.position(mark); - return cr; + return (nb == 0 || src.remaining() < nb) + ? CoderResult.UNDERFLOW : CoderResult.OVERFLOW; } private CoderResult decodeArrayLoop(ByteBuffer src, @@ -210,7 +226,6 @@ // ASCII only loop while (dp < dlASCII && sa[sp] >= 0) da[dp++] = (char) sa[sp++]; - while (sp < sl) { int b1 = sa[sp]; if (b1 >= 0) { @@ -219,13 +234,20 @@ return xflow(src, sp, sl, dst, dp, 1); da[dp++] = (char) b1; sp++; - } else if ((b1 >> 5) == -2) { + } else if ((b1 >> 5) == -2 && (b1 & 0x1e) != 0) { // 2 bytes, 11 bits: 110xxxxx 10xxxxxx + // [C2..DF] [80..BF] if (sl - sp < 2 || dp >= dl) return xflow(src, sp, sl, dst, dp, 2); int b2 = sa[sp + 1]; - if (isMalformed2(b1, b2)) - return malformed(src, sp, dst, dp, 2); + // Now we check the first byte of 2-byte sequence as + // if ((b1 >> 5) == -2 && (b1 & 0x1e) != 0) + // no longer need to check b1 against c1 & c0 for + // malformed as we did in previous version + // (b1 & 0x1e) == 0x0 || (b2 & 0xc0) != 0x80; + // only need to check the second byte b2. + if (isNotContinuation(b2)) + return malformedForLength(src, sp, dst, dp, 1); da[dp++] = (char) (((b1 << 6) ^ b2) ^ (((byte) 0xC0 << 6) ^ @@ -233,24 +255,37 @@ sp += 2; } else if ((b1 >> 4) == -2) { // 3 bytes, 16 bits: 1110xxxx 10xxxxxx 10xxxxxx - if (sl - sp < 3 || dp >= dl) + int srcRemaining = sl - sp; + if (srcRemaining < 3 || dp >= dl) { + if (srcRemaining > 1 && isMalformed3_2(b1, sa[sp + 1])) + return malformedForLength(src, sp, dst, dp, 1); return xflow(src, sp, sl, dst, dp, 3); + } int b2 = sa[sp + 1]; int b3 = sa[sp + 2]; if (isMalformed3(b1, b2, b3)) return malformed(src, sp, dst, dp, 3); - da[dp++] = (char) + char c = (char) ((b1 << 12) ^ (b2 << 6) ^ (b3 ^ (((byte) 0xE0 << 12) ^ ((byte) 0x80 << 6) ^ ((byte) 0x80 << 0)))); + if (Character.isSurrogate(c)) + return malformedForLength(src, sp, dst, dp, 3); + da[dp++] = c; sp += 3; } else if ((b1 >> 3) == -2) { // 4 bytes, 21 bits: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx - if (sl - sp < 4 || dl - dp < 2) + int srcRemaining = sl - sp; + if (srcRemaining < 4 || dl - dp < 2) { + if (srcRemaining > 1 && isMalformed4_2(b1, sa[sp + 1])) + return malformedForLength(src, sp, dst, dp, 1); + if (srcRemaining > 2 && isMalformed4_3(sa[sp + 2])) + return malformedForLength(src, sp, dst, dp, 2); return xflow(src, sp, sl, dst, dp, 4); + } int b2 = sa[sp + 1]; int b3 = sa[sp + 2]; int b4 = sa[sp + 3]; @@ -289,38 +324,51 @@ return xflow(src, mark, 1); // overflow dst.put((char) b1); mark++; - } else if ((b1 >> 5) == -2) { + } else if ((b1 >> 5) == -2 && (b1 & 0x1e) != 0) { // 2 bytes, 11 bits: 110xxxxx 10xxxxxx if (limit - mark < 2|| dst.remaining() < 1) return xflow(src, mark, 2); int b2 = src.get(); - if (isMalformed2(b1, b2)) - return malformed(src, mark, 2); - dst.put((char) (((b1 << 6) ^ b2) + if (isNotContinuation(b2)) + return malformedForLength(src, mark, 1); + dst.put((char) (((b1 << 6) ^ b2) ^ (((byte) 0xC0 << 6) ^ ((byte) 0x80 << 0)))); mark += 2; } else if ((b1 >> 4) == -2) { // 3 bytes, 16 bits: 1110xxxx 10xxxxxx 10xxxxxx - if (limit - mark < 3 || dst.remaining() < 1) + int srcRemaining = limit - mark; + if (srcRemaining < 3 || dst.remaining() < 1) { + if (srcRemaining > 1 && isMalformed3_2(b1, src.get())) + return malformedForLength(src, mark, 1); return xflow(src, mark, 3); + } int b2 = src.get(); int b3 = src.get(); if (isMalformed3(b1, b2, b3)) return malformed(src, mark, 3); - dst.put((char) - ((b1 << 12) ^ - (b2 << 6) ^ - (b3 ^ - (((byte) 0xE0 << 12) ^ - ((byte) 0x80 << 6) ^ - ((byte) 0x80 << 0))))); + char c = (char) + ((b1 << 12) ^ + (b2 << 6) ^ + (b3 ^ + (((byte) 0xE0 << 12) ^ + ((byte) 0x80 << 6) ^ + ((byte) 0x80 << 0)))); + if (Character.isSurrogate(c)) + return malformedForLength(src, mark, 3); + dst.put(c); mark += 3; } else if ((b1 >> 3) == -2) { // 4 bytes, 21 bits: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx - if (limit - mark < 4 || dst.remaining() < 2) + int srcRemaining = limit - mark; + if (srcRemaining < 4 || dst.remaining() < 2) { + if (srcRemaining > 1 && isMalformed4_2(b1, src.get())) + return malformedForLength(src, mark, 1); + if (srcRemaining > 2 && isMalformed4_3(src.get())) + return malformedForLength(src, mark, 2); return xflow(src, mark, 4); + } int b2 = src.get(); int b3 = src.get(); int b4 = src.get(); @@ -364,7 +412,7 @@ return bb; } - // returns -1 if there is malformed byte(s) and the + // returns -1 if there is/are malformed byte(s) and the // "action" for malformed input is not REPLACE. public int decode(byte[] sa, int sp, int len, char[] da) { final int sl = sp + len; @@ -381,11 +429,11 @@ if (b1 >= 0) { // 1 byte, 7 bits: 0xxxxxxx da[dp++] = (char) b1; - } else if ((b1 >> 5) == -2) { + } else if ((b1 >> 5) == -2 && (b1 & 0x1e) != 0) { // 2 bytes, 11 bits: 110xxxxx 10xxxxxx if (sp < sl) { int b2 = sa[sp++]; - if (isMalformed2(b1, b2)) { + if (isNotContinuation(b2)) { if (malformedInputAction() != CodingErrorAction.REPLACE) return -1; da[dp++] = replacement().charAt(0); @@ -410,21 +458,33 @@ if (malformedInputAction() != CodingErrorAction.REPLACE) return -1; da[dp++] = replacement().charAt(0); - sp -=3; + sp -= 3; bb = getByteBuffer(bb, sa, sp); sp += malformedN(bb, 3).length(); } else { - da[dp++] = (char)((b1 << 12) ^ + char c = (char)((b1 << 12) ^ (b2 << 6) ^ (b3 ^ (((byte) 0xE0 << 12) ^ ((byte) 0x80 << 6) ^ ((byte) 0x80 << 0)))); + if (Character.isSurrogate(c)) { + if (malformedInputAction() != CodingErrorAction.REPLACE) + return -1; + da[dp++] = replacement().charAt(0); + } else { + da[dp++] = c; + } } continue; } if (malformedInputAction() != CodingErrorAction.REPLACE) return -1; + if (sp < sl && isMalformed3_2(b1, sa[sp])) { + da[dp++] = replacement().charAt(0); + continue; + + } da[dp++] = replacement().charAt(0); return dp; } else if ((b1 >> 3) == -2) { @@ -458,28 +518,29 @@ } if (malformedInputAction() != CodingErrorAction.REPLACE) return -1; + + if (sp < sl && isMalformed4_2(b1, sa[sp])) { + da[dp++] = replacement().charAt(0); + continue; + } + sp++; + if (sp < sl && isMalformed4_3(sa[sp])) { + da[dp++] = replacement().charAt(0); + continue; + } da[dp++] = replacement().charAt(0); return dp; } else { if (malformedInputAction() != CodingErrorAction.REPLACE) return -1; da[dp++] = replacement().charAt(0); - sp--; - bb = getByteBuffer(bb, sa, sp); - CoderResult cr = malformedN(bb, 1); - if (!cr.isError()) { - // leading byte for 5 or 6-byte, but don't have enough - // bytes in buffer to check. Consumed rest as malformed. - return dp; - } - sp += cr.length(); } } return dp; } } - private static class Encoder extends CharsetEncoder + private static final class Encoder extends CharsetEncoder implements ArrayEncoder { private Encoder(Charset cs) {
--- a/src/share/classes/sun/nio/cs/standard-charsets Sat Nov 05 00:00:43 2011 -0700 +++ b/src/share/classes/sun/nio/cs/standard-charsets Mon Nov 07 13:46:02 2011 -0800 @@ -63,6 +63,10 @@ alias UTF8 # JDK historical alias unicode-1-1-utf-8 +charset CESU-8 CESU_8 + alias CESU8 + alias csCESU-8 + charset UTF-16 UTF_16 alias UTF_16 # JDK historical alias utf16
--- a/test/java/nio/charset/coders/Errors.java Sat Nov 05 00:00:43 2011 -0700 +++ b/test/java/nio/charset/coders/Errors.java Mon Nov 07 13:46:02 2011 -0800 @@ -23,7 +23,7 @@ /* @test * @summary Check that error cases are replaced correctly in String/ISR/OSW - * @bug 4457851 + * @bug 4457851 7096080 * * @build Errors Util * @run main Errors @@ -193,11 +193,9 @@ t.test("\uFFFF", new byte[] { (byte)0xEF, (byte)0xBF, (byte)0xBF }); t.test(new byte[] { X, (byte)0x7f, Y }, "x\u007Fy"); t.test(new byte[] { X, (byte)0x80, Y }, "x\uFFFDy"); - t.test(new byte[] { (byte)0xf0, (byte)0xf0 }, "\uFFFD"); } public static void main(String[] args) throws Exception { - test_US_ASCII(new TestString("US-ASCII")); test_US_ASCII(new TestStream("US-ASCII"));
--- a/test/sun/nio/cs/TestStringCoding.java Sat Nov 05 00:00:43 2011 -0700 +++ b/test/sun/nio/cs/TestStringCoding.java Mon Nov 07 13:46:02 2011 -0800 @@ -24,7 +24,7 @@ */ /* @test - @bug 6636323 6636319 7040220 + @bug 6636323 6636319 7040220 7096080 @summary Test if StringCoding and NIO result have the same de/encoding result * @run main/othervm/timeout=2000 TestStringCoding */ @@ -111,7 +111,8 @@ //encode unmappable surrogates if (enc instanceof sun.nio.cs.ArrayEncoder && cs.contains(Charset.forName("ASCII"))) { - if (cs.name().equals("UTF-8")) // utf8 handles surrogates + if (cs.name().equals("UTF-8") || // utf8 handles surrogates + cs.name().equals("CESU-8")) // utf8 handles surrogates return; enc.replaceWith(new byte[] { (byte)'A'}); sun.nio.cs.ArrayEncoder cae = (sun.nio.cs.ArrayEncoder)enc; @@ -136,7 +137,6 @@ cs.name()))) throw new RuntimeException("encode3(surrogates) failed -> " + cs.name()); - ba = new byte[str.length() - 1]; n = cae.encode(str.toCharArray(), 0, str.length(), ba); if (n != 7 || !"abABABc".equals(new String(ba, 0, n,
--- a/test/sun/nio/cs/TestStringCodingUTF8.java Sat Nov 05 00:00:43 2011 -0700 +++ b/test/sun/nio/cs/TestStringCodingUTF8.java Mon Nov 07 13:46:02 2011 -0800 @@ -33,14 +33,16 @@ public class TestStringCodingUTF8 { public static void main(String[] args) throws Throwable { - test(); + test("UTF-8"); + test("CESU-8"); // security manager on System.setSecurityManager(new PermissiveSecurityManger()); - test(); + test("UTF-8"); + test("CESU-8"); } - static void test() throws Throwable { - Charset cs = Charset.forName("UTF-8"); + static void test(String csn) throws Throwable { + Charset cs = Charset.forName(csn); char[] bmp = new char[0x10000]; for (int i = 0; i < 0x10000; i++) { bmp[i] = (char)i;
--- a/test/sun/nio/cs/TestUTF8.java Sat Nov 05 00:00:43 2011 -0700 +++ b/test/sun/nio/cs/TestUTF8.java Mon Nov 07 13:46:02 2011 -0800 @@ -23,7 +23,7 @@ /* * @test - * @bug 4486841 7040220 + * @bug 4486841 7040220 7096080 * @summary Test UTF-8 charset */ @@ -156,15 +156,22 @@ return 3; } + static int to4ByteUTF8(int uc, byte[] bb, int pos) { + bb[pos++] = (byte)(0xf0 | ((uc >> 18))); + bb[pos++] = (byte)(0x80 | ((uc >> 12) & 0x3f)); + bb[pos++] = (byte)(0x80 | ((uc >> 6) & 0x3f)); + bb[pos++] = (byte)(0x80 | (uc & 0x3f)); + return 4; + } + static void checkRoundtrip(String csn) throws Exception { System.out.printf(" Check roundtrip <%s>...", csn); char[] cc = getUTFChars(); byte[] bb = encode(cc, csn, false); char[] ccO = decode(bb, csn, false); - if (!Arrays.equals(cc, ccO)) { + if (!Arrays.equals(cc, ccO)) System.out.printf(" non-direct failed"); - } bb = encode(cc, csn, true); ccO = decode(bb, csn, true); if (!Arrays.equals(cc, ccO)) { @@ -180,6 +187,40 @@ System.out.println(); } + static void check4ByteSurrs(String csn) throws Exception { + System.out.printf(" Check 4-byte Surrogates <%s>...%n", csn); + byte[] bb = new byte[(0x110000 - 0x10000) * 4]; + char[] cc = new char[(0x110000 - 0x10000) * 2]; + int bpos = 0; + int cpos = 0; + for (int i = 0x10000; i < 0x110000; i++) { + Character.toChars(i, cc, cpos); + bpos += to4ByteUTF8(i, bb, bpos); + cpos += 2; + } + checkSurrs(csn, bb, cc); + } + + + static void checkSurrs(String csn, byte[] bb, char[] cc) + throws Exception + { + char[] ccO = decode(bb, csn, false); + if (!Arrays.equals(cc, ccO)) { + System.out.printf(" decoding failed%n"); + } + ccO = decode(bb, csn, true); + if (!Arrays.equals(cc, ccO)) { + System.out.printf(" decoding(direct) failed%n"); + } + if (!Arrays.equals(cc, new String(bb, csn).toCharArray())) { + System.out.printf(" String.toCharArray() failed"); + } + if (!Arrays.equals(bb, new String(cc).getBytes(csn))) { + System.out.printf(" String.getBytes() failed"); + } + } + static void check6ByteSurrs(String csn) throws Exception { System.out.printf(" Check 6-byte Surrogates <%s>...%n", csn); byte[] bb = new byte[(0x110000 - 0x10000) * 6]; @@ -192,22 +233,9 @@ bpos += to3ByteUTF8(cc[cpos + 1], bb, bpos); cpos += 2; } + checkSurrs(csn, bb, cc); + } - char[] ccO = decode(bb, csn, false); - if (!Arrays.equals(cc, ccO)) { - System.out.printf(" decoding failed%n"); - } - ccO = decode(bb, csn, true); - if (!Arrays.equals(cc, ccO)) { - System.out.printf(" decoding(direct) failed%n"); - } - // new String(bb, csn).getBytes(csn) will not return - // the 6 bytes surrogates as in bb, so only test - // toCharArray() here. - if (!Arrays.equals(cc, new String(bb, csn).toCharArray())) { - System.out.printf(" String.toCharArray() failed"); - } - } static void compare(String csn1, String csn2) throws Exception { System.out.printf(" Diff <%s> <%s>...%n", csn1, csn2); @@ -266,6 +294,10 @@ {1, (byte)0xFF, (byte)0xFF, (byte)0xFF }, // all ones {1, (byte)0xE0, (byte)0xC0, (byte)0x80 }, // invalid second byte {1, (byte)0xE0, (byte)0x80, (byte)0xC0 }, // invalid first byte + {1, (byte)0xE0, (byte)0x41,}, // invalid second byte & 2 bytes + {3, (byte)0xED, (byte)0xAE, (byte)0x80 }, // 3 bytes surrogate + {3, (byte)0xED, (byte)0xB0, (byte)0x80 }, // 3 bytes surrogate + // Four-byte sequences {1, (byte)0xF0, (byte)0x80, (byte)0x80, (byte)0x80 }, // U+0000 zero-padded @@ -276,8 +308,13 @@ {1, (byte)0xFF, (byte)0xFF, (byte)0xFF, (byte)0xFF }, // all ones {1, (byte)0xF0, (byte)0x80, (byte)0x80, (byte)0x80}, // invalid second byte {1, (byte)0xF0, (byte)0xC0, (byte)0x80, (byte)0x80 }, // invalid second byte + {1, (byte)0xF0, (byte)41 }, // invalid second byte + // & only 2 bytes + {2, (byte)0xF0, (byte)0x90, (byte)0xC0, (byte)0x80 }, // invalid third byte - {3, (byte)0xF0, (byte)0x90, (byte)0x80, (byte)0xC0 }, // invalid third byte + {3, (byte)0xF0, (byte)0x90, (byte)0x80, (byte)0xC0 }, // invalid forth byte + {2, (byte)0xF0, (byte)0x90, (byte)0x41 }, // invalid third byte + // & 3 bytes input {1, (byte)0xF1, (byte)0xC0, (byte)0x80, (byte)0x80 }, // invalid second byte {2, (byte)0xF1, (byte)0x80, (byte)0xC0, (byte)0x80 }, // invalid third byte @@ -287,30 +324,113 @@ {1, (byte)0xF5, (byte)0x80, (byte)0x80, (byte)0xC0 }, // out-range 4-byte // Five-byte sequences - {5, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x80}, // invalid first byte - {5, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x80 }, // U+0000 zero-padded - {5, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x81, (byte)0xBF }, // U+007F zero-padded - {5, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x9F, (byte)0xBF }, // U+07FF zero-padded - {5, (byte)0xF8, (byte)0x80, (byte)0x8F, (byte)0xBF, (byte)0xBF }, // U+FFFF zero-padded + {1, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x80}, // invalid first byte + {1, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x80 }, // U+0000 zero-padded + {1, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x81, (byte)0xBF }, // U+007F zero-padded + {1, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x9F, (byte)0xBF }, // U+07FF zero-padded + {1, (byte)0xF8, (byte)0x80, (byte)0x8F, (byte)0xBF, (byte)0xBF }, // U+FFFF zero-padded {1, (byte)0xF8, (byte)0xC0, (byte)0x80, (byte)0x80, (byte)0x80}, - {2, (byte)0xF8, (byte)0x80, (byte)0xC0, (byte)0x80, (byte)0x80 }, - {3, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0xC1, (byte)0xBF }, - {4, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x9F, (byte)0xC0 }, + {1, (byte)0xF8, (byte)0x80, (byte)0xC0, (byte)0x80, (byte)0x80 }, + {1, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0xC1, (byte)0xBF }, + {1, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x9F, (byte)0xC0 }, // Six-byte sequences - {6, (byte)0xFC, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x80 }, // U+0000 zero-padded - {6, (byte)0xFC, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x81, (byte)0xBF }, // U+007F zero-padded - {6, (byte)0xFC, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x9F, (byte)0xBF }, // U+07FF zero-padded - {6, (byte)0xFC, (byte)0x80, (byte)0x80, (byte)0x8F, (byte)0xBF, (byte)0xBF }, // U+FFFF zero-padded + {1, (byte)0xFC, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x80 }, // U+0000 zero-padded + {1, (byte)0xFC, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x81, (byte)0xBF }, // U+007F zero-padded + {1, (byte)0xFC, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x9F, (byte)0xBF }, // U+07FF zero-padded + {1, (byte)0xFC, (byte)0x80, (byte)0x80, (byte)0x8F, (byte)0xBF, (byte)0xBF }, // U+FFFF zero-padded {1, (byte)0xF8, (byte)0xC0, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x80 }, - {2, (byte)0xF8, (byte)0x80, (byte)0xC0, (byte)0x80, (byte)0x80, (byte)0x80 }, - {3, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0xC1, (byte)0xBF, (byte)0x80 }, - {4, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x9F, (byte)0xC0, (byte)0x80 }, - {5, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x9F, (byte)0x80, (byte)0xC0 }, + {1, (byte)0xF8, (byte)0x80, (byte)0xC0, (byte)0x80, (byte)0x80, (byte)0x80 }, + {1, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0xC1, (byte)0xBF, (byte)0x80 }, + {1, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x9F, (byte)0xC0, (byte)0x80 }, + {1, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x9F, (byte)0x80, (byte)0xC0 }, }; - static void checkMalformed(String csn) throws Exception { + // The first byte is the length of malformed bytes + static byte[][] malformed_cesu8 = { + // One-byte sequences: + {1, (byte)0xFF }, + {1, (byte)0xC0 }, + {1, (byte)0x80 }, + + {1, (byte)0xFF, (byte)0xFF}, // all ones + {1, (byte)0xA0, (byte)0x80}, // 101x first byte first nibble + + // Two-byte sequences: + {1, (byte)0xC0, (byte)0x80}, // invalid first byte + {1, (byte)0xC1, (byte)0xBF}, // invalid first byte + {1, (byte)0xC2, (byte)0x00}, // invalid second byte + {1, (byte)0xC2, (byte)0xC0}, // invalid second byte + {1, (byte)0xD0, (byte)0x00}, // invalid second byte + {1, (byte)0xD0, (byte)0xC0}, // invalid second byte + {1, (byte)0xDF, (byte)0x00}, // invalid second byte + {1, (byte)0xDF, (byte)0xC0}, // invalid second byte + + // Three-byte sequences + {1, (byte)0xE0, (byte)0x80, (byte)0x80}, // 111x first byte first nibble + {1, (byte)0xE0, (byte)0x80, (byte)0x80 }, // U+0000 zero-padded + {1, (byte)0xE0, (byte)0x81, (byte)0xBF }, // U+007F zero-padded + {1, (byte)0xE0, (byte)0x9F, (byte)0xBF }, // U+07FF zero-padded + + {1, (byte)0xE0, (byte)0xC0, (byte)0xBF }, // invalid second byte + {2, (byte)0xE0, (byte)0xA0, (byte)0x7F }, // invalid third byte + {2, (byte)0xE0, (byte)0xA0, (byte)0xC0 }, // invalid third byte + {1, (byte)0xFF, (byte)0xFF, (byte)0xFF }, // all ones + {1, (byte)0xE0, (byte)0xC0, (byte)0x80 }, // invalid second byte + {1, (byte)0xE0, (byte)0x80, (byte)0xC0 }, // invalid first byte + {1, (byte)0xE0, (byte)0x41,}, // invalid second byte & 2 bytes + + // CESU-8 does not have 4, 5, 6 bytes sequenc + // Four-byte sequences + {1, (byte)0xF0, (byte)0x80, (byte)0x80, (byte)0x80 }, // U+0000 zero-padded + {1, (byte)0xF0, (byte)0x80, (byte)0x81, (byte)0xBF }, // U+007F zero-padded + {1, (byte)0xF0, (byte)0x80, (byte)0x9F, (byte)0xBF }, // U+007F zero-padded + {1, (byte)0xF0, (byte)0x8F, (byte)0xBF, (byte)0xBF }, // U+07FF zero-padded + + {1, (byte)0xFF, (byte)0xFF, (byte)0xFF, (byte)0xFF }, // all ones + {1, (byte)0xF0, (byte)0x80, (byte)0x80, (byte)0x80}, // invalid second byte + {1, (byte)0xF0, (byte)0xC0, (byte)0x80, (byte)0x80 }, // invalid second byte + {1, (byte)0xF0, (byte)41 }, // invalid second byte + // & only 2 bytes + {1, (byte)0xF0, (byte)0x90, (byte)0xC0, (byte)0x80 }, // invalid third byte + {1, (byte)0xF0, (byte)0x90, (byte)0x80, (byte)0xC0 }, // invalid forth byte + {1, (byte)0xF0, (byte)0x90, (byte)0x41 }, // invalid third byte + // & 3 bytes input + + {1, (byte)0xF1, (byte)0xC0, (byte)0x80, (byte)0x80 }, // invalid second byte + {1, (byte)0xF1, (byte)0x80, (byte)0xC0, (byte)0x80 }, // invalid third byte + {1, (byte)0xF1, (byte)0x80, (byte)0x80, (byte)0xC0 }, // invalid forth byte + {1, (byte)0xF4, (byte)0x90, (byte)0x80, (byte)0xC0 }, // out-range 4-byte + {1, (byte)0xF4, (byte)0xC0, (byte)0x80, (byte)0xC0 }, // out-range 4-byte + {1, (byte)0xF5, (byte)0x80, (byte)0x80, (byte)0xC0 }, // out-range 4-byte + + // Five-byte sequences + {1, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x80}, // invalid first byte + {1, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x80 }, // U+0000 zero-padded + {1, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x81, (byte)0xBF }, // U+007F zero-padded + {1, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x9F, (byte)0xBF }, // U+07FF zero-padded + {1, (byte)0xF8, (byte)0x80, (byte)0x8F, (byte)0xBF, (byte)0xBF }, // U+FFFF zero-padded + + {1, (byte)0xF8, (byte)0xC0, (byte)0x80, (byte)0x80, (byte)0x80}, + {1, (byte)0xF8, (byte)0x80, (byte)0xC0, (byte)0x80, (byte)0x80 }, + {1, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0xC1, (byte)0xBF }, + {1, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x9F, (byte)0xC0 }, + + // Six-byte sequences + {1, (byte)0xFC, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x80 }, // U+0000 zero-padded + {1, (byte)0xFC, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x81, (byte)0xBF }, // U+007F zero-padded + {1, (byte)0xFC, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x9F, (byte)0xBF }, // U+07FF zero-padded + {1, (byte)0xFC, (byte)0x80, (byte)0x80, (byte)0x8F, (byte)0xBF, (byte)0xBF }, // U+FFFF zero-padded + {1, (byte)0xF8, (byte)0xC0, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x80 }, + {1, (byte)0xF8, (byte)0x80, (byte)0xC0, (byte)0x80, (byte)0x80, (byte)0x80 }, + {1, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0xC1, (byte)0xBF, (byte)0x80 }, + {1, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x9F, (byte)0xC0, (byte)0x80 }, + {1, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x9F, (byte)0x80, (byte)0xC0 }, + }; + + + static void checkMalformed(String csn, byte[][] malformed) throws Exception { boolean failed = false; System.out.printf(" Check malformed <%s>...%n", csn); Charset cs = Charset.forName(csn); @@ -430,9 +550,12 @@ public static void main(String[] args) throws Exception { checkRoundtrip("UTF-8"); - check6ByteSurrs("UTF-8"); - //compare("UTF-8", "UTF-8-OLD"); - checkMalformed("UTF-8"); + check4ByteSurrs("UTF-8"); + checkMalformed("UTF-8", malformed); checkUnderOverflow("UTF-8"); + + checkRoundtrip("CESU-8"); + check6ByteSurrs("CESU-8"); + checkMalformed("CESU-8", malformed_cesu8); } }