Mercurial > hg > openjdk > jdk8u > jdk
changeset 13202:b3de2b1e82fb jdk8u162-b35
8168628: (fc) SIGBUS when extending file size to map it
8171452: (ch) linux io_util_md: Operation not supported exception after 8168628
Reviewed-by: alanb, bpb
| author | dbuck |
|---|---|
| date | Tue, 06 Feb 2018 03:50:10 -0500 |
| parents | 46451609c8fc |
| children | 367961fe749f |
| files | make/mapfiles/libnio/mapfile-linux make/mapfiles/libnio/mapfile-macosx make/mapfiles/libnio/mapfile-solaris src/share/classes/sun/nio/ch/FileChannelImpl.java src/share/classes/sun/nio/ch/FileDispatcher.java src/solaris/classes/sun/nio/ch/FileDispatcherImpl.java src/solaris/native/java/io/io_util_md.c src/solaris/native/sun/nio/ch/FileDispatcherImpl.c src/windows/classes/sun/nio/ch/FileDispatcherImpl.java test/java/nio/channels/FileChannel/FileExtensionAndMap.java |
| diffstat | 10 files changed, 329 insertions(+), 53 deletions(-) [+] |
line wrap: on
line diff
--- a/make/mapfiles/libnio/mapfile-linux Wed Feb 07 09:44:50 2018 -0800 +++ b/make/mapfiles/libnio/mapfile-linux Tue Feb 06 03:50:10 2018 -0500 @@ -1,5 +1,5 @@ # -# Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2001, 2018, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # This code is free software; you can redistribute it and/or modify it @@ -59,6 +59,7 @@ Java_sun_nio_ch_FileChannelImpl_position0; Java_sun_nio_ch_FileChannelImpl_transferTo0; Java_sun_nio_ch_FileChannelImpl_unmap0; + Java_sun_nio_ch_FileDispatcherImpl_allocate0; Java_sun_nio_ch_FileDispatcherImpl_close0; Java_sun_nio_ch_FileDispatcherImpl_closeIntFD; Java_sun_nio_ch_FileDispatcherImpl_force0;
--- a/make/mapfiles/libnio/mapfile-macosx Wed Feb 07 09:44:50 2018 -0800 +++ b/make/mapfiles/libnio/mapfile-macosx Tue Feb 06 03:50:10 2018 -0500 @@ -1,5 +1,5 @@ # -# Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2001, 2018, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # This code is free software; you can redistribute it and/or modify it @@ -42,6 +42,7 @@ Java_sun_nio_ch_FileChannelImpl_position0; Java_sun_nio_ch_FileChannelImpl_transferTo0; Java_sun_nio_ch_FileChannelImpl_unmap0; + Java_sun_nio_ch_FileDispatcherImpl_allocate0; Java_sun_nio_ch_FileDispatcherImpl_close0; Java_sun_nio_ch_FileDispatcherImpl_closeIntFD; Java_sun_nio_ch_FileDispatcherImpl_force0;
--- a/make/mapfiles/libnio/mapfile-solaris Wed Feb 07 09:44:50 2018 -0800 +++ b/make/mapfiles/libnio/mapfile-solaris Tue Feb 06 03:50:10 2018 -0500 @@ -1,5 +1,5 @@ # -# Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2001, 2018, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # This code is free software; you can redistribute it and/or modify it @@ -47,6 +47,7 @@ Java_sun_nio_ch_FileChannelImpl_position0; Java_sun_nio_ch_FileChannelImpl_transferTo0; Java_sun_nio_ch_FileChannelImpl_unmap0; + Java_sun_nio_ch_FileDispatcherImpl_allocate0; Java_sun_nio_ch_FileDispatcherImpl_close0; Java_sun_nio_ch_FileDispatcherImpl_closeIntFD; Java_sun_nio_ch_FileDispatcherImpl_force0;
--- a/src/share/classes/sun/nio/ch/FileChannelImpl.java Wed Feb 07 09:44:50 2018 -0800 +++ b/src/share/classes/sun/nio/ch/FileChannelImpl.java Tue Feb 06 03:50:10 2018 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -889,57 +889,62 @@ if (!isOpen()) return null; - long filesize; - do { - filesize = nd.size(fd); - } while ((filesize == IOStatus.INTERRUPTED) && isOpen()); - if (!isOpen()) - return null; - - if (filesize < position + size) { // Extend file size - if (!writable) { - throw new IOException("Channel not open for writing " + - "- cannot extend file to required size"); - } - int rv; + long mapSize; + int pagePosition; + synchronized (positionLock) { + long filesize; do { - rv = nd.truncate(fd, position + size); - } while ((rv == IOStatus.INTERRUPTED) && isOpen()); + filesize = nd.size(fd); + } while ((filesize == IOStatus.INTERRUPTED) && isOpen()); if (!isOpen()) return null; - } - if (size == 0) { - addr = 0; - // a valid file descriptor is not required - FileDescriptor dummy = new FileDescriptor(); - if ((!writable) || (imode == MAP_RO)) - return Util.newMappedByteBufferR(0, 0, dummy, null); - else - return Util.newMappedByteBuffer(0, 0, dummy, null); - } + + if (filesize < position + size) { // Extend file size + if (!writable) { + throw new IOException("Channel not open for writing " + + "- cannot extend file to required size"); + } + int rv; + do { + rv = nd.allocate(fd, position + size); + } while ((rv == IOStatus.INTERRUPTED) && isOpen()); + if (!isOpen()) + return null; + } + + if (size == 0) { + addr = 0; + // a valid file descriptor is not required + FileDescriptor dummy = new FileDescriptor(); + if ((!writable) || (imode == MAP_RO)) + return Util.newMappedByteBufferR(0, 0, dummy, null); + else + return Util.newMappedByteBuffer(0, 0, dummy, null); + } - int pagePosition = (int)(position % allocationGranularity); - long mapPosition = position - pagePosition; - long mapSize = size + pagePosition; - try { - // If no exception was thrown from map0, the address is valid - addr = map0(imode, mapPosition, mapSize); - } catch (OutOfMemoryError x) { - // An OutOfMemoryError may indicate that we've exhausted memory - // so force gc and re-attempt map - System.gc(); + pagePosition = (int)(position % allocationGranularity); + long mapPosition = position - pagePosition; + mapSize = size + pagePosition; try { - Thread.sleep(100); - } catch (InterruptedException y) { - Thread.currentThread().interrupt(); - } - try { + // If map0 did not throw an exception, the address is valid addr = map0(imode, mapPosition, mapSize); - } catch (OutOfMemoryError y) { - // After a second OOME, fail - throw new IOException("Map failed", y); + } catch (OutOfMemoryError x) { + // An OutOfMemoryError may indicate that we've exhausted + // memory so force gc and re-attempt map + System.gc(); + try { + Thread.sleep(100); + } catch (InterruptedException y) { + Thread.currentThread().interrupt(); + } + try { + addr = map0(imode, mapPosition, mapSize); + } catch (OutOfMemoryError y) { + // After a second OOME, fail + throw new IOException("Map failed", y); + } } - } + } // synchronized // On Windows, and potentially other platforms, we need an open // file descriptor for some mapping operations.
--- a/src/share/classes/sun/nio/ch/FileDispatcher.java Wed Feb 07 09:44:50 2018 -0800 +++ b/src/share/classes/sun/nio/ch/FileDispatcher.java Tue Feb 06 03:50:10 2018 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2007, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -40,6 +40,8 @@ abstract int truncate(FileDescriptor fd, long size) throws IOException; + abstract int allocate(FileDescriptor fd, long size) throws IOException; + abstract long size(FileDescriptor fd) throws IOException; abstract int lock(FileDescriptor fd, boolean blocking, long pos, long size,
--- a/src/solaris/classes/sun/nio/ch/FileDispatcherImpl.java Wed Feb 07 09:44:50 2018 -0800 +++ b/src/solaris/classes/sun/nio/ch/FileDispatcherImpl.java Tue Feb 06 03:50:10 2018 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -80,6 +80,10 @@ return truncate0(fd, size); } + int allocate(FileDescriptor fd, long size) throws IOException { + return allocate0(fd, size); + } + long size(FileDescriptor fd) throws IOException { return size0(fd); } @@ -142,6 +146,9 @@ static native int truncate0(FileDescriptor fd, long size) throws IOException; + static native int allocate0(FileDescriptor fd, long size) + throws IOException; + static native long size0(FileDescriptor fd) throws IOException; static native int lock0(FileDescriptor fd, boolean blocking, long pos,
--- a/src/solaris/native/java/io/io_util_md.c Wed Feb 07 09:44:50 2018 -0800 +++ b/src/solaris/native/java/io/io_util_md.c Tue Feb 06 03:50:10 2018 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,6 +23,10 @@ * questions. */ +#if defined(__linux__) +#define _FILE_OFFSET_BITS 64 +#endif + #include "jni.h" #include "jni_util.h" #include "jvm.h" @@ -208,6 +212,25 @@ handleSetLength(FD fd, jlong length) { int result; +#if defined(__linux__) + /* + * On Linux, if the file size is being increased, then ftruncate64() + * will modify the metadata value of the size without actually allocating + * any blocks which can cause a SIGBUS error if the file is subsequently + * memory-mapped. + */ + struct stat64 sb; + + if (fstat64(fd, &sb) == 0 && length > sb.st_blocks*512) { + RESTARTABLE(posix_fallocate(fd, 0, length), result); + // Return on success or if errno is neither EOPNOTSUPP nor ENOSYS + if (result == 0) { + return 0; + } else if (errno != EOPNOTSUPP && errno != ENOSYS) { + return result; + } + } +#endif RESTARTABLE(ftruncate64(fd, length), result); return result; }
--- a/src/solaris/native/sun/nio/ch/FileDispatcherImpl.c Wed Feb 07 09:44:50 2018 -0800 +++ b/src/solaris/native/sun/nio/ch/FileDispatcherImpl.c Tue Feb 06 03:50:10 2018 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2012, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,6 +23,10 @@ * questions. */ +#if defined(__linux__) +#define _FILE_OFFSET_BITS 64 +#endif + #include "jni.h" #include "jni_util.h" #include "jvm.h" @@ -178,6 +182,30 @@ "Truncation failed"); } +JNIEXPORT jint JNICALL +Java_sun_nio_ch_FileDispatcherImpl_allocate0(JNIEnv *env, jobject this, + jobject fdo, jlong size) +{ +#if defined(__linux__) + /* + * On Linux, if the file size is being increased, then ftruncate64() + * will modify the metadata value of the size without actually allocating + * any blocks which can cause a SIGBUS error if the file is subsequently + * memory-mapped. + */ + // Return on success or if errno is neither EOPNOTSUPP nor ENOSYS + int result = posix_fallocate(fdval(env, fdo), 0, size); + if (result == 0) { + return 0; + } else if (errno != EOPNOTSUPP && errno != ENOSYS) { + return handle(env, result, "Allocation failed"); + } +#endif + return handle(env, + ftruncate64(fdval(env, fdo), size), + "Truncation failed"); +} + JNIEXPORT jlong JNICALL Java_sun_nio_ch_FileDispatcherImpl_size0(JNIEnv *env, jobject this, jobject fdo) {
--- a/src/windows/classes/sun/nio/ch/FileDispatcherImpl.java Wed Feb 07 09:44:50 2018 -0800 +++ b/src/windows/classes/sun/nio/ch/FileDispatcherImpl.java Tue Feb 06 03:50:10 2018 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -93,6 +93,11 @@ return truncate0(fd, size); } + int allocate(FileDescriptor fd, long size) throws IOException { + // truncate0() works for extending and truncating file size + return truncate0(fd, size); + } + long size(FileDescriptor fd) throws IOException { return size0(fd); }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/java/nio/channels/FileChannel/FileExtensionAndMap.java Tue Feb 06 03:50:10 2018 -0500 @@ -0,0 +1,203 @@ +/* + * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* @test + * @ignore This test has huge disk space requirements + * @bug 8168628 + * @summary Test extending files to very large sizes without hitting a SIGBUS + * @requires (os.family == "linux") + * @run main/othervm/timeout=600 -Xms4g -Xmx4g FileExtensionAndMap + * @run main/othervm/timeout=600 -Xms4g -Xmx4g FileExtensionAndMap true + */ + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.MappedByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.FileChannel; +import java.nio.channels.FileChannel.MapMode; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.Semaphore; +import java.util.stream.IntStream; + +public class FileExtensionAndMap { + + private static final ExecutorService CACHED_EXECUTORSERVICE = + Executors.newCachedThreadPool(); + + private static final String TMPDIR = System.getProperty("test.dir", "."); + + private static boolean useRaf = false; + + public static void main(String args[]) throws Exception { + if (args.length > 2) { + throw new IllegalArgumentException + ("Arguments: [true|false [targetFolder]]"); + } + + String defaultFolder = TMPDIR + File.separator + "target"; + if (args.length > 0) { + useRaf = Boolean.valueOf(args[0]); + if (args.length > 1) { + defaultFolder = args[1]; + } + } + final String targetFolder = defaultFolder; + Path p = Paths.get(targetFolder); + boolean targetExists = Files.exists(p); + if (!targetExists) { + Files.createDirectory(p); + } + + System.out.printf("Using RandomAccessFile: %s; target folder: %s%n", + useRaf, targetFolder); + + ForkJoinPool fjPool = new ForkJoinPool(3); + fjPool.submit(() -> { + IntStream.range(0, 20).parallel().forEach((index) -> { + String fileName = "testBigFile_" + index + ".dat"; + Path source = null; + Path target = null; + try { + source = Paths.get(TMPDIR, fileName); + testCreateBigFile(source); + target = Paths.get(targetFolder, fileName); + testFileCopy(source, target); + } catch (Throwable th) { + System.err.println("Error copying file with fileName: " + + fileName + " : " + th.getMessage()); + th.printStackTrace(System.err); + } finally { + try { + if (source != null) { + Files.deleteIfExists(source); + } + } catch (Throwable ignored) { + } + try { + if (target != null) { + Files.deleteIfExists(target); + } + } catch (Throwable ignored) { + } + } + }); + }).join(); + + if (!targetExists) { + Files.delete(p); + } + } + + private static void testFileCopy(Path source, Path target) + throws IOException { + Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING); + System.out.println("Finished copying file with fileName: " + + source.getFileName()); + } + + private static void testCreateBigFile(Path segmentFile) + throws IOException { + final Semaphore concurrencySemaphore = new Semaphore(5); + long fileSize = 3L * 1024L * 1024L * 1024L; + int blockSize = 10 * 1024 * 1024; + int loopCount = (int) Math.floorDiv(fileSize, blockSize); + + String fileName = segmentFile.getFileName().toString(); + if (useRaf) { + try (RandomAccessFile raf + = new RandomAccessFile(segmentFile.toFile(), "rw")) { + raf.setLength(fileSize); + try (FileChannel fc = raf.getChannel()) { + for (int i = 0; i < loopCount; i++) { + final long startPosition = 1L * blockSize * i; + concurrencySemaphore.acquireUninterruptibly(); + CACHED_EXECUTORSERVICE.submit(() -> { + writeTemplateData(fileName, fc, startPosition, + blockSize, concurrencySemaphore); + }); + } + } finally { + concurrencySemaphore.acquireUninterruptibly(5); + } + } + } else { + Path file = Files.createFile(segmentFile); + try (FileChannel fc = FileChannel.open(file, + StandardOpenOption.READ, StandardOpenOption.WRITE)) { + for (int i = 0; i < loopCount; i++) { + final long startPosition = 1L * blockSize * i; + concurrencySemaphore.acquireUninterruptibly(); + CACHED_EXECUTORSERVICE.submit(() -> { + writeTemplateData(fileName, fc, startPosition, + blockSize, concurrencySemaphore); + }); + } + } + } + } + + private static void writeTemplateData(String fileName, + FileChannel fc, long startPosition, int blockSize, + Semaphore concurrencySemaphore) { + try { + byte[] EMPTY_RECORD = new byte[blockSize / 256]; + + MappedByteBuffer mappedByteBuffer = fc.map(MapMode.READ_WRITE, + startPosition, blockSize); + IntStream.range(0, 256).forEach((recordIndex) -> { + try { + mappedByteBuffer.position((int) (recordIndex * + EMPTY_RECORD.length)); + mappedByteBuffer.put(EMPTY_RECORD, 0, EMPTY_RECORD.length); + } catch (Throwable th) { + System.err.println + ("Error in FileExtensionAndMap.writeTemplateData empty record for fileName: " + + fileName + ", startPosition: " + startPosition + ", recordIndex: " + + recordIndex + " : " + th.getMessage()); + th.printStackTrace(System.err); + } + }); + + mappedByteBuffer.force(); + } catch (Throwable th) { + if (!(th instanceof ClosedChannelException)) { + System.err.println + ("Error in FileExtensionAndMap.writeTemplateData empty record for fileName: " + + fileName + ", startPosition: " + startPosition + " : " + + th.getMessage()); + th.printStackTrace(System.err); + } + } finally { + concurrencySemaphore.release(); + } + } +}