changeset 9454:0bd3170be8c7 icedtea-3.4.0pre01

8043780, PR3368: Use open(O_CLOEXEC) instead of fcntl(FD_CLOEXEC) Summary: Use open(O_CLOEXEC) where available; fall back to FD_CLOEXEC when necessary Reviewed-by: rasbold, dholmes
author martin
date Sun, 07 May 2017 01:20:27 +0100
parents baf64c88538f
children f998c8dc320e
files src/os/linux/vm/os_linux.cpp
diffstat 1 files changed, 43 insertions(+), 28 deletions(-) [+]
line wrap: on
line diff
--- a/src/os/linux/vm/os_linux.cpp	Thu Sep 22 02:04:40 2016 -0700
+++ b/src/os/linux/vm/os_linux.cpp	Sun May 07 01:20:27 2017 +0100
@@ -5282,11 +5282,41 @@
     errno = ENAMETOOLONG;
     return -1;
   }
-  int fd;
   int o_delete = (oflag & O_DELETE);
   oflag = oflag & ~O_DELETE;
 
-  fd = ::open64(path, oflag, mode);
+
+    /* All file descriptors that are opened in the Java process and not
+     * specifically destined for a subprocess should have the close-on-exec
+     * flag set.  If we don't set it, then careless 3rd party native code
+     * might fork and exec without closing all appropriate file descriptors
+     * (e.g. as we do in closeDescriptors in UNIXProcess.c), and this in
+     * turn might:
+     *
+     * - cause end-of-file to fail to be detected on some file
+     *   descriptors, resulting in mysterious hangs, or
+     *
+     * - might cause an fopen in the subprocess to fail on a system
+     *   suffering from bug 1085341.
+     *
+     * (Yes, the default setting of the close-on-exec flag is a Unix
+     * design flaw)
+     *
+     * See:
+     * 1085341: 32-bit stdio routines should support file descriptors >255
+     * 4843136: (process) pipe file descriptor from Runtime.exec not being closed
+     * 6339493: (process) Runtime.exec does not close all file descriptors on Solaris 9
+     */
+  // Modern Linux kernels (after 2.6.23 2007) support O_CLOEXEC with open().
+  // O_CLOEXEC is preferable to using FD_CLOEXEC on an open file descriptor
+  // because it saves a system call and removes a small window where the flag
+  // is unset.  On ancient Linux kernels the O_CLOEXEC flag will be ignored
+  // and we fall back to using FD_CLOEXEC (see below).
+#ifdef O_CLOEXEC
+  oflag |= O_CLOEXEC;
+#endif
+
+  int fd = ::open64(path, oflag, mode);
   if (fd == -1) return -1;
 
   //If the open succeeded, the file might still be a directory
@@ -5307,34 +5337,19 @@
     }
   }
 
-    /*
-     * All file descriptors that are opened in the JVM and not
-     * specifically destined for a subprocess should have the
-     * close-on-exec flag set.  If we don't set it, then careless 3rd
-     * party native code might fork and exec without closing all
-     * appropriate file descriptors (e.g. as we do in closeDescriptors in
-     * UNIXProcess.c), and this in turn might:
-     *
-     * - cause end-of-file to fail to be detected on some file
-     *   descriptors, resulting in mysterious hangs, or
-     *
-     * - might cause an fopen in the subprocess to fail on a system
-     *   suffering from bug 1085341.
-     *
-     * (Yes, the default setting of the close-on-exec flag is a Unix
-     * design flaw)
-     *
-     * See:
-     * 1085341: 32-bit stdio routines should support file descriptors >255
-     * 4843136: (process) pipe file descriptor from Runtime.exec not being closed
-     * 6339493: (process) Runtime.exec does not close all file descriptors on Solaris 9
-     */
 #ifdef FD_CLOEXEC
-    {
-        int flags = ::fcntl(fd, F_GETFD);
-        if (flags != -1)
-            ::fcntl(fd, F_SETFD, flags | FD_CLOEXEC);
+  // Validate that the use of the O_CLOEXEC flag on open above worked.
+  // With recent kernels, we will perform this check exactly once.
+  static sig_atomic_t O_CLOEXEC_is_known_to_work = 0;
+  if (!O_CLOEXEC_is_known_to_work) {
+    int flags = ::fcntl(fd, F_GETFD);
+    if (flags != -1) {
+      if ((flags & FD_CLOEXEC) != 0)
+        O_CLOEXEC_is_known_to_work = 1;
+      else
+	::fcntl(fd, F_SETFD, flags | FD_CLOEXEC);
     }
+  }
 #endif
 
   if (o_delete != 0) {