# HG changeset patch
# User martin
# Date 1493178961 -3600
# Node ID 8d8fc01744986c518e31e9b6e50af697f9dc4a18
# Parent  83e23c65e3367d109d57de40564c273bb0dc926f
8043780, PR3369: Use open(O_CLOEXEC) instead of fcntl(FD_CLOEXEC)
Summary: Use open(O_CLOEXEC) where available; fall back to FD_CLOEXEC when necessary
Reviewed-by: rasbold, dholmes

diff -r 83e23c65e336 -r 8d8fc0174498 src/os/linux/vm/os_linux.cpp
--- a/src/os/linux/vm/os_linux.cpp	Tue Mar 28 06:12:49 2017 -0400
+++ b/src/os/linux/vm/os_linux.cpp	Wed Apr 26 04:56:01 2017 +0100
@@ -5331,33 +5331,11 @@
     errno = ENAMETOOLONG;
     return -1;
   }
-  int fd;
   int o_delete = (oflag & O_DELETE);
   oflag = oflag & ~O_DELETE;
 
-  fd = ::open64(path, oflag, mode);
-  if (fd == -1) return -1;
-
-  //If the open succeeded, the file might still be a directory
-  {
-    struct stat64 buf64;
-    int ret = ::fstat64(fd, &buf64);
-    int st_mode = buf64.st_mode;
-
-    if (ret != -1) {
-      if ((st_mode & S_IFMT) == S_IFDIR) {
-        errno = EISDIR;
-        ::close(fd);
-        return -1;
-      }
-    } else {
-      ::close(fd);
-      return -1;
-    }
-  }
-
     /*
-     * All file descriptors that are opened in the JVM and not
+     * All file descriptors that are opened in the Java process and not
      * specifically destined for a subprocess should have the
      * close-on-exec flag set.  If we don't set it, then careless 3rd
      * party native code might fork and exec without closing all
@@ -5378,12 +5356,49 @@
      * 4843136: (process) pipe file descriptor from Runtime.exec not being closed
      * 6339493: (process) Runtime.exec does not close all file descriptors on Solaris 9
      */
+  // Modern Linux kernels (after 2.6.23 2007) support O_CLOEXEC with open().
+  // O_CLOEXEC is preferable to using FD_CLOEXEC on an open file descriptor
+  // because it saves a system call and removes a small window where the flag
+  // is unset.  On ancient Linux kernels the O_CLOEXEC flag will be ignored
+  // and we fall back to using FD_CLOEXEC (see below).
+#ifdef O_CLOEXEC
+  oflag |= O_CLOEXEC;
+#endif
+
+  int fd = ::open64(path, oflag, mode);
+  if (fd == -1) return -1;
+
+  //If the open succeeded, the file might still be a directory
+  {
+    struct stat64 buf64;
+    int ret = ::fstat64(fd, &buf64);
+    int st_mode = buf64.st_mode;
+
+    if (ret != -1) {
+      if ((st_mode & S_IFMT) == S_IFDIR) {
+        errno = EISDIR;
+        ::close(fd);
+        return -1;
+      }
+    } else {
+      ::close(fd);
+      return -1;
+    }
+  }
+
 #ifdef FD_CLOEXEC
-    {
-        int flags = ::fcntl(fd, F_GETFD);
-        if (flags != -1)
-            ::fcntl(fd, F_SETFD, flags | FD_CLOEXEC);
+  // Validate that the use of the O_CLOEXEC flag on open above worked.
+  // With recent kernels, we will perform this check exactly once.
+  static sig_atomic_t O_CLOEXEC_is_known_to_work = 0;
+  if (!O_CLOEXEC_is_known_to_work) {
+    int flags = ::fcntl(fd, F_GETFD);
+    if (flags != -1) {
+      if ((flags & FD_CLOEXEC) != 0)
+        O_CLOEXEC_is_known_to_work = 1;
+      else
+        ::fcntl(fd, F_SETFD, flags | FD_CLOEXEC);
     }
+  }
 #endif
 
   if (o_delete != 0) {