Improve JNI loading efficiency (#1224)

A hash is stored for each native library with the name libraryName.hash.
If the library is not found on the system search path, it is extracted to a cache directory.
Extracted libraries are named with the hash appended, so the library will not be
re-extracted if one with the same hash already exists.

Hashing without the hash file requires double traversing if the file is not in the cache,
but it is still faster than creating a new file in most cases.  This won't be needed
after opencv is updated to provide a hash as well.

cscore/src/main/java/edu/wpi/cscore/CameraServerJNI.java

@@ -7,62 +7,28 @@
 
 package edu.wpi.cscore;
 
-import java.io.File;
-import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
 import java.util.function.Consumer;
 
 import org.opencv.core.Core;
 
-import edu.wpi.first.wpiutil.RuntimeDetector;
+import edu.wpi.first.wpiutil.RuntimeLoader;
 
 public class CameraServerJNI {
   static boolean libraryLoaded = false;
-  static File jniLibrary = null;
   static boolean cvLibraryLoaded = false;
-  static File cvJniLibrary = null;
+
+  static RuntimeLoader<CameraServerJNI> loader = null;
+  static RuntimeLoader<Core> cvLoader = null;
 
   static {
     if (!libraryLoaded) {
       try {
-        System.loadLibrary("cscore");
-      } catch (UnsatisfiedLinkError linkError) {
-        try {
-          String resname = RuntimeDetector.getLibraryResource("cscore");
-          InputStream is = CameraServerJNI.class.getResourceAsStream(resname);
-          if (is != null) {
-            // create temporary file
-            if (System.getProperty("os.name").startsWith("Windows")) {
-              jniLibrary = File.createTempFile("CameraServerJNI", ".dll");
-            } else if (System.getProperty("os.name").startsWith("Mac")) {
-              jniLibrary = File.createTempFile("libCameraServerJNI", ".dylib");
-            } else {
-              jniLibrary = File.createTempFile("libCameraServerJNI", ".so");
-            }
-            // flag for delete on exit
-            jniLibrary.deleteOnExit();
-            OutputStream os = new FileOutputStream(jniLibrary);
-
-            byte[] buffer = new byte[1024];
-            int readBytes;
-            try {
-              while ((readBytes = is.read(buffer)) != -1) {
-                os.write(buffer, 0, readBytes);
-              }
-            } finally {
-              os.close();
-              is.close();
-            }
-            System.load(jniLibrary.getAbsolutePath());
-          } else {
-            System.loadLibrary("cscore");
-          }
-        } catch (IOException ex) {
-          ex.printStackTrace();
-          System.exit(1);
-        }
+        loader = new RuntimeLoader<>("cscore", RuntimeLoader.getDefaultExtractionRoot(), CameraServerJNI.class);
+        loader.loadLibrary();
+      } catch (IOException ex) {
+        ex.printStackTrace();
+        System.exit(1);
       }
       libraryLoaded = true;
     }
@@ -70,43 +36,11 @@ public class CameraServerJNI {
     String opencvName = Core.NATIVE_LIBRARY_NAME;
     if (!cvLibraryLoaded) {
       try {
-
-        System.loadLibrary(opencvName);
-      } catch (UnsatisfiedLinkError linkError) {
-        try {
-          String resname = RuntimeDetector.getLibraryResource(opencvName);
-          InputStream is = CameraServerJNI.class.getResourceAsStream(resname);
-          if (is != null) {
-            // create temporary file
-            if (System.getProperty("os.name").startsWith("Windows")) {
-              cvJniLibrary = File.createTempFile("OpenCVJNI", ".dll");
-            } else if (System.getProperty("os.name").startsWith("Mac")) {
-              cvJniLibrary = File.createTempFile("libOpenCVJNI", ".dylib");
-            } else {
-              cvJniLibrary = File.createTempFile("libOpenCVJNI", ".so");
-            }
-            // flag for delete on exit
-            cvJniLibrary.deleteOnExit();
-            OutputStream os = new FileOutputStream(cvJniLibrary);
-
-            byte[] buffer = new byte[1024];
-            int readBytes;
-            try {
-              while ((readBytes = is.read(buffer)) != -1) {
-                os.write(buffer, 0, readBytes);
-              }
-            } finally {
-              os.close();
-              is.close();
-            }
-            System.load(cvJniLibrary.getAbsolutePath());
-          } else {
-            System.loadLibrary(opencvName);
-          }
-        } catch (IOException ex) {
-          ex.printStackTrace();
-          System.exit(1);
-        }
+        cvLoader = new RuntimeLoader<>(opencvName, RuntimeLoader.getDefaultExtractionRoot(), Core.class);
+        cvLoader.loadLibraryHashed();
+      } catch (IOException ex) {
+        ex.printStackTrace();
+        System.exit(1);
       }
       cvLibraryLoaded = true;
     }

hal/src/main/java/edu/wpi/first/wpilibj/hal/JNIWrapper.java

@@ -7,63 +7,28 @@
 
 package edu.wpi.first.wpilibj.hal;
 
-import java.io.File;
-import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
 
-import edu.wpi.first.wpiutil.RuntimeDetector;
+import edu.wpi.first.wpiutil.RuntimeLoader;
 
 /**
  * Base class for all JNI wrappers.
  */
 public class JNIWrapper {
   static boolean libraryLoaded = false;
-  static File jniLibrary = null;
+  static RuntimeLoader<JNIWrapper> loader = null;
 
   static {
     if (!libraryLoaded) {
-      String jniFileName = "wpiHal";
       try {
-        System.loadLibrary(jniFileName);
-      } catch (UnsatisfiedLinkError ule) {
-        try {
-          String resname = RuntimeDetector.getLibraryResource(jniFileName);
-          InputStream is = JNIWrapper.class.getResourceAsStream(resname);
-          if (is != null) {
-            // create temporary file
-            if (System.getProperty("os.name").startsWith("Windows")) {
-              jniLibrary = File.createTempFile(jniFileName, ".dll");
-            } else if (System.getProperty("os.name").startsWith("Mac")) {
-              jniLibrary = File.createTempFile(jniFileName, ".dylib");
-            } else {
-              jniLibrary = File.createTempFile(jniFileName, ".so");
-            }
-            // flag for delete on exit
-            jniLibrary.deleteOnExit();
-            OutputStream os = new FileOutputStream(jniLibrary);
-
-            byte[] buffer = new byte[1024];
-            int readBytes;
-            try {
-              while ((readBytes = is.read(buffer)) != -1) {
-                os.write(buffer, 0, readBytes);
-              }
-            } finally {
-              os.close();
-              is.close();
-            }
-            System.load(jniLibrary.getAbsolutePath());
-          } else {
-            System.loadLibrary(jniFileName);
-          }
-        } catch (IOException ex) {
-          ex.printStackTrace();
-          System.exit(1);
-        }
+        loader = new RuntimeLoader<>("wpiHal", RuntimeLoader.getDefaultExtractionRoot(), JNIWrapper.class);
+        loader.loadLibrary();
+      } catch (IOException ex) {
+        ex.printStackTrace();
+        System.exit(1);
       }
       libraryLoaded = true;
+      libraryLoaded = true;
     }
   }
 }

ntcore/src/main/java/edu/wpi/first/networktables/NetworkTablesJNI.java

@@ -7,58 +7,23 @@
 
 package edu.wpi.first.networktables;
 
-import java.io.File;
-import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
 import java.nio.ByteBuffer;
 
-import edu.wpi.first.wpiutil.RuntimeDetector;
+import edu.wpi.first.wpiutil.RuntimeLoader;
 
 public final class NetworkTablesJNI {
   static boolean libraryLoaded = false;
-  static File jniLibrary = null;
+  static RuntimeLoader<NetworkTablesJNI> loader = null;
 
   static {
     if (!libraryLoaded) {
       try {
-        System.loadLibrary("ntcore");
-      } catch (UnsatisfiedLinkError linkError) {
-        try {
-          String resname = RuntimeDetector.getLibraryResource("ntcore");
-          InputStream is = NetworkTablesJNI.class.getResourceAsStream(resname);
-          if (is != null) {
-            // create temporary file
-            if (System.getProperty("os.name").startsWith("Windows")) {
-              jniLibrary = File.createTempFile("NetworkTablesJNI", ".dll");
-            } else if (System.getProperty("os.name").startsWith("Mac")) {
-              jniLibrary = File.createTempFile("libNetworkTablesJNI", ".dylib");
-            } else {
-              jniLibrary = File.createTempFile("libNetworkTablesJNI", ".so");
-            }
-            // flag for delete on exit
-            jniLibrary.deleteOnExit();
-            OutputStream os = new FileOutputStream(jniLibrary);
-
-            byte[] buffer = new byte[1024];
-            int readBytes;
-            try {
-              while ((readBytes = is.read(buffer)) != -1) {
-                os.write(buffer, 0, readBytes);
-              }
-            } finally {
-              os.close();
-              is.close();
-            }
-            System.load(jniLibrary.getAbsolutePath());
-          } else {
-            System.loadLibrary("ntcore");
-          }
-        } catch (IOException ex) {
-          ex.printStackTrace();
-          System.exit(1);
-        }
+        loader = new RuntimeLoader<>("ntcore", RuntimeLoader.getDefaultExtractionRoot(), NetworkTablesJNI.class);
+        loader.loadLibrary();
+      } catch (IOException ex) {
+        ex.printStackTrace();
+        System.exit(1);
       }
       libraryLoaded = true;
     }

shared/config.gradle

@@ -268,9 +268,7 @@ ext.createAllCombined = { list, name, base, type, project ->
         duplicatesStrategy = 'exclude'
 
         list.each {
-            it.outputs.files.each {
-                from project.zipTree(it)
-            }
+            from project.zipTree(it.archivePath)
             dependsOn it
         }
     }
@@ -289,7 +287,7 @@ ext.includeStandardZipFormat = { task, value ->
     value.each { binary ->
         if (binary.buildable) {
             if (binary instanceof SharedLibraryBinarySpec) {
-                task.dependsOn binary.buildTask
+                task.dependsOn binary.tasks.link
                 task.from(new File(binary.sharedLibraryFile.absolutePath + ".debug")) {
                     into getPlatformPath(binary) + '/shared'
                 }
@@ -306,7 +304,7 @@ ext.includeStandardZipFormat = { task, value ->
                     into getPlatformPath(binary) + '/shared'
                 }
             } else if (binary instanceof StaticLibraryBinarySpec) {
-                task.dependsOn binary.buildTask
+                task.dependsOn binary.tasks.createStaticLib
                 task.from(binary.staticLibraryFile) {
                     into getPlatformPath(binary) + '/static'
                 }

shared/jni/publish.gradle

@@ -1,3 +1,4 @@
+import java.security.MessageDigest
 apply plugin: 'maven-publish'
 
 def pubVersion
@@ -81,7 +82,16 @@ model {
             value.each { binary ->
                 if (binary.buildable) {
                     if (binary instanceof SharedLibraryBinarySpec) {
-                        task.dependsOn binary.buildTask
+                        task.dependsOn binary.tasks.link
+                        def hashFile = new File(binary.sharedLibraryFile.parentFile.absolutePath, "${binary.component.baseName}.hash")
+                        task.outputs.file(hashFile)
+                        task.inputs.file(binary.sharedLibraryFile)
+                        task.from(hashFile) {
+                            into getPlatformPath(binary)
+                        }
+                        task.doFirst {
+                            hashFile.text = MessageDigest.getInstance("MD5").digest(binary.sharedLibraryFile.bytes).encodeHex().toString()
+                        }
                         task.from(binary.sharedLibraryFile) {
                             into getPlatformPath(binary)
                         }

wpiutil/src/main/java/edu/wpi/first/wpiutil/RuntimeDetector.java

@@ -94,6 +94,16 @@ public final class RuntimeDetector {
     return toReturn;
   }
 
+  /**
+   * Get the path to the hash to the requested resource.
+   */
+  public static synchronized String getHashLibraryResource(String libName) {
+    computePlatform();
+
+    String toReturn = filePath + libName + ".hash";
+    return toReturn;
+  }
+
   public static boolean isAthena() {
     File runRobotFile = new File("/usr/local/frc/bin/frcRunRobot.sh");
     return runRobotFile.exists();

wpiutil/src/main/java/edu/wpi/first/wpiutil/RuntimeLoader.java

@@ -0,0 +1,161 @@
+/*----------------------------------------------------------------------------*/
+/* Copyright (c) 2018 FIRST. All Rights Reserved.                             */
+/* Open Source Software - may be modified and shared by FRC teams. The code   */
+/* must be accompanied by the FIRST BSD license file in the root directory of */
+/* the project.                                                               */
+/*----------------------------------------------------------------------------*/
+
+package edu.wpi.first.wpiutil;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.security.DigestInputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Locale;
+import java.util.Scanner;
+
+public final class RuntimeLoader<T> {
+  private static String defaultExtractionRoot;
+
+  /**
+   * Gets the default extration root location (~/.wpilib/nativecache).
+   */
+  public static synchronized String getDefaultExtractionRoot() {
+    if (defaultExtractionRoot != null) {
+      return defaultExtractionRoot;
+    }
+    String home = System.getProperty("user.home");
+    defaultExtractionRoot = Paths.get(home, ".wpilib", "nativecache").toString();
+    return defaultExtractionRoot;
+  }
+
+  private final String m_libraryName;
+  private final Class<T> m_loadClass;
+  private final String m_extractionRoot;
+
+  /**
+   * Creates a new library loader.
+   *
+   * <p>Resources loaded on disk from extractionRoot, and from classpath from the
+   * passed in class. Library name is the passed in name.
+   */
+  public RuntimeLoader(String libraryName, String extractionRoot, Class<T> cls) {
+    m_libraryName = libraryName;
+    m_loadClass = cls;
+    m_extractionRoot = extractionRoot;
+  }
+
+  /**
+   * Loads a native library.
+   */
+  @SuppressWarnings("PMD.PreserveStackTrace")
+  public void loadLibrary() throws IOException {
+    try {
+      // First, try loading path
+      System.loadLibrary(m_libraryName);
+      return;
+    } catch (UnsatisfiedLinkError ule) {
+      // Then load the hash from the resources
+      String hashName = RuntimeDetector.getHashLibraryResource(m_libraryName);
+      String resname = RuntimeDetector.getLibraryResource(m_libraryName);
+      try (InputStream hashIs = m_loadClass.getResourceAsStream(hashName)) {
+        if (hashIs == null) {
+          throw new IOException(hashName + " Resource not found");
+        }
+        try (Scanner scanner = new Scanner(hashIs)) {
+          String hash = scanner.nextLine();
+          File jniLibrary = new File(m_extractionRoot, resname + "." + hash);
+          try {
+            // Try to load from an already extracted hash
+            System.load(jniLibrary.getAbsolutePath());
+          } catch (UnsatisfiedLinkError ule2) {
+            // If extraction failed, extract
+            try (InputStream resIs = m_loadClass.getResourceAsStream(resname)) {
+              if (resIs == null) {
+                throw new IOException(resname + " Resource not found");
+              }
+              jniLibrary.getParentFile().mkdirs();
+              try (OutputStream os = Files.newOutputStream(jniLibrary.toPath())) {
+                byte[] buffer = new byte[0xFFFF]; // 64K copy buffer
+                int readBytes;
+                while ((readBytes = resIs.read(buffer)) != -1) { // NOPMD
+                  os.write(buffer, 0, readBytes);
+                }
+              }
+              System.load(jniLibrary.getAbsolutePath());
+            }
+          }
+        }
+      }
+    }
+  }
+
+  /**
+   * Load a native library by directly hashing the file.
+   */
+  @SuppressWarnings({"PMD.NPathComplexity", "PMD.PreserveStackTrace", "PMD.EmptyWhileStmt",
+                     "PMD.AvoidThrowingRawExceptionTypes", "PMD.CyclomaticComplexity"})
+  public void loadLibraryHashed() throws IOException {
+    try {
+      // First, try loading path
+      System.loadLibrary(m_libraryName);
+      return;
+    } catch (UnsatisfiedLinkError ule) {
+      // Then load the hash from the input file
+      String resname = RuntimeDetector.getLibraryResource(m_libraryName);
+      String hash = null;
+      try (InputStream is = m_loadClass.getResourceAsStream(resname)) {
+        if (is == null) {
+          throw new IOException(resname + " Resource not found");
+        }
+        MessageDigest md = null;
+        try {
+          md = MessageDigest.getInstance("MD5");
+        } catch (NoSuchAlgorithmException nsae) {
+          throw new RuntimeException("Weird Hash Algorithm?");
+        }
+        try (DigestInputStream dis = new DigestInputStream(is, md)) {
+          // Read the entire buffer once to hash
+          byte[] buffer = new byte[0xFFFF];
+          while (dis.read(buffer) > -1) {}
+          MessageDigest digest = dis.getMessageDigest();
+          byte[] digestOutput = digest.digest();
+          StringBuilder builder = new StringBuilder();
+          for (byte b : digestOutput) {
+            builder.append(String.format("%02X", b));
+          }
+          hash = builder.toString().toLowerCase(Locale.ENGLISH);
+        }
+      }
+      if (hash == null) {
+        throw new IOException("Weird Hash?");
+      }
+      File jniLibrary = new File(m_extractionRoot, resname + "." + hash);
+      try {
+        // Try to load from an already extracted hash
+        System.load(jniLibrary.getAbsolutePath());
+      } catch (UnsatisfiedLinkError ule2) {
+        // If extraction failed, extract
+        try (InputStream resIs = m_loadClass.getResourceAsStream(resname)) {
+          if (resIs == null) {
+            throw new IOException(resname + " Resource not found");
+          }
+          jniLibrary.getParentFile().mkdirs();
+          try (OutputStream os = Files.newOutputStream(jniLibrary.toPath())) {
+            byte[] buffer = new byte[0xFFFF]; // 64K copy buffer
+            int readBytes;
+            while ((readBytes = resIs.read(buffer)) != -1) { // NOPMD
+              os.write(buffer, 0, readBytes);
+            }
+          }
+          System.load(jniLibrary.getAbsolutePath());
+        }
+      }
+    }
+  }
+}