From 00c2cd7dab374bba9531cdfb922a8f6274e271ad Mon Sep 17 00:00:00 2001 From: Thad House Date: Sun, 29 Jul 2018 10:20:41 -0700 Subject: [PATCH] Improve JNI loading efficiency (#1224) A hash is stored for each native library with the name libraryName.hash. If the library is not found on the system search path, it is extracted to a cache directory. Extracted libraries are named with the hash appended, so the library will not be re-extracted if one with the same hash already exists. Hashing without the hash file requires double traversing if the file is not in the cache, but it is still faster than creating a new file in most cases. This won't be needed after opencv is updated to provide a hash as well. --- .../java/edu/wpi/cscore/CameraServerJNI.java | 94 ++-------- .../edu/wpi/first/wpilibj/hal/JNIWrapper.java | 51 +----- .../first/networktables/NetworkTablesJNI.java | 49 +----- shared/config.gradle | 8 +- shared/jni/publish.gradle | 12 +- .../wpi/first/wpiutil/RuntimeDetector.java | 10 ++ .../edu/wpi/first/wpiutil/RuntimeLoader.java | 161 ++++++++++++++++++ 7 files changed, 214 insertions(+), 171 deletions(-) create mode 100644 wpiutil/src/main/java/edu/wpi/first/wpiutil/RuntimeLoader.java diff --git a/cscore/src/main/java/edu/wpi/cscore/CameraServerJNI.java b/cscore/src/main/java/edu/wpi/cscore/CameraServerJNI.java index a2a4052bac..76fd4342b4 100644 --- a/cscore/src/main/java/edu/wpi/cscore/CameraServerJNI.java +++ b/cscore/src/main/java/edu/wpi/cscore/CameraServerJNI.java @@ -7,62 +7,28 @@ package edu.wpi.cscore; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.util.function.Consumer; import org.opencv.core.Core; -import edu.wpi.first.wpiutil.RuntimeDetector; +import edu.wpi.first.wpiutil.RuntimeLoader; public class CameraServerJNI { static boolean libraryLoaded = false; - static File jniLibrary = null; static boolean cvLibraryLoaded = false; - static File cvJniLibrary = null; + + static RuntimeLoader loader = null; + static RuntimeLoader cvLoader = null; static { if (!libraryLoaded) { try { - System.loadLibrary("cscore"); - } catch (UnsatisfiedLinkError linkError) { - try { - String resname = RuntimeDetector.getLibraryResource("cscore"); - InputStream is = CameraServerJNI.class.getResourceAsStream(resname); - if (is != null) { - // create temporary file - if (System.getProperty("os.name").startsWith("Windows")) { - jniLibrary = File.createTempFile("CameraServerJNI", ".dll"); - } else if (System.getProperty("os.name").startsWith("Mac")) { - jniLibrary = File.createTempFile("libCameraServerJNI", ".dylib"); - } else { - jniLibrary = File.createTempFile("libCameraServerJNI", ".so"); - } - // flag for delete on exit - jniLibrary.deleteOnExit(); - OutputStream os = new FileOutputStream(jniLibrary); - - byte[] buffer = new byte[1024]; - int readBytes; - try { - while ((readBytes = is.read(buffer)) != -1) { - os.write(buffer, 0, readBytes); - } - } finally { - os.close(); - is.close(); - } - System.load(jniLibrary.getAbsolutePath()); - } else { - System.loadLibrary("cscore"); - } - } catch (IOException ex) { - ex.printStackTrace(); - System.exit(1); - } + loader = new RuntimeLoader<>("cscore", RuntimeLoader.getDefaultExtractionRoot(), CameraServerJNI.class); + loader.loadLibrary(); + } catch (IOException ex) { + ex.printStackTrace(); + System.exit(1); } libraryLoaded = true; } @@ -70,43 +36,11 @@ public class CameraServerJNI { String opencvName = Core.NATIVE_LIBRARY_NAME; if (!cvLibraryLoaded) { try { - - System.loadLibrary(opencvName); - } catch (UnsatisfiedLinkError linkError) { - try { - String resname = RuntimeDetector.getLibraryResource(opencvName); - InputStream is = CameraServerJNI.class.getResourceAsStream(resname); - if (is != null) { - // create temporary file - if (System.getProperty("os.name").startsWith("Windows")) { - cvJniLibrary = File.createTempFile("OpenCVJNI", ".dll"); - } else if (System.getProperty("os.name").startsWith("Mac")) { - cvJniLibrary = File.createTempFile("libOpenCVJNI", ".dylib"); - } else { - cvJniLibrary = File.createTempFile("libOpenCVJNI", ".so"); - } - // flag for delete on exit - cvJniLibrary.deleteOnExit(); - OutputStream os = new FileOutputStream(cvJniLibrary); - - byte[] buffer = new byte[1024]; - int readBytes; - try { - while ((readBytes = is.read(buffer)) != -1) { - os.write(buffer, 0, readBytes); - } - } finally { - os.close(); - is.close(); - } - System.load(cvJniLibrary.getAbsolutePath()); - } else { - System.loadLibrary(opencvName); - } - } catch (IOException ex) { - ex.printStackTrace(); - System.exit(1); - } + cvLoader = new RuntimeLoader<>(opencvName, RuntimeLoader.getDefaultExtractionRoot(), Core.class); + cvLoader.loadLibraryHashed(); + } catch (IOException ex) { + ex.printStackTrace(); + System.exit(1); } cvLibraryLoaded = true; } diff --git a/hal/src/main/java/edu/wpi/first/wpilibj/hal/JNIWrapper.java b/hal/src/main/java/edu/wpi/first/wpilibj/hal/JNIWrapper.java index e5ba15864a..e49705cee1 100644 --- a/hal/src/main/java/edu/wpi/first/wpilibj/hal/JNIWrapper.java +++ b/hal/src/main/java/edu/wpi/first/wpilibj/hal/JNIWrapper.java @@ -7,63 +7,28 @@ package edu.wpi.first.wpilibj.hal; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import edu.wpi.first.wpiutil.RuntimeDetector; +import edu.wpi.first.wpiutil.RuntimeLoader; /** * Base class for all JNI wrappers. */ public class JNIWrapper { static boolean libraryLoaded = false; - static File jniLibrary = null; + static RuntimeLoader loader = null; static { if (!libraryLoaded) { - String jniFileName = "wpiHal"; try { - System.loadLibrary(jniFileName); - } catch (UnsatisfiedLinkError ule) { - try { - String resname = RuntimeDetector.getLibraryResource(jniFileName); - InputStream is = JNIWrapper.class.getResourceAsStream(resname); - if (is != null) { - // create temporary file - if (System.getProperty("os.name").startsWith("Windows")) { - jniLibrary = File.createTempFile(jniFileName, ".dll"); - } else if (System.getProperty("os.name").startsWith("Mac")) { - jniLibrary = File.createTempFile(jniFileName, ".dylib"); - } else { - jniLibrary = File.createTempFile(jniFileName, ".so"); - } - // flag for delete on exit - jniLibrary.deleteOnExit(); - OutputStream os = new FileOutputStream(jniLibrary); - - byte[] buffer = new byte[1024]; - int readBytes; - try { - while ((readBytes = is.read(buffer)) != -1) { - os.write(buffer, 0, readBytes); - } - } finally { - os.close(); - is.close(); - } - System.load(jniLibrary.getAbsolutePath()); - } else { - System.loadLibrary(jniFileName); - } - } catch (IOException ex) { - ex.printStackTrace(); - System.exit(1); - } + loader = new RuntimeLoader<>("wpiHal", RuntimeLoader.getDefaultExtractionRoot(), JNIWrapper.class); + loader.loadLibrary(); + } catch (IOException ex) { + ex.printStackTrace(); + System.exit(1); } libraryLoaded = true; + libraryLoaded = true; } } } diff --git a/ntcore/src/main/java/edu/wpi/first/networktables/NetworkTablesJNI.java b/ntcore/src/main/java/edu/wpi/first/networktables/NetworkTablesJNI.java index 156adc4d09..1a3f4fe9d3 100644 --- a/ntcore/src/main/java/edu/wpi/first/networktables/NetworkTablesJNI.java +++ b/ntcore/src/main/java/edu/wpi/first/networktables/NetworkTablesJNI.java @@ -7,58 +7,23 @@ package edu.wpi.first.networktables; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.nio.ByteBuffer; -import edu.wpi.first.wpiutil.RuntimeDetector; +import edu.wpi.first.wpiutil.RuntimeLoader; public final class NetworkTablesJNI { static boolean libraryLoaded = false; - static File jniLibrary = null; + static RuntimeLoader loader = null; static { if (!libraryLoaded) { try { - System.loadLibrary("ntcore"); - } catch (UnsatisfiedLinkError linkError) { - try { - String resname = RuntimeDetector.getLibraryResource("ntcore"); - InputStream is = NetworkTablesJNI.class.getResourceAsStream(resname); - if (is != null) { - // create temporary file - if (System.getProperty("os.name").startsWith("Windows")) { - jniLibrary = File.createTempFile("NetworkTablesJNI", ".dll"); - } else if (System.getProperty("os.name").startsWith("Mac")) { - jniLibrary = File.createTempFile("libNetworkTablesJNI", ".dylib"); - } else { - jniLibrary = File.createTempFile("libNetworkTablesJNI", ".so"); - } - // flag for delete on exit - jniLibrary.deleteOnExit(); - OutputStream os = new FileOutputStream(jniLibrary); - - byte[] buffer = new byte[1024]; - int readBytes; - try { - while ((readBytes = is.read(buffer)) != -1) { - os.write(buffer, 0, readBytes); - } - } finally { - os.close(); - is.close(); - } - System.load(jniLibrary.getAbsolutePath()); - } else { - System.loadLibrary("ntcore"); - } - } catch (IOException ex) { - ex.printStackTrace(); - System.exit(1); - } + loader = new RuntimeLoader<>("ntcore", RuntimeLoader.getDefaultExtractionRoot(), NetworkTablesJNI.class); + loader.loadLibrary(); + } catch (IOException ex) { + ex.printStackTrace(); + System.exit(1); } libraryLoaded = true; } diff --git a/shared/config.gradle b/shared/config.gradle index 6a0ac14ad5..a46d44a12a 100644 --- a/shared/config.gradle +++ b/shared/config.gradle @@ -268,9 +268,7 @@ ext.createAllCombined = { list, name, base, type, project -> duplicatesStrategy = 'exclude' list.each { - it.outputs.files.each { - from project.zipTree(it) - } + from project.zipTree(it.archivePath) dependsOn it } } @@ -289,7 +287,7 @@ ext.includeStandardZipFormat = { task, value -> value.each { binary -> if (binary.buildable) { if (binary instanceof SharedLibraryBinarySpec) { - task.dependsOn binary.buildTask + task.dependsOn binary.tasks.link task.from(new File(binary.sharedLibraryFile.absolutePath + ".debug")) { into getPlatformPath(binary) + '/shared' } @@ -306,7 +304,7 @@ ext.includeStandardZipFormat = { task, value -> into getPlatformPath(binary) + '/shared' } } else if (binary instanceof StaticLibraryBinarySpec) { - task.dependsOn binary.buildTask + task.dependsOn binary.tasks.createStaticLib task.from(binary.staticLibraryFile) { into getPlatformPath(binary) + '/static' } diff --git a/shared/jni/publish.gradle b/shared/jni/publish.gradle index 4a92b148a8..ea041d78d9 100644 --- a/shared/jni/publish.gradle +++ b/shared/jni/publish.gradle @@ -1,3 +1,4 @@ +import java.security.MessageDigest apply plugin: 'maven-publish' def pubVersion @@ -81,7 +82,16 @@ model { value.each { binary -> if (binary.buildable) { if (binary instanceof SharedLibraryBinarySpec) { - task.dependsOn binary.buildTask + task.dependsOn binary.tasks.link + def hashFile = new File(binary.sharedLibraryFile.parentFile.absolutePath, "${binary.component.baseName}.hash") + task.outputs.file(hashFile) + task.inputs.file(binary.sharedLibraryFile) + task.from(hashFile) { + into getPlatformPath(binary) + } + task.doFirst { + hashFile.text = MessageDigest.getInstance("MD5").digest(binary.sharedLibraryFile.bytes).encodeHex().toString() + } task.from(binary.sharedLibraryFile) { into getPlatformPath(binary) } diff --git a/wpiutil/src/main/java/edu/wpi/first/wpiutil/RuntimeDetector.java b/wpiutil/src/main/java/edu/wpi/first/wpiutil/RuntimeDetector.java index de4fd8bdfc..d5ddfe998d 100644 --- a/wpiutil/src/main/java/edu/wpi/first/wpiutil/RuntimeDetector.java +++ b/wpiutil/src/main/java/edu/wpi/first/wpiutil/RuntimeDetector.java @@ -94,6 +94,16 @@ public final class RuntimeDetector { return toReturn; } + /** + * Get the path to the hash to the requested resource. + */ + public static synchronized String getHashLibraryResource(String libName) { + computePlatform(); + + String toReturn = filePath + libName + ".hash"; + return toReturn; + } + public static boolean isAthena() { File runRobotFile = new File("/usr/local/frc/bin/frcRunRobot.sh"); return runRobotFile.exists(); diff --git a/wpiutil/src/main/java/edu/wpi/first/wpiutil/RuntimeLoader.java b/wpiutil/src/main/java/edu/wpi/first/wpiutil/RuntimeLoader.java new file mode 100644 index 0000000000..85b0f5792c --- /dev/null +++ b/wpiutil/src/main/java/edu/wpi/first/wpiutil/RuntimeLoader.java @@ -0,0 +1,161 @@ +/*----------------------------------------------------------------------------*/ +/* Copyright (c) 2018 FIRST. All Rights Reserved. */ +/* Open Source Software - may be modified and shared by FRC teams. The code */ +/* must be accompanied by the FIRST BSD license file in the root directory of */ +/* the project. */ +/*----------------------------------------------------------------------------*/ + +package edu.wpi.first.wpiutil; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.DigestInputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Locale; +import java.util.Scanner; + +public final class RuntimeLoader { + private static String defaultExtractionRoot; + + /** + * Gets the default extration root location (~/.wpilib/nativecache). + */ + public static synchronized String getDefaultExtractionRoot() { + if (defaultExtractionRoot != null) { + return defaultExtractionRoot; + } + String home = System.getProperty("user.home"); + defaultExtractionRoot = Paths.get(home, ".wpilib", "nativecache").toString(); + return defaultExtractionRoot; + } + + private final String m_libraryName; + private final Class m_loadClass; + private final String m_extractionRoot; + + /** + * Creates a new library loader. + * + *

Resources loaded on disk from extractionRoot, and from classpath from the + * passed in class. Library name is the passed in name. + */ + public RuntimeLoader(String libraryName, String extractionRoot, Class cls) { + m_libraryName = libraryName; + m_loadClass = cls; + m_extractionRoot = extractionRoot; + } + + /** + * Loads a native library. + */ + @SuppressWarnings("PMD.PreserveStackTrace") + public void loadLibrary() throws IOException { + try { + // First, try loading path + System.loadLibrary(m_libraryName); + return; + } catch (UnsatisfiedLinkError ule) { + // Then load the hash from the resources + String hashName = RuntimeDetector.getHashLibraryResource(m_libraryName); + String resname = RuntimeDetector.getLibraryResource(m_libraryName); + try (InputStream hashIs = m_loadClass.getResourceAsStream(hashName)) { + if (hashIs == null) { + throw new IOException(hashName + " Resource not found"); + } + try (Scanner scanner = new Scanner(hashIs)) { + String hash = scanner.nextLine(); + File jniLibrary = new File(m_extractionRoot, resname + "." + hash); + try { + // Try to load from an already extracted hash + System.load(jniLibrary.getAbsolutePath()); + } catch (UnsatisfiedLinkError ule2) { + // If extraction failed, extract + try (InputStream resIs = m_loadClass.getResourceAsStream(resname)) { + if (resIs == null) { + throw new IOException(resname + " Resource not found"); + } + jniLibrary.getParentFile().mkdirs(); + try (OutputStream os = Files.newOutputStream(jniLibrary.toPath())) { + byte[] buffer = new byte[0xFFFF]; // 64K copy buffer + int readBytes; + while ((readBytes = resIs.read(buffer)) != -1) { // NOPMD + os.write(buffer, 0, readBytes); + } + } + System.load(jniLibrary.getAbsolutePath()); + } + } + } + } + } + } + + /** + * Load a native library by directly hashing the file. + */ + @SuppressWarnings({"PMD.NPathComplexity", "PMD.PreserveStackTrace", "PMD.EmptyWhileStmt", + "PMD.AvoidThrowingRawExceptionTypes", "PMD.CyclomaticComplexity"}) + public void loadLibraryHashed() throws IOException { + try { + // First, try loading path + System.loadLibrary(m_libraryName); + return; + } catch (UnsatisfiedLinkError ule) { + // Then load the hash from the input file + String resname = RuntimeDetector.getLibraryResource(m_libraryName); + String hash = null; + try (InputStream is = m_loadClass.getResourceAsStream(resname)) { + if (is == null) { + throw new IOException(resname + " Resource not found"); + } + MessageDigest md = null; + try { + md = MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException nsae) { + throw new RuntimeException("Weird Hash Algorithm?"); + } + try (DigestInputStream dis = new DigestInputStream(is, md)) { + // Read the entire buffer once to hash + byte[] buffer = new byte[0xFFFF]; + while (dis.read(buffer) > -1) {} + MessageDigest digest = dis.getMessageDigest(); + byte[] digestOutput = digest.digest(); + StringBuilder builder = new StringBuilder(); + for (byte b : digestOutput) { + builder.append(String.format("%02X", b)); + } + hash = builder.toString().toLowerCase(Locale.ENGLISH); + } + } + if (hash == null) { + throw new IOException("Weird Hash?"); + } + File jniLibrary = new File(m_extractionRoot, resname + "." + hash); + try { + // Try to load from an already extracted hash + System.load(jniLibrary.getAbsolutePath()); + } catch (UnsatisfiedLinkError ule2) { + // If extraction failed, extract + try (InputStream resIs = m_loadClass.getResourceAsStream(resname)) { + if (resIs == null) { + throw new IOException(resname + " Resource not found"); + } + jniLibrary.getParentFile().mkdirs(); + try (OutputStream os = Files.newOutputStream(jniLibrary.toPath())) { + byte[] buffer = new byte[0xFFFF]; // 64K copy buffer + int readBytes; + while ((readBytes = resIs.read(buffer)) != -1) { // NOPMD + os.write(buffer, 0, readBytes); + } + } + System.load(jniLibrary.getAbsolutePath()); + } + } + } + } +}