[datalogtool] Add datalogtool

This is a support tool for datalog file conversion (and eventually
download/remote datalog file management).

CMakeLists.txt

@@ -262,6 +262,7 @@ if (WITH_GUI)
     add_subdirectory(outlineviewer)
     if (LIBSSH_FOUND)
         add_subdirectory(roborioteamnumbersetter)
+        add_subdirectory(datalogtool)
     endif()
 endif()
 

datalogtool/.styleguide

@@ -0,0 +1,29 @@
+cppHeaderFileInclude {
+  \.h$
+  \.inc$
+  \.inl$
+}
+
+cppSrcFileInclude {
+  \.cpp$
+}
+
+generatedFileExclude {
+  src/main/native/resources/
+  src/main/native/win/datalogtool.ico
+  src/main/native/mac/datalogtool.icns
+}
+
+repoRootNameOverride {
+  datalogtool
+}
+
+includeOtherLibs {
+  ^GLFW
+  ^fmt/
+  ^glass/
+  ^imgui
+  ^portable-file-dialog
+  ^wpi/
+  ^wpigui
+}

datalogtool/CMakeLists.txt

@@ -0,0 +1,29 @@
+project(datalogtool)
+
+include(CompileWarnings)
+include(GenResources)
+include(LinkMacOSGUI)
+
+configure_file(src/main/generate/WPILibVersion.cpp.in WPILibVersion.cpp)
+GENERATE_RESOURCES(src/main/native/resources generated/main/cpp DLT dlt datalogtool_resources_src)
+
+file(GLOB datalogtool_src src/main/native/cpp/*.cpp ${CMAKE_CURRENT_BINARY_DIR}/WPILibVersion.cpp)
+
+if (WIN32)
+    set(datalogtool_rc src/main/native/win/datalogtool.rc)
+elseif(APPLE)
+    set(MACOSX_BUNDLE_ICON_FILE datalogtool.icns)
+    set(APP_ICON_MACOSX src/main/native/mac/datalogtool.icns)
+    set_source_files_properties(${APP_ICON_MACOSX} PROPERTIES MACOSX_PACKAGE_LOCATION "Resources")
+endif()
+
+add_executable(datalogtool ${datalogtool_src} ${datalogtool_resources_src} ${datalogtool_rc} ${APP_ICON_MACOSX})
+wpilib_link_macos_gui(datalogtool)
+target_link_libraries(datalogtool libglass ${LIBSSH_LIBRARIES})
+target_include_directories(datalogtool PRIVATE ${LIBSSH_INCLUDE_DIRS})
+
+if (WIN32)
+    set_target_properties(datalogtool PROPERTIES WIN32_EXECUTABLE YES)
+elseif(APPLE)
+    set_target_properties(datalogtool PROPERTIES MACOSX_BUNDLE YES OUTPUT_NAME "datalogTool")
+endif()

datalogtool/Info.plist

@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>CFBundleName</key>
+	<string>datalogTool</string>
+	<key>CFBundleExecutable</key>
+	<string>datalogtool</string>
+	<key>CFBundleDisplayName</key>
+	<string>datalogTool</string>
+	<key>CFBundleIdentifier</key>
+	<string>edu.wpi.first.tools.datalogTool</string>
+	<key>CFBundleIconFile</key>
+	<string>datalogtool.icns</string>
+	<key>CFBundlePackageType</key>
+	<string>APPL</string>
+	<key>CFBundleSupportedPlatforms</key>
+	<array>
+		<string>MacOSX</string>
+	</array>
+	<key>CFBundleInfoDictionaryVersion</key>
+	<string>6.0</string>
+	<key>CFBundleShortVersionString</key>
+	<string>2021</string>
+	<key>CFBundleVersion</key>
+	<string>2021</string>
+	<key>LSMinimumSystemVersion</key>
+	<string>10.11</string>
+	<key>NSHighResolutionCapable</key>
+	<true/>
+</dict>
+</plist>

datalogtool/build.gradle

@@ -0,0 +1,134 @@
+import org.gradle.internal.os.OperatingSystem
+
+if (!project.hasProperty('onlylinuxathena') && !project.hasProperty('onlylinuxraspbian') && !project.hasProperty('onlylinuxaarch64bionic')) {
+
+    description = "roboRIO Team Number Setter"
+
+    apply plugin: 'cpp'
+    apply plugin: 'c'
+    apply plugin: 'google-test-test-suite'
+    apply plugin: 'visual-studio'
+    apply plugin: 'edu.wpi.first.NativeUtils'
+
+    if (OperatingSystem.current().isWindows()) {
+        apply plugin: 'windows-resources'
+    }
+
+    ext {
+        nativeName = 'datalogtool'
+    }
+
+    apply from: "${rootDir}/shared/resources.gradle"
+    apply from: "${rootDir}/shared/config.gradle"
+
+    def wpilibVersionFileInput = file("src/main/generate/WPILibVersion.cpp.in")
+    def wpilibVersionFileOutput = file("$buildDir/generated/main/cpp/WPILibVersion.cpp")
+
+    nativeUtils {
+        nativeDependencyContainer {
+            libssh(getNativeDependencyTypeClass('WPIStaticMavenDependency')) {
+                groupId = "edu.wpi.first.thirdparty.frc2022"
+                artifactId = "libssh"
+                headerClassifier = "headers"
+                sourceClassifier = "sources"
+                ext = "zip"
+                version = '0.95-1'
+                targetPlatforms.addAll(nativeUtils.wpi.platforms.desktopPlatforms)
+            }
+        }
+    }
+
+    task generateCppVersion() {
+        description = 'Generates the wpilib version class'
+        group = 'WPILib'
+
+        outputs.file wpilibVersionFileOutput
+        inputs.file wpilibVersionFileInput
+
+        if (wpilibVersioning.releaseMode) {
+            outputs.upToDateWhen { false }
+        }
+
+        // We follow a simple set of checks to determine whether we should generate a new version file:
+        // 1. If the release type is not development, we generate a new version file
+        // 2. If there is no generated version number, we generate a new version file
+        // 3. If there is a generated build number, and the release type is development, then we will
+        //    only generate if the publish task is run.
+        doLast {
+            def version = wpilibVersioning.version.get()
+            println "Writing version ${version} to $wpilibVersionFileOutput"
+
+            if (wpilibVersionFileOutput.exists()) {
+                wpilibVersionFileOutput.delete()
+            }
+            def read = wpilibVersionFileInput.text.replace('${wpilib_version}', version)
+            wpilibVersionFileOutput.write(read)
+        }
+    }
+
+    gradle.taskGraph.addTaskExecutionGraphListener { graph ->
+        def willPublish = graph.hasTask(publish)
+        if (willPublish) {
+            generateCppVersion.outputs.upToDateWhen { false }
+        }
+    }
+
+    def generateTask = createGenerateResourcesTask('main', 'DLT', 'dlt', project)
+
+    project(':').libraryBuild.dependsOn build
+    tasks.withType(CppCompile) {
+        dependsOn generateTask
+        dependsOn generateCppVersion
+    }
+
+    model {
+        components {
+            // By default, a development executable will be generated. This is to help the case of
+            // testing specific functionality of the library.
+            "${nativeName}"(NativeExecutableSpec) {
+                baseName = 'datalogtool'
+                sources {
+                    cpp {
+                        source {
+                            srcDirs 'src/main/native/cpp', "$buildDir/generated/main/cpp"
+                            include '**/*.cpp'
+                        }
+                        exportedHeaders {
+                            srcDirs 'src/main/native/include'
+                        }
+                    }
+                    if (OperatingSystem.current().isWindows()) {
+                        rc {
+                            source {
+                                srcDirs 'src/main/native/win'
+                                include '*.rc'
+                            }
+                        }
+                    }
+                }
+                binaries.all {
+                    if (it.targetPlatform.name == nativeUtils.wpi.platforms.roborio || it.targetPlatform.name == nativeUtils.wpi.platforms.raspbian || it.targetPlatform.name == nativeUtils.wpi.platforms.aarch64bionic) {
+                        it.buildable = false
+                        return
+                    }
+                    it.cppCompiler.define("LIBSSH_STATIC")
+                    lib project: ':glass', library: 'glass', linkage: 'static'
+                    lib project: ':wpiutil', library: 'wpiutil', linkage: 'static'
+                    lib project: ':wpigui', library: 'wpigui', linkage: 'static'
+                    nativeUtils.useRequiredLibrary(it, 'imgui_static', 'libssh')
+                    if (it.targetPlatform.operatingSystem.isWindows()) {
+                        it.linker.args << 'Gdi32.lib' << 'Shell32.lib' << 'd3d11.lib' << 'd3dcompiler.lib'
+                        it.linker.args << 'ws2_32.lib' << 'advapi32.lib' << 'crypt32.lib' << 'user32.lib'
+                    } else if (it.targetPlatform.operatingSystem.isMacOsX()) {
+                        it.linker.args << '-framework' << 'Metal' << '-framework' << 'MetalKit' << '-framework' << 'Cocoa' << '-framework' << 'IOKit' << '-framework' << 'CoreFoundation' << '-framework' << 'CoreVideo' << '-framework' << 'QuartzCore'
+                        it.linker.args << '-framework' << 'Kerberos'
+                    } else {
+                        it.linker.args << '-lX11'
+                    }
+                }
+            }
+        }
+    }
+
+    apply from: 'publish.gradle'
+}

datalogtool/publish.gradle

@@ -0,0 +1,107 @@
+apply plugin: 'maven-publish'
+
+def baseArtifactId = 'DataLogTool'
+def artifactGroupId = 'edu.wpi.first.tools'
+def zipBaseName = '_GROUP_edu_wpi_first_tools_ID_DataLogTool_CLS'
+
+def outputsFolder = file("$project.buildDir/outputs")
+
+model {
+    tasks {
+        // Create the run task.
+        $.components.datalogtool.binaries.each { bin ->
+            if (bin.buildable && bin.name.toLowerCase().contains("debug")) {
+                Task run = project.tasks.create("run", Exec) {
+                    commandLine bin.tasks.install.runScriptFile.get().asFile.toString()
+                }
+                run.dependsOn bin.tasks.install
+            }
+        }
+    }
+    publishing {
+        def dataLogToolTaskList = []
+        $.components.each { component ->
+            component.binaries.each { binary ->
+                if (binary in NativeExecutableBinarySpec && binary.component.name.contains("datalogtool")) {
+                    if (binary.buildable && binary.name.contains("Release")) {
+                        // We are now in the binary that we want.
+                        // This is the default application path for the ZIP task.
+                        def applicationPath = binary.executable.file
+                        def icon = file("$project.projectDir/src/main/native/mac/datalogtool.icns")
+
+                        // Create the macOS bundle.
+                        def bundleTask = project.tasks.create("bundleDataLogToolOsxApp", Copy) {
+                            description("Creates a macOS application bundle for DataLogTool")
+                            from(file("$project.projectDir/Info.plist"))
+                            into(file("$project.buildDir/outputs/bundles/DataLogTool.app/Contents"))
+                            into("MacOS") { with copySpec { from binary.executable.file } }
+                            into("Resources") { with copySpec { from icon } }
+
+                            doLast {
+                                if (project.hasProperty("developerID")) {
+                                    // Get path to binary.
+                                    exec {
+                                        workingDir rootDir
+                                        def args = [
+                                            "sh",
+                                            "-c",
+                                            "codesign --force --strict --deep " +
+                                            "--timestamp --options=runtime " +
+                                            "--verbose -s ${project.findProperty("developerID")} " +
+                                            "$project.buildDir/outputs/bundles/DataLogTool.app/"
+                                        ]
+                                        commandLine args
+                                    }
+                                }
+                            }
+                        }
+
+                        // Reset the application path if we are creating a bundle.
+                        if (binary.targetPlatform.operatingSystem.isMacOsX()) {
+                            applicationPath = file("$project.buildDir/outputs/bundles")
+                            project.build.dependsOn bundleTask
+                        }
+
+                        // Create the ZIP.
+                        def task = project.tasks.create("copyDataLogToolExecutable", Zip) {
+                            description("Copies the DataLogTool executable to the outputs directory.")
+                            destinationDirectory = outputsFolder
+
+                            archiveBaseName = '_M_' + zipBaseName
+                            duplicatesStrategy = 'exclude'
+                            classifier = nativeUtils.getPublishClassifier(binary)
+
+                            from(licenseFile) {
+                                into '/'
+                            }
+
+                            from(applicationPath)
+                            into(nativeUtils.getPlatformPath(binary))
+                        }
+
+                        if (binary.targetPlatform.operatingSystem.isMacOsX()) {
+                            bundleTask.dependsOn binary.tasks.link
+                            task.dependsOn(bundleTask)
+                        }
+
+                        task.dependsOn binary.tasks.link
+                        dataLogToolTaskList.add(task)
+                        project.build.dependsOn task
+                        project.artifacts { task }
+                        addTaskToCopyAllOutputs(task)
+                    }
+                }
+            }
+        }
+
+        publications {
+            datalogtool(MavenPublication) {
+                dataLogToolTaskList.each { artifact it }
+
+                artifactId = baseArtifactId
+                groupId = artifactGroupId
+                version wpilibVersioning.version.get()
+            }
+        }
+    }
+}

datalogtool/src/main/generate/WPILibVersion.cpp.in

@@ -0,0 +1,7 @@
+/*
+ * Autogenerated file! Do not manually edit this file. This version is regenerated
+ * any time the publish task is run, or when this file is deleted.
+ */
+const char* GetWPILibVersion() {
+  return "${wpilib_version}";
+}

datalogtool/src/main/native/cpp/App.cpp

@@ -0,0 +1,156 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#include "App.h"
+
+#include <libssh/libssh.h>
+
+#include <memory>
+#include <string_view>
+
+#define IMGUI_DEFINE_MATH_OPERATORS
+
+#include <glass/Context.h>
+#include <glass/MainMenuBar.h>
+#include <glass/Storage.h>
+#include <imgui.h>
+#include <imgui_internal.h>
+#include <wpigui.h>
+
+#include "Downloader.h"
+#include "Exporter.h"
+
+namespace gui = wpi::gui;
+
+const char* GetWPILibVersion();
+
+namespace dlt {
+std::string_view GetResource_dlt_16_png();
+std::string_view GetResource_dlt_32_png();
+std::string_view GetResource_dlt_48_png();
+std::string_view GetResource_dlt_64_png();
+std::string_view GetResource_dlt_128_png();
+std::string_view GetResource_dlt_256_png();
+std::string_view GetResource_dlt_512_png();
+}  // namespace dlt
+
+bool gShutdown = false;
+
+static std::unique_ptr<Downloader> gDownloader;
+static bool* gDownloadVisible;
+static float gDefaultScale = 1.0;
+
+void SetNextWindowPos(const ImVec2& pos, ImGuiCond cond, const ImVec2& pivot) {
+  if ((cond & ImGuiCond_FirstUseEver) != 0) {
+    ImGui::SetNextWindowPos(pos * gDefaultScale, cond, pivot);
+  } else {
+    ImGui::SetNextWindowPos(pos, cond, pivot);
+  }
+}
+
+void SetNextWindowSize(const ImVec2& size, ImGuiCond cond) {
+  if ((cond & ImGuiCond_FirstUseEver) != 0) {
+    ImGui::SetNextWindowSize(size * gDefaultScale, cond);
+  } else {
+    ImGui::SetNextWindowPos(size, cond);
+  }
+}
+
+static void DisplayDownload() {
+  if (!*gDownloadVisible) {
+    return;
+  }
+  SetNextWindowPos(ImVec2{0, 250}, ImGuiCond_FirstUseEver);
+  SetNextWindowSize(ImVec2{375, 260}, ImGuiCond_FirstUseEver);
+  if (ImGui::Begin("Download", gDownloadVisible)) {
+    if (!gDownloader) {
+      gDownloader = std::make_unique<Downloader>(
+          glass::GetStorageRoot().GetChild("download"));
+    }
+    gDownloader->Display();
+  }
+  ImGui::End();
+}
+
+static void DisplayMainMenu() {
+  ImGui::BeginMainMenuBar();
+
+  static glass::MainMenuBar mainMenu;
+  mainMenu.WorkspaceMenu();
+  gui::EmitViewMenu();
+
+  if (ImGui::BeginMenu("Window")) {
+    ImGui::MenuItem("Download", nullptr, gDownloadVisible);
+    ImGui::EndMenu();
+  }
+
+  bool about = false;
+  if (ImGui::BeginMenu("Info")) {
+    if (ImGui::MenuItem("About")) {
+      about = true;
+    }
+    ImGui::EndMenu();
+  }
+
+  ImGui::EndMainMenuBar();
+
+  if (about) {
+    ImGui::OpenPopup("About");
+  }
+  if (ImGui::BeginPopupModal("About")) {
+    ImGui::Text("Datalog Tool");
+    ImGui::Separator();
+    ImGui::Text("v%s", GetWPILibVersion());
+    ImGui::Separator();
+    ImGui::Text("Save location: %s", glass::GetStorageDir().c_str());
+    if (ImGui::Button("Close")) {
+      ImGui::CloseCurrentPopup();
+    }
+    ImGui::EndPopup();
+  }
+}
+
+static void DisplayGui() {
+  DisplayMainMenu();
+  DisplayInputFiles();
+  DisplayEntries();
+  DisplayOutput(glass::GetStorageRoot().GetChild("output"));
+  DisplayDownload();
+}
+
+void Application(std::string_view saveDir) {
+  ssh_init();
+
+  gui::CreateContext();
+  glass::CreateContext();
+
+  // Add icons
+  gui::AddIcon(dlt::GetResource_dlt_16_png());
+  gui::AddIcon(dlt::GetResource_dlt_32_png());
+  gui::AddIcon(dlt::GetResource_dlt_48_png());
+  gui::AddIcon(dlt::GetResource_dlt_64_png());
+  gui::AddIcon(dlt::GetResource_dlt_128_png());
+  gui::AddIcon(dlt::GetResource_dlt_256_png());
+  gui::AddIcon(dlt::GetResource_dlt_512_png());
+
+  glass::SetStorageName("datalogtool");
+  glass::SetStorageDir(saveDir.empty() ? gui::GetPlatformSaveFileDir()
+                                       : saveDir);
+
+  gui::AddWindowScaler([](float scale) { gDefaultScale = scale; });
+  gui::AddLateExecute(DisplayGui);
+  gui::Initialize("Datalog Tool", 925, 510);
+
+  gDownloadVisible =
+      &glass::GetStorageRoot().GetChild("download").GetBool("visible", true);
+
+  gui::Main();
+
+  gShutdown = true;
+  glass::DestroyContext();
+  gui::DestroyContext();
+
+  gDownloader.reset();
+  ssh_finalize();
+}

datalogtool/src/main/native/cpp/App.h

@@ -0,0 +1,11 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#pragma once
+
+#include <imgui.h>
+
+void SetNextWindowPos(const ImVec2& pos, ImGuiCond cond = 0,
+                      const ImVec2& pivot = ImVec2(0, 0));
+void SetNextWindowSize(const ImVec2& size, ImGuiCond cond = 0);

datalogtool/src/main/native/cpp/DataLogThread.cpp

@@ -0,0 +1,72 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#include "DataLogThread.h"
+
+#include <fmt/format.h>
+
+DataLogThread::~DataLogThread() {
+  if (m_thread.joinable()) {
+    m_active = false;
+    m_thread.join();
+  }
+}
+
+void DataLogThread::ReadMain() {
+  for (auto record : m_reader) {
+    if (!m_active) {
+      break;
+    }
+    ++m_numRecords;
+    if (record.IsStart()) {
+      wpi::log::StartRecordData data;
+      if (record.GetStartData(&data)) {
+        std::scoped_lock lock{m_mutex};
+        if (m_entries.find(data.entry) != m_entries.end()) {
+          fmt::print("...DUPLICATE entry ID, overriding\n");
+        }
+        m_entries[data.entry] = data;
+        m_entryNames.emplace(data.name, data);
+        sigEntryAdded(data);
+      } else {
+        fmt::print("Start(INVALID)\n");
+      }
+    } else if (record.IsFinish()) {
+      int entry;
+      if (record.GetFinishEntry(&entry)) {
+        std::scoped_lock lock{m_mutex};
+        auto it = m_entries.find(entry);
+        if (it == m_entries.end()) {
+          fmt::print("...ID not found\n");
+        } else {
+          m_entries.erase(it);
+        }
+      } else {
+        fmt::print("Finish(INVALID)\n");
+      }
+    } else if (record.IsSetMetadata()) {
+      wpi::log::MetadataRecordData data;
+      if (record.GetSetMetadataData(&data)) {
+        std::scoped_lock lock{m_mutex};
+        auto it = m_entries.find(data.entry);
+        if (it == m_entries.end()) {
+          fmt::print("...ID not found\n");
+        } else {
+          it->second.metadata = data.metadata;
+          auto nameIt = m_entryNames.find(it->second.name);
+          if (nameIt != m_entryNames.end()) {
+            nameIt->second.metadata = data.metadata;
+          }
+        }
+      } else {
+        fmt::print("SetMetadata(INVALID)\n");
+      }
+    } else if (record.IsControl()) {
+      fmt::print("Unrecognized control record\n");
+    }
+  }
+
+  sigDone();
+  m_done = true;
+}

datalogtool/src/main/native/cpp/DataLogThread.h

@@ -0,0 +1,71 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#pragma once
+
+#include <atomic>
+#include <functional>
+#include <map>
+#include <string>
+#include <string_view>
+#include <thread>
+#include <utility>
+
+#include <wpi/DataLogReader.h>
+#include <wpi/DenseMap.h>
+#include <wpi/Signal.h>
+#include <wpi/mutex.h>
+
+class DataLogThread {
+ public:
+  explicit DataLogThread(wpi::log::DataLogReader reader)
+      : m_reader{std::move(reader)}, m_thread{[=] { ReadMain(); }} {}
+  ~DataLogThread();
+
+  bool IsDone() const { return m_done; }
+  std::string_view GetBufferIdentifier() const {
+    return m_reader.GetBufferIdentifier();
+  }
+  unsigned int GetNumRecords() const { return m_numRecords; }
+  unsigned int GetNumEntries() const {
+    std::scoped_lock lock{m_mutex};
+    return m_entryNames.size();
+  }
+
+  // Passes wpi::log::StartRecordData to func
+  template <typename T>
+  void ForEachEntryName(T&& func) {
+    std::scoped_lock lock{m_mutex};
+    for (auto&& kv : m_entryNames) {
+      func(kv.second);
+    }
+  }
+
+  wpi::log::StartRecordData GetEntry(std::string_view name) const {
+    std::scoped_lock lock{m_mutex};
+    auto it = m_entryNames.find(name);
+    if (it == m_entryNames.end()) {
+      return {};
+    }
+    return it->second;
+  }
+
+  const wpi::log::DataLogReader& GetReader() const { return m_reader; }
+
+  // note: these are called on separate thread
+  wpi::sig::Signal_mt<const wpi::log::StartRecordData&> sigEntryAdded;
+  wpi::sig::Signal_mt<> sigDone;
+
+ private:
+  void ReadMain();
+
+  wpi::log::DataLogReader m_reader;
+  mutable wpi::mutex m_mutex;
+  std::atomic_bool m_active{true};
+  std::atomic_bool m_done{false};
+  std::atomic<unsigned int> m_numRecords{0};
+  std::map<std::string, wpi::log::StartRecordData, std::less<>> m_entryNames;
+  wpi::DenseMap<int, wpi::log::StartRecordData> m_entries;
+  std::thread m_thread;
+};

datalogtool/src/main/native/cpp/Downloader.cpp

@@ -0,0 +1,393 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#include "Downloader.h"
+
+#include <libssh/sftp.h>
+
+#ifdef _WIN32
+#include <fcntl.h>
+#include <io.h>
+#else
+#include <sys/fcntl.h>
+#endif
+
+#include <algorithm>
+#include <filesystem>
+
+#include <fmt/format.h>
+#include <glass/Storage.h>
+#include <imgui.h>
+#include <imgui_stdlib.h>
+#include <portable-file-dialogs.h>
+#include <wpi/StringExtras.h>
+#include <wpi/fs.h>
+
+#include "Sftp.h"
+
+Downloader::Downloader(glass::Storage& storage)
+    : m_serverTeam{storage.GetString("serverTeam")},
+      m_remoteDir{storage.GetString("remoteDir", "/home/lvuser")},
+      m_username{storage.GetString("username", "lvuser")},
+      m_localDir{storage.GetString("localDir")},
+      m_deleteAfter{storage.GetBool("deleteAfter", true)},
+      m_thread{[this] { ThreadMain(); }} {}
+
+Downloader::~Downloader() {
+  {
+    std::scoped_lock lock{m_mutex};
+    m_state = kExit;
+  }
+  m_cv.notify_all();
+  m_thread.join();
+}
+
+void Downloader::DisplayConnect() {
+  // IP or Team Number text box
+  ImGui::SetNextItemWidth(ImGui::GetFontSize() * 12);
+  ImGui::InputText("Team Number / Address", &m_serverTeam);
+
+  // Username/password
+  ImGui::SetNextItemWidth(ImGui::GetFontSize() * 12);
+  ImGui::InputText("Username", &m_username);
+  ImGui::SetNextItemWidth(ImGui::GetFontSize() * 12);
+  ImGui::InputText("Password", &m_password, ImGuiInputTextFlags_Password);
+
+  // Connect button
+  if (ImGui::Button("Connect")) {
+    m_state = kConnecting;
+    m_cv.notify_all();
+  }
+}
+
+void Downloader::DisplayDisconnectButton() {
+  if (ImGui::Button("Disconnect")) {
+    m_state = kDisconnecting;
+    m_cv.notify_all();
+  }
+}
+
+void Downloader::DisplayRemoteDirSelector() {
+  ImGui::SameLine();
+  if (ImGui::Button("Refresh")) {
+    m_state = kGetFiles;
+    m_cv.notify_all();
+  }
+
+  // Remote directory text box
+  ImGui::SetNextItemWidth(ImGui::GetFontSize() * 20);
+  if (ImGui::InputText("Remote Dir", &m_remoteDir,
+                       ImGuiInputTextFlags_EnterReturnsTrue)) {
+    m_state = kGetFiles;
+    m_cv.notify_all();
+  }
+
+  // List directories
+  for (auto&& dir : m_dirList) {
+    if (ImGui::Selectable(dir.c_str())) {
+      if (dir == "..") {
+        if (wpi::ends_with(m_remoteDir, '/')) {
+          m_remoteDir.resize(m_remoteDir.size() - 1);
+        }
+        m_remoteDir = wpi::rsplit(m_remoteDir, '/').first;
+        if (m_remoteDir.empty()) {
+          m_remoteDir = "/";
+        }
+      } else {
+        if (!wpi::ends_with(m_remoteDir, '/')) {
+          m_remoteDir += '/';
+        }
+        m_remoteDir += dir;
+      }
+      m_state = kGetFiles;
+      m_cv.notify_all();
+    }
+  }
+}
+
+void Downloader::DisplayLocalDirSelector() {
+  // Local directory text / select button
+  if (ImGui::Button("Select Download Folder...")) {
+    m_localDirSelector =
+        std::make_unique<pfd::select_folder>("Select Download Folder");
+  }
+  ImGui::TextUnformatted(m_localDir.c_str());
+
+  // Delete after download (checkbox)
+  ImGui::Checkbox("Delete after download", &m_deleteAfter);
+
+  // Download button
+  if (!m_localDir.empty()) {
+    if (ImGui::Button("Download")) {
+      m_state = kDownload;
+      m_cv.notify_all();
+    }
+  }
+}
+
+size_t Downloader::DisplayFiles() {
+  // List of files (multi-select) (changes to progress bar for downloading)
+  size_t fileCount = 0;
+  if (ImGui::BeginTable(
+          "files", 3,
+          ImGuiTableFlags_Borders | ImGuiTableFlags_SizingStretchProp)) {
+    ImGui::TableSetupColumn("File");
+    ImGui::TableSetupColumn("Size");
+    ImGui::TableSetupColumn("Download");
+    ImGui::TableHeadersRow();
+    for (auto&& download : m_downloadList) {
+      if ((m_state == kDownload || m_state == kDownloadDone) &&
+          !download.enabled) {
+        continue;
+      }
+
+      ++fileCount;
+
+      ImGui::TableNextRow();
+      ImGui::TableNextColumn();
+      ImGui::TextUnformatted(download.name.c_str());
+      ImGui::TableNextColumn();
+      auto sizeText = fmt::format("{}", download.size);
+      ImGui::TextUnformatted(sizeText.c_str());
+      ImGui::TableNextColumn();
+      if (m_state == kDownload || m_state == kDownloadDone) {
+        if (!download.status.empty()) {
+          ImGui::TextUnformatted(download.status.c_str());
+        } else {
+          ImGui::ProgressBar(download.complete);
+        }
+      } else {
+        auto checkboxLabel = fmt::format("##{}", download.name);
+        ImGui::Checkbox(checkboxLabel.c_str(), &download.enabled);
+      }
+    }
+    ImGui::EndTable();
+  }
+
+  return fileCount;
+}
+
+void Downloader::Display() {
+  if (m_localDirSelector && m_localDirSelector->ready(0)) {
+    m_localDir = m_localDirSelector->result();
+    m_localDirSelector.reset();
+  }
+
+  std::scoped_lock lock{m_mutex};
+
+  if (!m_error.empty()) {
+    ImGui::TextUnformatted(m_error.c_str());
+  }
+
+  switch (m_state) {
+    case kDisconnected:
+      DisplayConnect();
+      break;
+    case kConnecting:
+      DisplayDisconnectButton();
+      ImGui::Text("Connecting to %s...", m_serverTeam.c_str());
+      break;
+    case kDisconnecting:
+      ImGui::TextUnformatted("Disconnecting...");
+      break;
+    case kConnected:
+    case kGetFiles:
+      DisplayDisconnectButton();
+      DisplayRemoteDirSelector();
+      if (DisplayFiles() > 0) {
+        DisplayLocalDirSelector();
+      }
+      break;
+    case kDownload:
+    case kDownloadDone:
+      DisplayDisconnectButton();
+      DisplayFiles();
+      if (m_state == kDownloadDone) {
+        if (ImGui::Button("Download complete!")) {
+          m_state = kGetFiles;
+          m_cv.notify_all();
+        }
+      }
+      break;
+    default:
+      break;
+  }
+}
+
+void Downloader::ThreadMain() {
+  std::unique_ptr<sftp::Session> session;
+
+  static constexpr size_t kBufSize = 32 * 1024;
+  std::unique_ptr<uint8_t[]> copyBuf = std::make_unique<uint8_t[]>(kBufSize);
+
+  std::unique_lock lock{m_mutex};
+  while (m_state != kExit) {
+    State prev = m_state;
+    m_cv.wait(lock, [&] { return m_state != prev; });
+    m_error.clear();
+    try {
+      switch (m_state) {
+        case kConnecting:
+          if (auto team = wpi::parse_integer<unsigned int>(m_serverTeam, 10)) {
+            // team number
+            session = std::make_unique<sftp::Session>(
+                fmt::format("roborio-{}-frc.local", team.value()), 22,
+                m_username, m_password);
+          } else {
+            session = std::make_unique<sftp::Session>(m_serverTeam, 22,
+                                                      m_username, m_password);
+          }
+          lock.unlock();
+          try {
+            session->Connect();
+          } catch (...) {
+            lock.lock();
+            throw;
+          }
+          lock.lock();
+          // FALLTHROUGH
+        case kGetFiles: {
+          std::string dir = m_remoteDir;
+          std::vector<sftp::Attributes> fileList;
+          lock.unlock();
+          try {
+            fileList = session->ReadDir(dir);
+          } catch (sftp::Exception& ex) {
+            lock.lock();
+            if (ex.err == SSH_FX_OK || ex.err == SSH_FX_CONNECTION_LOST) {
+              throw;
+            }
+            m_error = ex.what();
+            m_dirList.clear();
+            m_downloadList.clear();
+            m_state = kConnected;
+            break;
+          }
+          std::sort(
+              fileList.begin(), fileList.end(),
+              [](const auto& l, const auto& r) { return l.name < r.name; });
+          lock.lock();
+
+          m_dirList.clear();
+          m_downloadList.clear();
+          for (auto&& attr : fileList) {
+            if (attr.type == SSH_FILEXFER_TYPE_DIRECTORY) {
+              if (attr.name != ".") {
+                m_dirList.emplace_back(attr.name);
+              }
+            } else if (attr.type == SSH_FILEXFER_TYPE_REGULAR &&
+                       (attr.flags & SSH_FILEXFER_ATTR_SIZE) != 0 &&
+                       wpi::ends_with(attr.name, ".wpilog")) {
+              m_downloadList.emplace_back(attr.name, attr.size);
+            }
+          }
+
+          m_state = kConnected;
+          break;
+        }
+        case kDisconnecting:
+          session.reset();
+          m_state = kDisconnected;
+          break;
+        case kDownload: {
+          for (auto&& download : m_downloadList) {
+            if (m_state != kDownload) {
+              // user aborted
+              break;
+            }
+            if (!download.enabled) {
+              continue;
+            }
+
+            auto remoteFilename = fmt::format(
+                "{}{}{}", m_remoteDir,
+                wpi::ends_with(m_remoteDir, '/') ? "" : "/", download.name);
+            auto localFilename = fs::path{m_localDir} / download.name;
+            uint64_t fileSize = download.size;
+
+            lock.unlock();
+
+            // open local file
+            std::error_code ec;
+            fs::file_t of = fs::OpenFileForWrite(localFilename, ec,
+                                                 fs::CD_CreateNew, fs::OF_None);
+            if (ec) {
+              // failed to open
+              lock.lock();
+              download.status = ec.message();
+              continue;
+            }
+            int ofd = fs::FileToFd(of, ec, fs::OF_None);
+            if (ofd == -1 || ec) {
+              // failed to convert to fd
+              lock.lock();
+              download.status = ec.message();
+              continue;
+            }
+
+            try {
+              // open remote file
+              sftp::File f = session->Open(remoteFilename, O_RDONLY, 0);
+
+              // copy in chunks
+              uint64_t total = 0;
+              while (total < fileSize) {
+                uint64_t toCopy = (std::min)(fileSize - total,
+                                             static_cast<uint64_t>(kBufSize));
+                auto copied = f.Read(copyBuf.get(), toCopy);
+                if (write(ofd, copyBuf.get(), copied) !=
+                    static_cast<int64_t>(copied)) {
+                  // error writing
+                  close(ofd);
+                  fs::remove(localFilename, ec);
+                  lock.lock();
+                  download.status = "error writing local file";
+                  goto err;
+                }
+                total += copied;
+                lock.lock();
+                download.complete = static_cast<float>(total) / fileSize;
+                lock.unlock();
+              }
+
+              // close local file
+              close(ofd);
+              ofd = -1;
+
+              // delete remote file (if enabled)
+              if (m_deleteAfter) {
+                f = sftp::File{};
+                session->Unlink(remoteFilename);
+              }
+            } catch (sftp::Exception& ex) {
+              if (ofd != -1) {
+                // close local file and delete it (due to failure)
+                close(ofd);
+                fs::remove(localFilename, ec);
+              }
+              lock.lock();
+              download.status = ex.what();
+              if (ex.err == SSH_FX_OK || ex.err == SSH_FX_CONNECTION_LOST) {
+                throw;
+              }
+              continue;
+            }
+            lock.lock();
+          err : {}
+          }
+          if (m_state == kDownload) {
+            m_state = kDownloadDone;
+          }
+          break;
+        }
+        default:
+          break;
+      }
+    } catch (sftp::Exception& ex) {
+      m_error = ex.what();
+      session.reset();
+      m_state = kDisconnected;
+    }
+  }
+}

datalogtool/src/main/native/cpp/Downloader.h

@@ -0,0 +1,78 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include <wpi/condition_variable.h>
+#include <wpi/mutex.h>
+
+namespace glass {
+class Storage;
+}  // namespace glass
+
+namespace pfd {
+class select_folder;
+}  // namespace pfd
+
+class Downloader {
+ public:
+  explicit Downloader(glass::Storage& storage);
+  ~Downloader();
+
+  void Display();
+
+ private:
+  void DisplayConnect();
+  void DisplayDisconnectButton();
+  void DisplayRemoteDirSelector();
+  void DisplayLocalDirSelector();
+  size_t DisplayFiles();
+
+  void ThreadMain();
+
+  wpi::mutex m_mutex;
+  enum State {
+    kDisconnected,
+    kConnecting,
+    kConnected,
+    kDisconnecting,
+    kGetFiles,
+    kDownload,
+    kDownloadDone,
+    kExit
+  } m_state = kDisconnected;
+  std::condition_variable m_cv;
+
+  std::string& m_serverTeam;
+  std::string& m_remoteDir;
+  std::string& m_username;
+  std::string m_password;
+
+  std::string& m_localDir;
+  std::unique_ptr<pfd::select_folder> m_localDirSelector;
+
+  bool& m_deleteAfter;
+
+  std::vector<std::string> m_dirList;
+  struct DownloadState {
+    DownloadState(std::string_view name, uint64_t size)
+        : name{name}, size{size} {}
+
+    std::string name;
+    uint64_t size;
+    bool enabled = true;
+    float complete = 0.0;
+    std::string status;
+  };
+  std::vector<DownloadState> m_downloadList;
+
+  std::string m_error;
+
+  std::thread m_thread;
+};

datalogtool/src/main/native/cpp/Exporter.cpp

@@ -0,0 +1,655 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#include "Exporter.h"
+
+#include <atomic>
+#include <ctime>
+#include <future>
+#include <map>
+#include <memory>
+#include <set>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include <fmt/chrono.h>
+#include <fmt/format.h>
+#include <glass/Storage.h>
+#include <imgui.h>
+#include <imgui_internal.h>
+#include <imgui_stdlib.h>
+#include <portable-file-dialogs.h>
+#include <wpi/DenseMap.h>
+#include <wpi/MemoryBuffer.h>
+#include <wpi/SmallVector.h>
+#include <wpi/SpanExtras.h>
+#include <wpi/StringExtras.h>
+#include <wpi/fmt/raw_ostream.h>
+#include <wpi/fs.h>
+#include <wpi/mutex.h>
+#include <wpi/raw_ostream.h>
+
+#include "App.h"
+#include "DataLogThread.h"
+
+namespace {
+struct InputFile {
+  explicit InputFile(std::unique_ptr<DataLogThread> datalog);
+
+  InputFile(std::string_view filename, std::string_view status)
+      : filename{filename},
+        stem{fs::path{filename}.stem().string()},
+        status{status} {}
+
+  ~InputFile();
+
+  std::string filename;
+  std::string stem;
+  std::unique_ptr<DataLogThread> datalog;
+  std::string status;
+  bool highlight = false;
+};
+
+struct Entry {
+  explicit Entry(const wpi::log::StartRecordData& srd)
+      : name{srd.name}, type{srd.type}, metadata{srd.metadata} {}
+
+  std::string name;
+  std::string type;
+  std::string metadata;
+  std::set<InputFile*> inputFiles;
+  bool typeConflict = false;
+  bool metadataConflict = false;
+  bool selected = true;
+
+  // used only during export
+  int column = -1;
+};
+
+struct EntryTreeNode {
+  explicit EntryTreeNode(std::string_view name) : name{name} {}
+  std::string name;  // name of just this node
+  std::string path;  // full path if entry is nullptr
+  Entry* entry = nullptr;
+  std::vector<EntryTreeNode> children;  // children, sorted by name
+  int selected = 1;
+};
+}  // namespace
+
+static std::map<std::string, std::unique_ptr<InputFile>, std::less<>>
+    gInputFiles;
+static wpi::mutex gEntriesMutex;
+static std::map<std::string, std::unique_ptr<Entry>, std::less<>> gEntries;
+static std::vector<EntryTreeNode> gEntryTree;
+std::atomic_int gExportCount{0};
+
+// must be called with gEntriesMutex held
+static void RebuildEntryTree() {
+  gEntryTree.clear();
+  wpi::SmallVector<std::string_view, 16> parts;
+  for (auto& kv : gEntries) {
+    parts.clear();
+    // split on first : if one is present
+    auto [prefix, mainpart] = wpi::split(kv.first, ':');
+    if (mainpart.empty() || wpi::contains(prefix, '/')) {
+      mainpart = kv.first;
+    } else {
+      parts.emplace_back(prefix);
+    }
+    wpi::split(mainpart, parts, '/', -1, false);
+
+    // ignore a raw "/" key
+    if (parts.empty()) {
+      continue;
+    }
+
+    // get to leaf
+    auto nodes = &gEntryTree;
+    for (auto part : wpi::drop_back(wpi::span{parts.begin(), parts.end()})) {
+      auto it =
+          std::find_if(nodes->begin(), nodes->end(),
+                       [&](const auto& node) { return node.name == part; });
+      if (it == nodes->end()) {
+        nodes->emplace_back(part);
+        // path is from the beginning of the string to the end of the current
+        // part; this works because part is a reference to the internals of
+        // kv.first
+        nodes->back().path.assign(kv.first.data(),
+                                  part.data() + part.size() - kv.first.data());
+        it = nodes->end() - 1;
+      }
+      nodes = &it->children;
+    }
+
+    auto it = std::find_if(nodes->begin(), nodes->end(), [&](const auto& node) {
+      return node.name == parts.back();
+    });
+    if (it == nodes->end()) {
+      nodes->emplace_back(parts.back());
+      // no need to set path, as it's identical to kv.first
+      it = nodes->end() - 1;
+    }
+    it->entry = kv.second.get();
+  }
+}
+
+InputFile::InputFile(std::unique_ptr<DataLogThread> datalog_)
+    : filename{datalog_->GetBufferIdentifier()},
+      stem{fs::path{filename}.stem().string()},
+      datalog{std::move(datalog_)} {
+  datalog->sigEntryAdded.connect([this](const wpi::log::StartRecordData& srd) {
+    std::scoped_lock lock{gEntriesMutex};
+    auto it = gEntries.find(srd.name);
+    if (it == gEntries.end()) {
+      it = gEntries.emplace(srd.name, std::make_unique<Entry>(srd)).first;
+      RebuildEntryTree();
+    } else {
+      if (it->second->type != srd.type) {
+        it->second->typeConflict = true;
+      }
+      if (it->second->metadata != srd.metadata) {
+        it->second->metadataConflict = true;
+      }
+    }
+    it->second->inputFiles.emplace(this);
+  });
+}
+
+InputFile::~InputFile() {
+  if (gShutdown || !datalog) {
+    return;
+  }
+  std::scoped_lock lock{gEntriesMutex};
+  bool changed = false;
+  for (auto it = gEntries.begin(); it != gEntries.end();) {
+    it->second->inputFiles.erase(this);
+    if (it->second->inputFiles.empty()) {
+      it = gEntries.erase(it);
+      changed = true;
+    } else {
+      ++it;
+    }
+  }
+  if (changed) {
+    RebuildEntryTree();
+  }
+}
+
+static std::unique_ptr<InputFile> LoadDataLog(std::string_view filename) {
+  std::error_code ec;
+  auto buf = wpi::MemoryBuffer::GetFile(filename, ec);
+  std::string fn{filename};
+  if (ec) {
+    return std::make_unique<InputFile>(
+        fn, fmt::format("Could not open file: {}", ec.message()));
+  }
+
+  wpi::log::DataLogReader reader{std::move(buf)};
+  if (!reader.IsValid()) {
+    return std::make_unique<InputFile>(fn, "Not a valid datalog file");
+  }
+
+  return std::make_unique<InputFile>(
+      std::make_unique<DataLogThread>(std::move(reader)));
+}
+
+void DisplayInputFiles() {
+  static std::unique_ptr<pfd::open_file> dataFileSelector;
+
+  SetNextWindowPos(ImVec2{0, 20}, ImGuiCond_FirstUseEver);
+  SetNextWindowSize(ImVec2{375, 230}, ImGuiCond_FirstUseEver);
+  if (ImGui::Begin("Input Files")) {
+    if (ImGui::Button("Open File(s)...")) {
+      dataFileSelector = std::make_unique<pfd::open_file>(
+          "Select Data Log", "",
+          std::vector<std::string>{"DataLog Files", "*.wpilog"},
+          pfd::opt::multiselect);
+    }
+    ImGui::BeginTable(
+        "Input Files", 3,
+        ImGuiTableFlags_Borders | ImGuiTableFlags_SizingStretchProp);
+    ImGui::TableSetupColumn("File");
+    ImGui::TableSetupColumn("Status");
+    ImGui::TableSetupColumn("X", ImGuiTableColumnFlags_WidthFixed |
+                                     ImGuiTableColumnFlags_NoHeaderLabel |
+                                     ImGuiTableColumnFlags_NoHeaderWidth);
+    ImGui::TableHeadersRow();
+    for (auto it = gInputFiles.begin(); it != gInputFiles.end();) {
+      ImGui::TableNextRow();
+      ImGui::TableNextColumn();
+      if (it->second->highlight) {
+        ImGui::TableSetBgColor(ImGuiTableBgTarget_RowBg0,
+                               IM_COL32(0, 64, 0, 255));
+        it->second->highlight = false;
+      }
+      ImGui::TextUnformatted(it->first.c_str());
+      if (ImGui::IsItemHovered()) {
+        ImGui::SetTooltip("%s", it->second->filename.c_str());
+      }
+
+      ImGui::TableNextColumn();
+      if (it->second->datalog) {
+        ImGui::Text("%u records, %u entries%s",
+                    it->second->datalog->GetNumRecords(),
+                    it->second->datalog->GetNumEntries(),
+                    it->second->datalog->IsDone() ? "" : " (working)");
+      } else {
+        ImGui::TextUnformatted(it->second->status.c_str());
+      }
+
+      ImGui::TableNextColumn();
+      ImGui::PushID(it->first.c_str());
+      if (ImGui::SmallButton("X")) {
+        it = gInputFiles.erase(it);
+        gExportCount = 0;
+      } else {
+        ++it;
+      }
+      ImGui::PopID();
+    }
+    ImGui::EndTable();
+  }
+  ImGui::End();
+
+  // Load data file(s)
+  if (dataFileSelector && dataFileSelector->ready(0)) {
+    auto result = dataFileSelector->result();
+    for (auto&& filename : result) {
+      // don't allow duplicates
+      std::string stem = fs::path{filename}.stem().string();
+      auto it = gInputFiles.find(stem);
+      if (it == gInputFiles.end()) {
+        gInputFiles.emplace(std::move(stem), LoadDataLog(filename));
+        gExportCount = 0;
+      }
+    }
+    dataFileSelector.reset();
+  }
+}
+
+static bool EmitEntry(const std::string& name, Entry& entry) {
+  ImGui::TableNextColumn();
+  bool rv = ImGui::Checkbox(name.c_str(), &entry.selected);
+  if (ImGui::IsItemHovered() && gInputFiles.size() > 1) {
+    for (auto inputFile : entry.inputFiles) {
+      inputFile->highlight = true;
+    }
+  }
+
+  ImGui::TableNextColumn();
+  if (entry.typeConflict) {
+    ImGui::TextUnformatted("(Inconsistent)");
+    if (ImGui::IsItemHovered()) {
+      ImGui::BeginTooltip();
+      for (auto inputFile : entry.inputFiles) {
+        ImGui::Text(
+            "%s: %s", inputFile->stem.c_str(),
+            std::string{inputFile->datalog->GetEntry(entry.name).type}.c_str());
+      }
+      ImGui::EndTooltip();
+    }
+  } else {
+    ImGui::TextUnformatted(entry.type.c_str());
+  }
+
+  ImGui::TableNextColumn();
+  if (entry.metadataConflict) {
+    ImGui::TextUnformatted("(Inconsistent)");
+    if (ImGui::IsItemHovered()) {
+      ImGui::BeginTooltip();
+      for (auto inputFile : entry.inputFiles) {
+        ImGui::Text(
+            "%s: %s", inputFile->stem.c_str(),
+            std::string{inputFile->datalog->GetEntry(entry.name).metadata}
+                .c_str());
+      }
+      ImGui::EndTooltip();
+    }
+  } else {
+    ImGui::TextUnformatted(entry.metadata.c_str());
+  }
+  return rv;
+}
+
+static bool EmitEntryTree(std::vector<EntryTreeNode>& tree) {
+  bool rv = false;
+  for (auto&& node : tree) {
+    if (node.entry) {
+      if (EmitEntry(node.name, *node.entry)) {
+        rv = true;
+      }
+    }
+
+    if (!node.children.empty()) {
+      ImGui::TableNextColumn();
+      auto label = fmt::format("##check_{}", node.name);
+      if (node.selected == -1) {
+        ImGui::PushItemFlag(ImGuiItemFlags_MixedValue, true);
+        bool b = false;
+        if (ImGui::Checkbox(label.c_str(), &b)) {
+          node.selected = 3;  // 3 = enable group
+          rv = true;
+        }
+        ImGui::PopItemFlag();
+      } else {
+        bool b = node.selected == 1 || node.selected == 3;
+        if (ImGui::Checkbox(label.c_str(), &b)) {
+          node.selected = b ? 3 : 2;  // 2 = disable group
+          rv = true;
+        }
+      }
+      ImGui::SameLine();
+      bool open = ImGui::TreeNodeEx(node.name.c_str(),
+                                    ImGuiTreeNodeFlags_SpanFullWidth);
+      ImGui::TableNextColumn();
+      ImGui::TableNextColumn();
+      if (open) {
+        if (EmitEntryTree(node.children)) {
+          rv = true;
+        }
+        ImGui::TreePop();
+      }
+    }
+  }
+  return rv;
+}
+
+static void RefreshTreeCheckboxes(std::vector<EntryTreeNode>& tree,
+                                  int* selected) {
+  bool first = true;
+  for (auto&& node : tree) {
+    if (node.entry) {
+      if (first && *selected == -1) {
+        *selected = node.entry->selected ? 1 : 0;
+      }
+      if ((*selected == 0 && node.entry->selected) ||
+          (*selected == 1 && !node.entry->selected)) {
+        *selected = -1;             // inconsistent
+      } else if (*selected == 2) {  // disable group
+        node.entry->selected = false;
+      } else if (*selected == 3) {  // enable group
+        node.entry->selected = true;
+      }
+    }
+
+    if (!node.children.empty()) {
+      if (*selected == 2) {  // disable group
+        node.selected = 2;
+      } else if (*selected == 3) {  // enable group
+        node.selected = 3;
+      }
+      RefreshTreeCheckboxes(node.children, &node.selected);
+      if (node.selected == 2) {
+        node.selected = 0;
+      } else if (node.selected == 3) {
+        node.selected = 1;
+      }
+      if (first && *selected == -1) {
+        *selected = node.selected;
+      } else if (node.selected == -1 ||
+                 (*selected == 0 && node.selected == 1) ||
+                 (*selected == 1 && node.selected == 0)) {
+        *selected = -1;  // inconsistent
+      }
+    }
+
+    first = false;
+  }
+}
+
+void DisplayEntries() {
+  SetNextWindowPos(ImVec2{380, 20}, ImGuiCond_FirstUseEver);
+  SetNextWindowSize(ImVec2{540, 365}, ImGuiCond_FirstUseEver);
+  if (ImGui::Begin("Entries")) {
+    static bool treeView = true;
+    if (ImGui::BeginPopupContextItem()) {
+      ImGui::MenuItem("Tree View", "", &treeView);
+      ImGui::EndPopup();
+    }
+    std::scoped_lock lock{gEntriesMutex};
+    ImGui::BeginTable(
+        "Entries", 3,
+        ImGuiTableFlags_Borders | ImGuiTableFlags_SizingStretchProp);
+    ImGui::TableSetupColumn("Name");
+    ImGui::TableSetupColumn("Type");
+    ImGui::TableSetupColumn("Metadata");
+    ImGui::TableHeadersRow();
+    if (treeView) {
+      if (EmitEntryTree(gEntryTree)) {
+        int selected = -1;
+        RefreshTreeCheckboxes(gEntryTree, &selected);
+      }
+    } else {
+      for (auto&& kv : gEntries) {
+        EmitEntry(kv.first, *kv.second);
+      }
+    }
+    ImGui::EndTable();
+  }
+  ImGui::End();
+}
+
+static wpi::mutex gExportMutex;
+static std::vector<std::string> gExportErrors;
+
+static void PrintEscapedCsvString(wpi::raw_ostream& os, std::string_view str) {
+  auto s = str;
+  while (!s.empty()) {
+    std::string_view fragment;
+    std::tie(fragment, s) = wpi::split(s, '"');
+    os << fragment;
+    if (!s.empty()) {
+      os << '"' << '"';
+    }
+  }
+  if (wpi::ends_with(str, '"')) {
+    os << '"' << '"';
+  }
+}
+
+static void ValueToCsv(wpi::raw_ostream& os, const Entry& entry,
+                       const wpi::log::DataLogRecord& record) {
+  // handle systemTime specially
+  if (entry.name == "systemTime" && entry.type == "int64") {
+    int64_t val;
+    if (record.GetInteger(&val)) {
+      std::time_t timeval = val / 1000000;
+      fmt::print(os, "{:%Y-%m-%d %H:%M:%S}.{:06}", *std::localtime(&timeval),
+                 val % 1000000);
+      return;
+    }
+  } else if (entry.type == "double") {
+    double val;
+    if (record.GetDouble(&val)) {
+      fmt::print(os, "{}", val);
+      return;
+    }
+  } else if (entry.type == "int64") {
+    int64_t val;
+    if (record.GetInteger(&val)) {
+      fmt::print(os, "{}", val);
+      return;
+    }
+  } else if (entry.type == "string" || entry.type == "json") {
+    std::string_view val;
+    record.GetString(&val);
+    os << '"';
+    PrintEscapedCsvString(os, val);
+    os << '"';
+    return;
+  } else if (entry.type == "boolean") {
+    bool val;
+    if (record.GetBoolean(&val)) {
+      fmt::print(os, "{}", val);
+      return;
+    }
+  } else if (entry.type == "double[]") {
+    std::vector<double> val;
+    if (record.GetDoubleArray(&val)) {
+      fmt::print(os, "{}", fmt::join(val, ";"));
+      return;
+    }
+  } else if (entry.type == "float[]") {
+    std::vector<float> val;
+    if (record.GetFloatArray(&val)) {
+      fmt::print(os, "{}", fmt::join(val, ";"));
+      return;
+    }
+  } else if (entry.type == "int64[]") {
+    std::vector<int64_t> val;
+    if (record.GetIntegerArray(&val)) {
+      fmt::print(os, "{}", fmt::join(val, ";"));
+      return;
+    }
+  } else if (entry.type == "string[]") {
+    std::vector<std::string_view> val;
+    if (record.GetStringArray(&val)) {
+      os << '"';
+      bool first = true;
+      for (auto&& v : val) {
+        if (!first) {
+          os << ';';
+        }
+        first = false;
+        PrintEscapedCsvString(os, v);
+      }
+      os << '"';
+      return;
+    }
+  }
+  fmt::print(os, "<invalid>");
+}
+
+static void ExportCsvFile(InputFile& f, wpi::raw_ostream& os, int style) {
+  // header
+  if (style == 0) {
+    os << "Timestamp,Name,Value\n";
+  } else if (style == 1) {
+    // scan for exported fields for this file to print header and assign columns
+    os << "Timestamp";
+    int columnNum = 0;
+    for (auto&& entry : gEntries) {
+      if (entry.second->selected &&
+          entry.second->inputFiles.find(&f) != entry.second->inputFiles.end()) {
+        os << ',' << '"';
+        PrintEscapedCsvString(os, entry.first);
+        os << '"';
+        entry.second->column = columnNum++;
+      } else {
+        entry.second->column = -1;
+      }
+    }
+    os << '\n';
+  }
+
+  wpi::DenseMap<int, Entry*> nameMap;
+  for (auto&& record : f.datalog->GetReader()) {
+    if (record.IsStart()) {
+      wpi::log::StartRecordData data;
+      if (record.GetStartData(&data)) {
+        auto it = gEntries.find(data.name);
+        if (it != gEntries.end() && it->second->selected) {
+          nameMap[data.entry] = it->second.get();
+        }
+      }
+    } else if (record.IsFinish()) {
+      int entry;
+      if (record.GetFinishEntry(&entry)) {
+        nameMap.erase(entry);
+      }
+    } else if (!record.IsControl()) {
+      auto entryIt = nameMap.find(record.GetEntry());
+      if (entryIt == nameMap.end()) {
+        continue;
+      }
+      Entry* entry = entryIt->second;
+
+      if (style == 0) {
+        fmt::print(os, "{},\"", record.GetTimestamp() / 1000000.0);
+        PrintEscapedCsvString(os, entry->name);
+        os << '"' << ',';
+        ValueToCsv(os, *entry, record);
+        os << '\n';
+      } else if (style == 1 && entry->column != -1) {
+        fmt::print(os, "{},", record.GetTimestamp() / 1000000.0);
+        for (int i = 0; i < entry->column; ++i) {
+          os << ',';
+        }
+        ValueToCsv(os, *entry, record);
+        os << '\n';
+      }
+    }
+  }
+}
+
+static void ExportCsv(std::string_view outputFolder, int style) {
+  fs::path outPath{outputFolder};
+  for (auto&& f : gInputFiles) {
+    if (f.second->datalog) {
+      std::error_code ec;
+      auto of = fs::OpenFileForWrite(
+          outPath / fs::path{f.first}.replace_extension("csv"), ec,
+          fs::CD_CreateNew, fs::OF_Text);
+      if (ec) {
+        std::scoped_lock lock{gExportMutex};
+        gExportErrors.emplace_back(
+            fmt::format("{}: {}", f.first, ec.message()));
+        ++gExportCount;
+        continue;
+      }
+      wpi::raw_fd_ostream os{fs::FileToFd(of, ec, fs::OF_Text), true};
+      ExportCsvFile(*f.second, os, style);
+    }
+    ++gExportCount;
+  }
+}
+
+void DisplayOutput(glass::Storage& storage) {
+  static std::string& outputFolder = storage.GetString("outputFolder");
+  static std::unique_ptr<pfd::select_folder> outputFolderSelector;
+
+  SetNextWindowPos(ImVec2{380, 390}, ImGuiCond_FirstUseEver);
+  SetNextWindowSize(ImVec2{540, 120}, ImGuiCond_FirstUseEver);
+  if (ImGui::Begin("Output")) {
+    if (ImGui::Button("Select Output Folder...")) {
+      outputFolderSelector =
+          std::make_unique<pfd::select_folder>("Select Output Folder");
+    }
+    ImGui::TextUnformatted(outputFolder.c_str());
+
+    static const char* const options[] = {"List", "Table"};
+    static int style = 0;
+    ImGui::SetNextItemWidth(ImGui::GetFontSize() * 8);
+    ImGui::Combo("Style", &style, options,
+                 sizeof(options) / sizeof(const char*));
+
+    static std::future<void> exporter;
+    if (!gInputFiles.empty() && !outputFolder.empty() &&
+        ImGui::Button("Export CSV") &&
+        (gExportCount == 0 ||
+         gExportCount == static_cast<int>(gInputFiles.size()))) {
+      gExportCount = 0;
+      gExportErrors.clear();
+      exporter = std::async(std::launch::async, ExportCsv, outputFolder, style);
+    }
+    if (exporter.valid()) {
+      ImGui::SameLine();
+      ImGui::Text("Exported %d/%d", gExportCount.load(),
+                  static_cast<int>(gInputFiles.size()));
+    }
+    {
+      std::scoped_lock lock{gExportMutex};
+      for (auto&& err : gExportErrors) {
+        ImGui::TextUnformatted(err.c_str());
+      }
+    }
+  }
+  ImGui::End();
+
+  if (outputFolderSelector && outputFolderSelector->ready(0)) {
+    outputFolder = outputFolderSelector->result();
+    outputFolderSelector.reset();
+  }
+}

datalogtool/src/main/native/cpp/Exporter.h

@@ -0,0 +1,15 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#pragma once
+
+namespace glass {
+class Storage;
+}  // namespace glass
+
+void DisplayInputFiles();
+void DisplayEntries();
+void DisplayOutput(glass::Storage& storage);
+
+extern bool gShutdown;

datalogtool/src/main/native/cpp/Sftp.cpp

@@ -0,0 +1,215 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#include "Sftp.h"
+
+#include <fmt/format.h>
+
+using namespace sftp;
+
+Attributes::Attributes(sftp_attributes&& attr)
+    : name{attr->name}, flags{attr->flags}, type{attr->type}, size{attr->size} {
+  sftp_attributes_free(attr);
+}
+
+static std::string GetError(sftp_session sftp) {
+  switch (sftp_get_error(sftp)) {
+    case SSH_FX_EOF:
+      return "end of file";
+    case SSH_FX_NO_SUCH_FILE:
+      return "no such file";
+    case SSH_FX_PERMISSION_DENIED:
+      return "permission denied";
+    case SSH_FX_FAILURE:
+      return "SFTP failure";
+    case SSH_FX_BAD_MESSAGE:
+      return "SFTP bad message";
+    case SSH_FX_NO_CONNECTION:
+      return "SFTP no connection";
+    case SSH_FX_CONNECTION_LOST:
+      return "SFTP connection lost";
+    case SSH_FX_OP_UNSUPPORTED:
+      return "SFTP operation unsupported";
+    case SSH_FX_INVALID_HANDLE:
+      return "SFTP invalid handle";
+    case SSH_FX_NO_SUCH_PATH:
+      return "no such path";
+    case SSH_FX_FILE_ALREADY_EXISTS:
+      return "file already exists";
+    case SSH_FX_WRITE_PROTECT:
+      return "write protected filesystem";
+    case SSH_FX_NO_MEDIA:
+      return "no media inserted";
+    default:
+      return ssh_get_error(sftp->session);
+  }
+}
+
+Exception::Exception(sftp_session sftp)
+    : runtime_error{GetError(sftp)}, err{sftp_get_error(sftp)} {}
+
+File::~File() {
+  if (m_handle) {
+    sftp_close(m_handle);
+  }
+}
+
+Attributes File::Stat() const {
+  sftp_attributes attr = sftp_fstat(m_handle);
+  if (!attr) {
+    throw Exception{m_handle->sftp};
+  }
+  return Attributes{std::move(attr)};
+}
+
+size_t File::Read(void* buf, uint32_t count) {
+  auto rv = sftp_read(m_handle, buf, count);
+  if (rv < 0) {
+    throw Exception{m_handle->sftp};
+  }
+  return rv;
+}
+
+File::AsyncId File::AsyncReadBegin(uint32_t len) const {
+  int rv = sftp_async_read_begin(m_handle, len);
+  if (rv < 0) {
+    throw Exception{m_handle->sftp};
+  }
+  return rv;
+}
+
+size_t File::AsyncRead(void* data, uint32_t len, AsyncId id) {
+  auto rv = sftp_async_read(m_handle, data, len, id);
+  if (rv == SSH_ERROR) {
+    throw Exception{ssh_get_error(m_handle->sftp->session)};
+  }
+  if (rv == SSH_AGAIN) {
+    return 0;
+  }
+  return rv;
+}
+
+size_t File::Write(wpi::span<const uint8_t> data) {
+  auto rv = sftp_write(m_handle, data.data(), data.size());
+  if (rv < 0) {
+    throw Exception{m_handle->sftp};
+  }
+  return rv;
+}
+
+void File::Seek(uint64_t offset) {
+  if (sftp_seek64(m_handle, offset) < 0) {
+    throw Exception{m_handle->sftp};
+  }
+}
+
+uint64_t File::Tell() const {
+  return sftp_tell64(m_handle);
+}
+
+void File::Rewind() {
+  sftp_rewind(m_handle);
+}
+
+void File::Sync() {
+  if (sftp_fsync(m_handle) < 0) {
+    throw Exception{m_handle->sftp};
+  }
+}
+
+Session::Session(std::string_view host, int port, std::string_view user,
+                 std::string_view pass)
+    : m_host{host}, m_port{port}, m_username{user}, m_password{pass} {
+  // Create a new SSH session.
+  m_session = ssh_new();
+  if (!m_session) {
+    throw Exception{"The SSH session could not be allocated."};
+  }
+
+  // Set the host, user, and port.
+  ssh_options_set(m_session, SSH_OPTIONS_HOST, m_host.c_str());
+  ssh_options_set(m_session, SSH_OPTIONS_USER, m_username.c_str());
+  ssh_options_set(m_session, SSH_OPTIONS_PORT, &m_port);
+
+  // Set timeout to 3 seconds.
+  int64_t timeout = 3L;
+  ssh_options_set(m_session, SSH_OPTIONS_TIMEOUT, &timeout);
+
+  // Set other miscellaneous options.
+  ssh_options_set(m_session, SSH_OPTIONS_STRICTHOSTKEYCHECK, "no");
+}
+
+Session::~Session() {
+  if (m_sftp) {
+    sftp_free(m_sftp);
+  }
+  if (m_session) {
+    ssh_free(m_session);
+  }
+}
+
+void Session::Connect() {
+  // Connect to the server.
+  int rc = ssh_connect(m_session);
+  if (rc != SSH_OK) {
+    throw Exception{ssh_get_error(m_session)};
+  }
+
+  // Authenticate with password.
+  rc = ssh_userauth_password(m_session, nullptr, m_password.c_str());
+  if (rc != SSH_AUTH_SUCCESS) {
+    throw Exception{ssh_get_error(m_session)};
+  }
+
+  // Allocate the SFTP session.
+  m_sftp = sftp_new(m_session);
+  if (!m_sftp) {
+    throw Exception{ssh_get_error(m_session)};
+  }
+
+  // Initialize.
+  rc = sftp_init(m_sftp);
+  if (rc != SSH_OK) {
+    sftp_free(m_sftp);
+    m_sftp = nullptr;
+    throw Exception{ssh_get_error(m_session)};
+  }
+}
+
+void Session::Disconnect() {
+  if (m_sftp) {
+    sftp_free(m_sftp);
+    m_sftp = nullptr;
+  }
+  ssh_disconnect(m_session);
+}
+
+std::vector<Attributes> Session::ReadDir(const std::string& path) {
+  sftp_dir dir = sftp_opendir(m_sftp, path.c_str());
+  if (!dir) {
+    throw Exception{m_sftp};
+  }
+
+  std::vector<Attributes> rv;
+  while (sftp_attributes attr = sftp_readdir(m_sftp, dir)) {
+    rv.emplace_back(std::move(attr));
+  }
+
+  sftp_closedir(dir);
+  return rv;
+}
+
+void Session::Unlink(const std::string& filename) {
+  if (sftp_unlink(m_sftp, filename.c_str()) < 0) {
+    throw Exception{m_sftp};
+  }
+}
+
+File Session::Open(const std::string& filename, int accesstype, mode_t mode) {
+  sftp_file f = sftp_open(m_sftp, filename.c_str(), accesstype, mode);
+  if (!f) {
+    throw Exception{m_sftp};
+  }
+  return File{std::move(f)};
+}

datalogtool/src/main/native/cpp/Sftp.h

@@ -0,0 +1,144 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#pragma once
+
+#include <libssh/libssh.h>
+#include <libssh/sftp.h>
+
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include <wpi/span.h>
+
+namespace sftp {
+
+struct Attributes {
+  Attributes() = default;
+  explicit Attributes(sftp_attributes&& attr);
+
+  std::string name;
+  uint32_t flags = 0;
+  uint8_t type = 0;
+  uint64_t size = 0;
+};
+
+/**
+ * This is the exception that will be thrown if something goes wrong.
+ */
+class Exception : public std::runtime_error {
+ public:
+  explicit Exception(const std::string& msg) : std::runtime_error{msg} {}
+  explicit Exception(sftp_session sftp);
+
+  int err = 0;
+};
+
+class File {
+ public:
+  File() = default;
+  explicit File(sftp_file&& handle) : m_handle{handle} {}
+  ~File();
+
+  Attributes Stat() const;
+
+  void SetNonblocking() { sftp_file_set_nonblocking(m_handle); }
+  void SetBlocking() { sftp_file_set_blocking(m_handle); }
+
+  using AsyncId = uint32_t;
+
+  size_t Read(void* buf, uint32_t count);
+  AsyncId AsyncReadBegin(uint32_t len) const;
+  size_t AsyncRead(void* data, uint32_t len, AsyncId id);
+  size_t Write(wpi::span<const uint8_t> data);
+
+  void Seek(uint64_t offset);
+  uint64_t Tell() const;
+  void Rewind();
+
+  void Sync();
+
+  std::string_view GetName() const { return m_handle->name; }
+  uint64_t GetOffset() const { return m_handle->offset; }
+  bool IsEof() const { return m_handle->eof; }
+  bool IsNonblocking() const { return m_handle->nonblocking; }
+
+ private:
+  sftp_file m_handle{nullptr};
+};
+
+/**
+ * This class is a C++ implementation of the SshSessionController in
+ * wpilibsuite/deploy-utils. It handles connecting to an SSH server, running
+ * commands, and transferring files.
+ */
+class Session {
+ public:
+  /**
+   * Constructs a new session controller.
+   *
+   * @param host The hostname of the server to connect to.
+   * @param port The port that the sshd server is operating on.
+   * @param user The username to login as.
+   * @param pass The password for the given username.
+   */
+  Session(std::string_view host, int port, std::string_view user,
+          std::string_view pass);
+
+  /**
+   * Destroys the controller object. This also disconnects the session from the
+   * server.
+   */
+  ~Session();
+
+  /**
+   * Opens the SSH connection to the given host.
+   */
+  void Connect();
+
+  /**
+   * Disconnects the SSH connection.
+   */
+  void Disconnect();
+
+  /**
+   * Reads directory entries
+   *
+   * @param path remote path
+   * @return vector of file attributes
+   */
+  std::vector<Attributes> ReadDir(const std::string& path);
+
+  /**
+   * Unlinks (deletes) a file.
+   *
+   * @param filename filename
+   */
+  void Unlink(const std::string& filename);
+
+  /**
+   * Opens a file.
+   *
+   * @param filename filename
+   * @param accesstype O_RDONLY, O_WRONLY, or O_RDWR, combined with O_CREAT,
+   *                   O_EXCL, or O_TRUNC
+   * @param mode permissions to use if a new file is created
+   * @return File
+   */
+  File Open(const std::string& filename, int accesstype, mode_t mode);
+
+ private:
+  ssh_session m_session{nullptr};
+  sftp_session m_sftp{nullptr};
+  std::string m_host;
+
+  int m_port;
+
+  std::string m_username;
+  std::string m_password;
+};
+
+}  // namespace sftp

datalogtool/src/main/native/cpp/main.cpp

@@ -0,0 +1,25 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#include <string_view>
+
+void Application(std::string_view saveDir);
+
+#ifdef _WIN32
+int __stdcall WinMain(void* hInstance, void* hPrevInstance, char* pCmdLine,
+                      int nCmdShow) {
+  int argc = __argc;
+  char** argv = __argv;
+#else
+int main(int argc, char** argv) {
+#endif
+  std::string_view saveDir;
+  if (argc == 2) {
+    saveDir = argv[1];
+  }
+
+  Application(saveDir);
+
+  return 0;
+}

datalogtool/src/main/native/win/datalogtool.rc

@@ -0,0 +1 @@
+IDI_ICON1 ICON "datalogtool.ico"

settings.gradle

@@ -33,6 +33,7 @@ include 'fieldImages'
 include 'glass'
 include 'outlineviewer'
 include 'roborioteamnumbersetter'
+include 'datalogtool'
 include 'simulation:gz_msgs'
 include 'simulation:frc_gazebo_plugins'
 include 'simulation:halsim_gazebo'