[sysid] Load DataLog files directly for analysis (#6103)

Co-authored-by: Oblarg <emichaelbrnett@gmail.com>

sysid/src/main/native/cpp/view/Analyzer.cpp

@@ -28,7 +28,7 @@
 using namespace sysid;
 
 Analyzer::Analyzer(glass::Storage& storage, wpi::Logger& logger)
-    : m_location(""), m_logger(logger) {
+    : m_logger(logger) {
   // Fill the StringMap with preset values.
   m_presets["Default"] = presets::kDefault;
   m_presets["WPILib (2020-)"] = presets::kWPILibNew;
@@ -48,16 +48,14 @@ Analyzer::Analyzer(glass::Storage& storage, wpi::Logger& logger)
 void Analyzer::UpdateFeedforwardGains() {
   WPI_INFO(m_logger, "{}", "Gain calc");
   try {
-    const auto& [ff, trackWidth] = m_manager->CalculateFeedforward();
+    const auto& [ff] = m_manager->CalculateFeedforward();
     m_ff = ff.coeffs;
     m_accelRSquared = ff.rSquared;
     m_accelRMSE = ff.rmse;
-    m_trackWidth = trackWidth;
     m_settings.preset.measurementDelay =
         m_settings.type == FeedbackControllerLoopType::kPosition
             ? m_manager->GetPositionDelay()
             : m_manager->GetVelocityDelay();
-    m_conversionFactor = m_manager->GetFactor();
     PrepareGraphs();
   } catch (const sysid::InvalidDataError& e) {
     m_state = AnalyzerState::kGeneralDataError;
@@ -81,6 +79,7 @@ void Analyzer::UpdateFeedforwardGains() {
 }
 
 void Analyzer::UpdateFeedbackGains() {
+  WPI_INFO(m_logger, "{}", "Updating feedback gains");
   if (m_ff[1] > 0 && m_ff[2] > 0) {
     const auto& fb = m_manager->CalculateFeedback(m_ff);
     m_timescale = units::second_t{m_ff[2] / m_ff[1]};
@@ -119,27 +118,9 @@ bool Analyzer::IsDataErrorState() {
          m_state == AnalyzerState::kGeneralDataError;
 }
 
-void Analyzer::DisplayFileSelector() {
-  // Get the current width of the window. This will be used to scale
-  // our UI elements.
-  float width = ImGui::GetContentRegionAvail().x;
-
-  // Show the file location along with an option to choose.
-  if (ImGui::Button("Select")) {
-    m_selector = std::make_unique<pfd::open_file>(
-        "Select Data", "",
-        std::vector<std::string>{"JSON File", SYSID_PFD_JSON_EXT});
-  }
-  ImGui::SameLine();
-  ImGui::SetNextItemWidth(width - ImGui::CalcTextSize("Select").x -
-                          ImGui::GetFontSize() * 5);
-  ImGui::InputText("##location", &m_location, ImGuiInputTextFlags_ReadOnly);
-}
-
 void Analyzer::ResetData() {
   m_plot.ResetData();
   m_manager = std::make_unique<AnalysisManager>(m_settings, m_logger);
-  m_location = "";
   m_ff = std::vector<double>{1, 1, 1};
   UpdateFeedbackGains();
 }
@@ -152,38 +133,15 @@ bool Analyzer::DisplayResetAndUnitOverride() {
   ImGui::SameLine(width - ImGui::CalcTextSize("Reset").x);
   if (ImGui::Button("Reset")) {
     ResetData();
-    m_state = AnalyzerState::kWaitingForJSON;
+    m_state = AnalyzerState::kWaitingForData;
     return true;
   }
 
-  if (type == analysis::kDrivetrain) {
-    ImGui::SetNextItemWidth(ImGui::GetFontSize() * kTextBoxWidthMultiple);
-    if (ImGui::Combo("Dataset", &m_dataset, kDatasets, 3)) {
-      m_settings.dataset =
-          static_cast<AnalysisManager::Settings::DrivetrainDataset>(m_dataset);
-      PrepareData();
-    }
-    ImGui::SameLine();
-  } else {
-    m_settings.dataset =
-        AnalysisManager::Settings::DrivetrainDataset::kCombined;
-  }
-
   ImGui::Spacing();
   ImGui::Text(
       "Units:              %s\n"
-      "Units Per Rotation: %.4f\n"
       "Type:               %s",
-      std::string(unit).c_str(), m_conversionFactor, type.name);
-
-  if (type == analysis::kDrivetrainAngular) {
-    ImGui::SameLine();
-    sysid::CreateTooltip(
-        "Here, the units and units per rotation represent what the wheel "
-        "positions and velocities were captured in. The track width value "
-        "will reflect the unit selected here. However, the Kv and Ka will "
-        "always be in Vs/rad and Vs^2 / rad respectively.");
-  }
+      std::string(unit).c_str(), type.name);
 
   if (ImGui::Button("Override Units")) {
     ImGui::OpenPopup("Override Units");
@@ -197,24 +155,11 @@ bool Analyzer::DisplayResetAndUnitOverride() {
                  IM_ARRAYSIZE(kUnits));
     unit = kUnits[m_selectedOverrideUnit];
 
-    if (unit == "Degrees") {
-      m_conversionFactor = 360.0;
-    } else if (unit == "Radians") {
-      m_conversionFactor = 2 * std::numbers::pi;
-    } else if (unit == "Rotations") {
-      m_conversionFactor = 1.0;
-    }
-
-    bool isRotational = m_selectedOverrideUnit > 2;
-
     ImGui::SetNextItemWidth(ImGui::GetFontSize() * 7);
-    ImGui::InputDouble(
-        "Units Per Rotation", &m_conversionFactor, 0.0, 0.0, "%.4f",
-        isRotational ? ImGuiInputTextFlags_ReadOnly : ImGuiInputTextFlags_None);
 
     if (ImGui::Button("Close")) {
       ImGui::CloseCurrentPopup();
-      m_manager->OverrideUnits(unit, m_conversionFactor);
+      m_manager->OverrideUnits(unit);
       PrepareData();
     }
 
@@ -234,22 +179,21 @@ void Analyzer::ConfigParamsOnFileSelect() {
   WPI_INFO(m_logger, "{}", "Configuring Params");
   m_stepTestDuration = m_settings.stepTestDuration.to<float>();
 
-  // Estimate qp as 1/8 * units-per-rot
-  m_settings.lqr.qp = 0.125 * m_manager->GetFactor();
+  // Estimate qp as 1/10 native distance unit
+  m_settings.lqr.qp = 0.1;
   // Estimate qv as 1/4 * max velocity = 1/4 * (12V - kS) / kV
   m_settings.lqr.qv = 0.25 * (12.0 - m_ff[0]) / m_ff[1];
 }
 
 void Analyzer::Display() {
-  DisplayFileSelector();
   DisplayGraphs();
 
   switch (m_state) {
-    case AnalyzerState::kWaitingForJSON: {
+    case AnalyzerState::kWaitingForData: {
       ImGui::Text(
           "SysId is currently in theoretical analysis mode.\n"
           "To analyze recorded test data, select a "
-          "data JSON.");
+          "data file (.wpilog).");
       sysid::CreateTooltip(
           "Theoretical feedback gains can be calculated from a "
           "physical model of the mechanism being controlled. "
@@ -295,7 +239,7 @@ void Analyzer::Display() {
     case AnalyzerState::kFileError: {
       CreateErrorPopup(m_errorPopup, m_exception);
       if (!m_errorPopup) {
-        m_state = AnalyzerState::kWaitingForJSON;
+        m_state = AnalyzerState::kWaitingForData;
         return;
       }
       break;
@@ -313,20 +257,10 @@ void Analyzer::Display() {
       break;
     }
   }
-
-  // Periodic functions
-  try {
-    SelectFile();
-  } catch (const AnalysisManager::FileReadingError& e) {
-    m_state = AnalyzerState::kFileError;
-    HandleError(e.what());
-  } catch (const wpi::json::exception& e) {
-    m_state = AnalyzerState::kFileError;
-    HandleError(e.what());
-  }
 }
 
 void Analyzer::PrepareData() {
+  WPI_INFO(m_logger, "{}", "Preparing data");
   try {
     m_manager->PrepareData();
     UpdateFeedforwardGains();
@@ -379,9 +313,6 @@ void Analyzer::PrepareGraphs() {
 void Analyzer::HandleError(std::string_view msg) {
   m_exception = msg;
   m_errorPopup = true;
-  if (m_state == AnalyzerState::kFileError) {
-    m_location = "";
-  }
   PrepareRawGraphs();
 }
 
@@ -458,23 +389,12 @@ void Analyzer::DisplayGraphs() {
   ImGui::End();
 }
 
-void Analyzer::SelectFile() {
-  // If the selector exists and is ready with a result, we can store it.
-  if (m_selector && m_selector->ready() && !m_selector->result().empty()) {
-    // Store the location of the file and reset the selector.
-    WPI_INFO(m_logger, "Opening File: {}", m_selector->result()[0]);
-    m_location = m_selector->result()[0];
-    m_selector.reset();
-    WPI_INFO(m_logger, "{}", "Opened File");
-    m_manager =
-        std::make_unique<AnalysisManager>(m_location, m_settings, m_logger);
-    PrepareData();
-    m_dataset = 0;
-    m_settings.dataset =
-        AnalysisManager::Settings::DrivetrainDataset::kCombined;
-    ConfigParamsOnFileSelect();
-    UpdateFeedbackGains();
-  }
+void Analyzer::AnalyzeData() {
+  m_manager = std::make_unique<AnalysisManager>(m_data, m_settings, m_logger);
+  PrepareData();
+  m_dataset = 0;
+  ConfigParamsOnFileSelect();
+  UpdateFeedbackGains();
 }
 
 void Analyzer::AbortDataPrep() {
@@ -625,8 +545,6 @@ void Analyzer::DisplayFeedforwardGains(float beginX, float beginY) {
         "This is the angle offset which, when added to the angle measurement, "
         "zeroes it out when the arm is horizontal. This is needed for the arm "
         "feedforward to work.");
-  } else if (m_trackWidth) {
-    DisplayGain("Track Width", &*m_trackWidth);
   }
   double endY = ImGui::GetCursorPosY();
 
@@ -790,7 +708,7 @@ void Analyzer::DisplayFeedbackGains() {
                    IM_ARRAYSIZE(kLoopTypes))) {
     m_settings.type =
         static_cast<FeedbackControllerLoopType>(m_selectedLoopType);
-    if (m_state == AnalyzerState::kWaitingForJSON) {
+    if (m_state == AnalyzerState::kWaitingForData) {
       m_settings.preset.measurementDelay = 0_ms;
     } else {
       if (m_settings.type == FeedbackControllerLoopType::kPosition) {
@@ -817,7 +735,7 @@ void Analyzer::DisplayFeedbackGains() {
 
   if (m_selectedLoopType == 0) {
     std::string unit;
-    if (m_state != AnalyzerState::kWaitingForJSON) {
+    if (m_state != AnalyzerState::kWaitingForData) {
       unit = fmt::format(" ({})", GetAbbreviation(m_manager->GetUnit()));
     }
 
@@ -831,7 +749,7 @@ void Analyzer::DisplayFeedbackGains() {
   }
 
   std::string unit;
-  if (m_state != AnalyzerState::kWaitingForJSON) {
+  if (m_state != AnalyzerState::kWaitingForData) {
     unit = fmt::format(" ({}/s)", GetAbbreviation(m_manager->GetUnit()));
   }
 

sysid/src/main/native/cpp/view/DataSelector.cpp

@@ -0,0 +1,244 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#include "sysid/view/DataSelector.h"
+
+#include <fmt/format.h>
+#include <glass/support/DataLogReaderThread.h>
+#include <imgui.h>
+#include <wpi/DataLogReader.h>
+#include <wpi/Logger.h>
+#include <wpi/StringExtras.h>
+
+#include "sysid/Util.h"
+#include "sysid/analysis/AnalysisType.h"
+#include "sysid/analysis/Storage.h"
+
+using namespace sysid;
+
+static constexpr const char* kAnalysisTypes[] = {"Elevator", "Arm", "Simple"};
+
+static bool EmitEntryTarget(const char* name, bool isString,
+                            const glass::DataLogReaderEntry** entry) {
+  if (*entry) {
+    auto text =
+        fmt::format("{}: {} ({})", name, (*entry)->name, (*entry)->type);
+    ImGui::TextUnformatted(text.c_str());
+  } else {
+    ImGui::Text("%s: <none (DROP HERE)> (%s)", name,
+                isString ? "string" : "number");
+  }
+  bool rv = false;
+  if (ImGui::BeginDragDropTarget()) {
+    if (const ImGuiPayload* payload = ImGui::AcceptDragDropPayload(
+            isString ? "DataLogEntryString" : "DataLogEntry")) {
+      assert(payload->DataSize == sizeof(const glass::DataLogReaderEntry*));
+      *entry = *static_cast<const glass::DataLogReaderEntry**>(payload->Data);
+      rv = true;
+    }
+    ImGui::EndDragDropTarget();
+  }
+  return rv;
+}
+
+void DataSelector::Display() {
+  using namespace std::chrono_literals;
+
+  // building test data is modal (due to async access)
+  if (m_testdataFuture.valid()) {
+    if (m_testdataFuture.wait_for(0s) == std::future_status::ready) {
+      TestData data = m_testdataFuture.get();
+      for (auto&& motordata : data.motorData) {
+        m_testdataStats.emplace_back(
+            fmt::format("Test State: {}", motordata.first()));
+        int i = 0;
+        for (auto&& run : motordata.second.runs) {
+          m_testdataStats.emplace_back(fmt::format(
+              "  Run {} samples: {} Volt {} Pos {} Vel", ++i,
+              run.voltage.size(), run.position.size(), run.velocity.size()));
+        }
+      }
+      if (testdata) {
+        testdata(std::move(data));
+      }
+    }
+    ImGui::Text("Loading data...");
+    return;
+  }
+
+  if (!m_testdataStats.empty()) {
+    for (auto&& line : m_testdataStats) {
+      ImGui::TextUnformatted(line.c_str());
+    }
+    if (ImGui::Button("Ok")) {
+      m_testdataStats.clear();
+    }
+    return;
+  }
+
+  if (EmitEntryTarget("Test State", true, &m_testStateEntry)) {
+    m_testsFuture =
+        std::async(std::launch::async, [testStateEntry = m_testStateEntry] {
+          return LoadTests(*testStateEntry);
+        });
+  }
+
+  if (!m_testStateEntry) {
+    return;
+  }
+
+  if (m_testsFuture.valid() &&
+      m_testsFuture.wait_for(0s) == std::future_status::ready) {
+    m_tests = m_testsFuture.get();
+  }
+
+  if (m_tests.empty()) {
+    if (m_testsFuture.valid()) {
+      ImGui::TextUnformatted("Reading tests...");
+    } else {
+      ImGui::TextUnformatted("No tests found");
+    }
+    return;
+  }
+
+#if 0
+  // Test filtering
+  if (ImGui::BeginCombo("Test", m_selectedTest.c_str())) {
+    for (auto&& test : m_tests) {
+      if (ImGui::Selectable(test.first.c_str(), test.first == m_selectedTest)) {
+        m_selectedTest = test.first;
+      }
+    }
+    ImGui::EndCombo();
+  }
+#endif
+
+  ImGui::Combo("Analysis Type", &m_selectedAnalysis, kAnalysisTypes,
+               IM_ARRAYSIZE(kAnalysisTypes));
+
+  // DND targets
+  EmitEntryTarget("Velocity", false, &m_velocityEntry);
+  EmitEntryTarget("Position", false, &m_positionEntry);
+  EmitEntryTarget("Voltage", false, &m_voltageEntry);
+
+  ImGui::SetNextItemWidth(ImGui::GetFontSize() * 7);
+  ImGui::Combo("Units", &m_selectedUnit, kUnits, IM_ARRAYSIZE(kUnits));
+
+  ImGui::InputDouble("Velocity scaling", &m_velocityScale);
+  ImGui::InputDouble("Position scaling", &m_positionScale);
+
+  if (/*!m_selectedTest.empty() &&*/ m_velocityEntry && m_positionEntry &&
+      m_voltageEntry) {
+    if (ImGui::Button("Load")) {
+      m_testdataFuture =
+          std::async(std::launch::async, [this] { return BuildTestData(); });
+    }
+  }
+}
+
+void DataSelector::Reset() {
+  m_testsFuture = {};
+  m_tests.clear();
+  m_selectedTest.clear();
+  m_testStateEntry = nullptr;
+  m_velocityEntry = nullptr;
+  m_positionEntry = nullptr;
+  m_voltageEntry = nullptr;
+  m_testdataFuture = {};
+}
+
+DataSelector::Tests DataSelector::LoadTests(
+    const glass::DataLogReaderEntry& testStateEntry) {
+  Tests tests;
+  for (auto&& range : testStateEntry.ranges) {
+    std::string_view prevState;
+    Runs* curRuns = nullptr;
+    wpi::log::DataLogReader::iterator lastStart = range.begin();
+    for (auto it = range.begin(), end = range.end(); it != end; ++it) {
+      std::string_view testState;
+      if (it->GetEntry() != testStateEntry.entry ||
+          !it->GetString(&testState)) {
+        continue;
+      }
+
+      // track runs as iterator ranges of the same test
+      if (testState != prevState) {
+        if (curRuns) {
+          curRuns->emplace_back(lastStart, it);
+        }
+        lastStart = it;
+      }
+      prevState = testState;
+
+      if (testState == "none") {
+        curRuns = nullptr;
+        continue;
+      }
+
+      auto [testName, direction] = wpi::rsplit(testState, '-');
+      auto testIt = tests.find(testName);
+      if (testIt == tests.end()) {
+        testIt = tests.emplace(std::string{testName}, State{}).first;
+      }
+      auto stateIt = testIt->second.find(testState);
+      if (stateIt == testIt->second.end()) {
+        stateIt = testIt->second.emplace(std::string{testState}, Runs{}).first;
+      }
+      curRuns = &stateIt->second;
+    }
+
+    if (curRuns) {
+      curRuns->emplace_back(lastStart, range.end());
+    }
+  }
+  return tests;
+}
+
+template <typename T>
+static void AddSample(std::vector<MotorData::Run::Sample<T>>& samples,
+                      const wpi::log::DataLogRecord& record, bool isDouble,
+                      double scale) {
+  if (isDouble) {
+    double val;
+    if (record.GetDouble(&val)) {
+      samples.emplace_back(units::second_t{record.GetTimestamp() * 1.0e-6},
+                           T{val * scale});
+    }
+  } else {
+    float val;
+    if (record.GetFloat(&val)) {
+      samples.emplace_back(units::second_t{record.GetTimestamp() * 1.0e-6},
+                           T{static_cast<double>(val * scale)});
+    }
+  }
+}
+
+TestData DataSelector::BuildTestData() {
+  TestData data;
+  data.distanceUnit = kUnits[m_selectedUnit];
+  data.mechanismType = analysis::FromName(kAnalysisTypes[m_selectedAnalysis]);
+  bool voltageDouble = m_voltageEntry->type == "double";
+  bool positionDouble = m_positionEntry->type == "double";
+  bool velocityDouble = m_velocityEntry->type == "double";
+
+  for (auto&& test : m_tests) {
+    for (auto&& state : test.second) {
+      auto& motorData = data.motorData[state.first];
+      for (auto&& range : state.second) {
+        auto& run = motorData.runs.emplace_back();
+        for (auto&& record : range) {
+          if (record.GetEntry() == m_voltageEntry->entry) {
+            AddSample(run.voltage, record, voltageDouble, 1.0);
+          } else if (record.GetEntry() == m_positionEntry->entry) {
+            AddSample(run.position, record, positionDouble, m_positionScale);
+          } else if (record.GetEntry() == m_velocityEntry->entry) {
+            AddSample(run.velocity, record, velocityDouble, m_velocityScale);
+          }
+        }
+      }
+    }
+  }
+
+  return data;
+}

sysid/src/main/native/cpp/view/JSONConverter.cpp

@@ -1,64 +0,0 @@
-// Copyright (c) FIRST and other WPILib contributors.
-// Open Source Software; you can modify and/or share it under the terms of
-// the WPILib BSD license file in the root directory of this project.
-
-#include "sysid/analysis/JSONConverter.h"
-#include "sysid/view/JSONConverter.h"
-
-#include <exception>
-
-#include <imgui.h>
-#include <portable-file-dialogs.h>
-#include <wpi/timestamp.h>
-
-#include "sysid/Util.h"
-
-using namespace sysid;
-
-void JSONConverter::DisplayConverter(
-    const char* tooltip,
-    std::function<std::string(std::string_view, wpi::Logger&)> converter) {
-  if (ImGui::Button(tooltip)) {
-    m_opener = std::make_unique<pfd::open_file>(
-        tooltip, "", std::vector<std::string>{"JSON File", SYSID_PFD_JSON_EXT});
-  }
-
-  if (m_opener && m_opener->ready()) {
-    if (!m_opener->result().empty()) {
-      m_location = m_opener->result()[0];
-      try {
-        converter(m_location, m_logger);
-        m_timestamp = wpi::Now() * 1E-6;
-      } catch (const std::exception& e) {
-        ImGui::OpenPopup("Exception Caught!");
-        m_exception = e.what();
-      }
-    }
-    m_opener.reset();
-  }
-
-  if (wpi::Now() * 1E-6 - m_timestamp < 5) {
-    ImGui::SameLine();
-    ImGui::Text("Saved!");
-  }
-
-  // Handle exceptions.
-  ImGui::SetNextWindowSize(ImVec2(480.f, 0.0f));
-  if (ImGui::BeginPopupModal("Exception Caught!")) {
-    ImGui::PushTextWrapPos(0.0f);
-    ImGui::Text(
-        "An error occurred when parsing the JSON. This most likely means that "
-        "the JSON data is incorrectly formatted.");
-    ImGui::TextColored(ImVec4(1.0f, 0.4f, 0.4f, 1.0f), "%s",
-                       m_exception.c_str());
-    ImGui::PopTextWrapPos();
-    if (ImGui::Button("Close")) {
-      ImGui::CloseCurrentPopup();
-    }
-    ImGui::EndPopup();
-  }
-}
-
-void JSONConverter::DisplayCSVConvert() {
-  DisplayConverter("Select SysId JSON", sysid::ToCSV);
-}

sysid/src/main/native/cpp/view/LogLoader.cpp

@@ -0,0 +1,208 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#include "sysid/view/LogLoader.h"
+
+#include <algorithm>
+#include <memory>
+#include <span>
+#include <string_view>
+
+#include <glass/support/DataLogReaderThread.h>
+#include <imgui.h>
+#include <imgui_stdlib.h>
+#include <portable-file-dialogs.h>
+#include <wpi/SpanExtras.h>
+#include <wpi/StringExtras.h>
+#include <wpi/fs.h>
+
+using namespace sysid;
+
+LogLoader::LogLoader(glass::Storage& storage, wpi::Logger& logger) {}
+
+LogLoader::~LogLoader() = default;
+
+void LogLoader::Display() {
+  if (ImGui::Button("Open data log file...")) {
+    m_opener = std::make_unique<pfd::open_file>(
+        "Select Data Log", "",
+        std::vector<std::string>{"DataLog Files", "*.wpilog"});
+  }
+
+  // Handle opening the file
+  if (m_opener && m_opener->ready(0)) {
+    if (!m_opener->result().empty()) {
+      m_filename = m_opener->result()[0];
+
+      std::error_code ec;
+      auto buf = wpi::MemoryBuffer::GetFile(m_filename, ec);
+      if (ec) {
+        ImGui::OpenPopup("Error");
+        m_error = fmt::format("Could not open file: {}", ec.message());
+        return;
+      }
+
+      wpi::log::DataLogReader reader{std::move(buf)};
+      if (!reader.IsValid()) {
+        ImGui::OpenPopup("Error");
+        m_error = "Not a valid datalog file";
+        return;
+      }
+      unload();
+      m_reader =
+          std::make_unique<glass::DataLogReaderThread>(std::move(reader));
+      m_entryTree.clear();
+    }
+    m_opener.reset();
+  }
+
+  // Handle errors
+  ImGui::SetNextWindowSize(ImVec2(480.f, 0.0f));
+  if (ImGui::BeginPopupModal("Error")) {
+    ImGui::PushTextWrapPos(0.0f);
+    ImGui::TextUnformatted(m_error.c_str());
+    ImGui::PopTextWrapPos();
+    if (ImGui::Button("Close")) {
+      ImGui::CloseCurrentPopup();
+    }
+    ImGui::EndPopup();
+  }
+
+  if (!m_reader) {
+    return;
+  }
+
+  // Summary info
+  ImGui::TextUnformatted(fs::path{m_filename}.stem().string().c_str());
+  ImGui::Text("%u records, %u entries%s", m_reader->GetNumRecords(),
+              m_reader->GetNumEntries(),
+              m_reader->IsDone() ? "" : " (working)");
+
+  if (!m_reader->IsDone()) {
+    return;
+  }
+
+  bool refilter = ImGui::InputText("Filter", &m_filter);
+
+  // Display tree of entries
+  if (m_entryTree.empty() || refilter) {
+    RebuildEntryTree();
+  }
+
+  ImGui::BeginTable(
+      "Entries", 2,
+      ImGuiTableFlags_Borders | ImGuiTableFlags_SizingStretchProp);
+  ImGui::TableSetupColumn("Name");
+  ImGui::TableSetupColumn("Type");
+  // ImGui::TableSetupColumn("Metadata");
+  ImGui::TableHeadersRow();
+  DisplayEntryTree(m_entryTree);
+  ImGui::EndTable();
+}
+
+void LogLoader::RebuildEntryTree() {
+  m_entryTree.clear();
+  wpi::SmallVector<std::string_view, 16> parts;
+  m_reader->ForEachEntryName([&](const glass::DataLogReaderEntry& entry) {
+    // only show double/float/string entries (TODO: support struct/protobuf)
+    if (entry.type != "double" && entry.type != "float" &&
+        entry.type != "string") {
+      return;
+    }
+
+    // filter on name
+    if (!m_filter.empty() && !wpi::contains_lower(entry.name, m_filter)) {
+      return;
+    }
+
+    parts.clear();
+    // split on first : if one is present
+    auto [prefix, mainpart] = wpi::split(entry.name, ':');
+    if (mainpart.empty() || wpi::contains(prefix, '/')) {
+      mainpart = entry.name;
+    } else {
+      parts.emplace_back(prefix);
+    }
+    wpi::split(mainpart, parts, '/', -1, false);
+
+    // ignore a raw "/" key
+    if (parts.empty()) {
+      return;
+    }
+
+    // get to leaf
+    auto nodes = &m_entryTree;
+    for (auto part : wpi::drop_back(std::span{parts.begin(), parts.end()})) {
+      auto it =
+          std::find_if(nodes->begin(), nodes->end(),
+                       [&](const auto& node) { return node.name == part; });
+      if (it == nodes->end()) {
+        nodes->emplace_back(part);
+        // path is from the beginning of the string to the end of the current
+        // part; this works because part is a reference to the internals of
+        // entry.name
+        nodes->back().path.assign(
+            entry.name.data(), part.data() + part.size() - entry.name.data());
+        it = nodes->end() - 1;
+      }
+      nodes = &it->children;
+    }
+
+    auto it = std::find_if(nodes->begin(), nodes->end(), [&](const auto& node) {
+      return node.name == parts.back();
+    });
+    if (it == nodes->end()) {
+      nodes->emplace_back(parts.back());
+      // no need to set path, as it's identical to entry.name
+      it = nodes->end() - 1;
+    }
+    it->entry = &entry;
+  });
+}
+
+static void EmitEntry(const std::string& name,
+                      const glass::DataLogReaderEntry& entry) {
+  ImGui::TableNextColumn();
+  ImGui::Selectable(name.c_str());
+  if (ImGui::BeginDragDropSource()) {
+    auto entryPtr = &entry;
+    ImGui::SetDragDropPayload(
+        entry.type == "string" ? "DataLogEntryString" : "DataLogEntry",
+        &entryPtr,
+        sizeof(entryPtr));  // NOLINT
+    ImGui::TextUnformatted(entry.name.data(),
+                           entry.name.data() + entry.name.size());
+    ImGui::EndDragDropSource();
+  }
+  ImGui::TableNextColumn();
+  ImGui::TextUnformatted(entry.type.data(),
+                         entry.type.data() + entry.type.size());
+#if 0
+  ImGui::TableNextColumn();
+  ImGui::TextUnformatted(entry.metadata.data(),
+                         entry.metadata.data() + entry.metadata.size());
+#endif
+}
+
+void LogLoader::DisplayEntryTree(const std::vector<EntryTreeNode>& tree) {
+  for (auto&& node : tree) {
+    if (node.entry) {
+      EmitEntry(node.name, *node.entry);
+    }
+
+    if (!node.children.empty()) {
+      ImGui::TableNextColumn();
+      bool open = ImGui::TreeNodeEx(node.name.c_str(),
+                                    ImGuiTreeNodeFlags_SpanFullWidth);
+      ImGui::TableNextColumn();
+#if 0
+      ImGui::TableNextColumn();
+#endif
+      if (open) {
+        DisplayEntryTree(node.children);
+        ImGui::TreePop();
+      }
+    }
+  }
+}

sysid/src/main/native/cpp/view/Logger.cpp

@@ -1,222 +0,0 @@
-// Copyright (c) FIRST and other WPILib contributors.
-// Open Source Software; you can modify and/or share it under the terms of
-// the WPILib BSD license file in the root directory of this project.
-
-#include "sysid/view/Logger.h"
-
-#include <exception>
-#include <numbers>
-
-#include <glass/Context.h>
-#include <glass/Storage.h>
-#include <imgui.h>
-#include <imgui_internal.h>
-#include <imgui_stdlib.h>
-#include <networktables/NetworkTable.h>
-#include <units/angle.h>
-#include <wpigui.h>
-
-#include "sysid/Util.h"
-#include "sysid/analysis/AnalysisType.h"
-#include "sysid/view/UILayout.h"
-
-using namespace sysid;
-
-Logger::Logger(glass::Storage& storage, wpi::Logger& logger)
-    : m_logger{logger}, m_ntSettings{"sysid", storage} {
-  wpi::gui::AddEarlyExecute([&] { m_ntSettings.Update(); });
-
-  m_ntSettings.EnableServerOption(false);
-}
-
-void Logger::Display() {
-  // Get the current width of the window. This will be used to scale
-  // our UI elements.
-  float width = ImGui::GetContentRegionAvail().x;
-
-  // Add team number input and apply button for NT connection.
-  m_ntSettings.Display();
-
-  // Reset and clear the internal manager state.
-  ImGui::SameLine();
-  if (ImGui::Button("Reset Telemetry")) {
-    m_settings = TelemetryManager::Settings{};
-    m_manager = std::make_unique<TelemetryManager>(m_settings, m_logger);
-    m_settings.mechanism = analysis::FromName(kTypes[m_selectedType]);
-  }
-
-  // Add NT connection indicator.
-  static ImVec4 kColorDisconnected{1.0f, 0.4f, 0.4f, 1.0f};
-  static ImVec4 kColorConnected{0.2f, 1.0f, 0.2f, 1.0f};
-  ImGui::SameLine();
-  bool ntConnected = nt::NetworkTableInstance::GetDefault().IsConnected();
-  ImGui::TextColored(ntConnected ? kColorConnected : kColorDisconnected,
-                     ntConnected ? "NT Connected" : "NT Disconnected");
-
-  // Create a Section for project configuration
-  ImGui::Separator();
-  ImGui::Spacing();
-  ImGui::Text("Project Parameters");
-
-  // Add a dropdown for mechanism type.
-  ImGui::SetNextItemWidth(ImGui::GetFontSize() * kTextBoxWidthMultiple);
-
-  if (ImGui::Combo("Mechanism", &m_selectedType, kTypes,
-                   IM_ARRAYSIZE(kTypes))) {
-    m_settings.mechanism = analysis::FromName(kTypes[m_selectedType]);
-  }
-
-  // Add Dropdown for Units
-  ImGui::SetNextItemWidth(ImGui::GetFontSize() * kTextBoxWidthMultiple);
-  if (ImGui::Combo("Unit Type", &m_selectedUnit, kUnits,
-                   IM_ARRAYSIZE(kUnits))) {
-    m_settings.units = kUnits[m_selectedUnit];
-  }
-
-  sysid::CreateTooltip(
-      "This is the type of units that your gains will be in. For example, if "
-      "you want your flywheel gains in terms of radians, then use the radians "
-      "unit. On the other hand, if your drivetrain will use gains in meters, "
-      "choose meters.");
-
-  // Rotational units have fixed Units per rotations
-  m_isRotationalUnits =
-      (m_settings.units == "Rotations" || m_settings.units == "Degrees" ||
-       m_settings.units == "Radians");
-  if (m_settings.units == "Degrees") {
-    m_settings.unitsPerRotation = 360.0;
-  } else if (m_settings.units == "Radians") {
-    m_settings.unitsPerRotation = 2 * std::numbers::pi;
-  } else if (m_settings.units == "Rotations") {
-    m_settings.unitsPerRotation = 1.0;
-  }
-
-  // Units Per Rotations entry
-  ImGui::SetNextItemWidth(ImGui::GetFontSize() * kTextBoxWidthMultiple);
-  ImGui::InputDouble("Units Per Rotation", &m_settings.unitsPerRotation, 0.0f,
-                     0.0f, "%.4f",
-                     m_isRotationalUnits ? ImGuiInputTextFlags_ReadOnly
-                                         : ImGuiInputTextFlags_None);
-  sysid::CreateTooltip(
-      "The logger assumes that the code will be sending recorded motor shaft "
-      "rotations over NetworkTables. This value will then be multiplied by the "
-      "units per rotation to get the measurement in the units you "
-      "specified.\n\nFor non-rotational units (e.g. meters), this value is "
-      "usually the wheel diameter times pi (should not include gearing).");
-  // Create a section for voltage parameters.
-  ImGui::Separator();
-  ImGui::Spacing();
-  ImGui::Text("Voltage Parameters");
-
-  auto CreateVoltageParameters = [this](const char* text, double* data,
-                                        float min, float max) {
-    ImGui::SetNextItemWidth(ImGui::GetFontSize() * 6);
-    ImGui::PushItemFlag(ImGuiItemFlags_Disabled,
-                        m_manager && m_manager->IsActive());
-    float value = static_cast<float>(*data);
-    if (ImGui::SliderFloat(text, &value, min, max, "%.2f")) {
-      *data = value;
-    }
-    ImGui::PopItemFlag();
-  };
-
-  CreateVoltageParameters("Quasistatic Ramp Rate (V/s)",
-                          &m_settings.quasistaticRampRate, 0.10f, 0.60f);
-  sysid::CreateTooltip(
-      "This is the rate at which the voltage will increase during the "
-      "quasistatic test.");
-
-  CreateVoltageParameters("Dynamic Step Voltage (V)", &m_settings.stepVoltage,
-                          0.0f, 10.0f);
-  sysid::CreateTooltip(
-      "This is the voltage that will be applied for the "
-      "dynamic voltage (acceleration) tests.");
-
-  // Create a section for tests.
-  ImGui::Separator();
-  ImGui::Spacing();
-  ImGui::Text("Tests");
-
-  auto CreateTest = [this, width](const char* text, const char* itext) {
-    // Display buttons if we have an NT connection.
-    if (nt::NetworkTableInstance::GetDefault().IsConnected()) {
-      // Create button to run tests.
-      if (ImGui::Button(text)) {
-        // Open the warning message.
-        ImGui::OpenPopup("Warning");
-        m_manager->BeginTest(itext);
-        m_opened = text;
-      }
-      if (m_opened == text && ImGui::BeginPopupModal("Warning")) {
-        ImGui::TextWrapped("%s", m_popupText.c_str());
-        if (ImGui::Button(m_manager->IsActive() ? "End Test" : "Close")) {
-          m_manager->EndTest();
-          ImGui::CloseCurrentPopup();
-          m_opened = "";
-        }
-        ImGui::EndPopup();
-      }
-    } else {
-      // Show disabled text when there is no connection.
-      ImGui::TextDisabled("%s", text);
-    }
-
-    // Show whether the tests were run or not.
-    bool run = m_manager->HasRunTest(itext);
-    ImGui::SameLine(width * 0.7);
-    ImGui::Text(run ? "Run" : "Not Run");
-  };
-
-  CreateTest("Quasistatic Forward", "slow-forward");
-  CreateTest("Quasistatic Backward", "slow-backward");
-  CreateTest("Dynamic Forward", "fast-forward");
-  CreateTest("Dynamic Backward", "fast-backward");
-
-  m_manager->RegisterDisplayCallback(
-      [this](const auto& str) { m_popupText = str; });
-
-  // Display the path to where the JSON will be saved and a button to select the
-  // location.
-  ImGui::Separator();
-  ImGui::Spacing();
-  ImGui::Text("Save Location");
-  if (ImGui::Button("Choose")) {
-    m_selector = std::make_unique<pfd::select_folder>("Select Folder");
-  }
-  ImGui::SameLine();
-  ImGui::InputText("##savelocation", &m_jsonLocation,
-                   ImGuiInputTextFlags_ReadOnly);
-
-  // Add button to save.
-  ImGui::SameLine(width * 0.9);
-  if (ImGui::Button("Save")) {
-    try {
-      m_manager->SaveJSON(m_jsonLocation);
-    } catch (const std::exception& e) {
-      ImGui::OpenPopup("Exception Caught!");
-      m_exception = e.what();
-    }
-  }
-
-  // Handle exceptions.
-  if (ImGui::BeginPopupModal("Exception Caught!")) {
-    ImGui::Text("%s", m_exception.c_str());
-    if (ImGui::Button("Close")) {
-      ImGui::CloseCurrentPopup();
-    }
-    ImGui::EndPopup();
-  }
-
-  // Run periodic methods.
-  SelectDataFolder();
-  m_ntSettings.Update();
-  m_manager->Update();
-}
-
-void Logger::SelectDataFolder() {
-  // If the selector exists and is ready with a result, we can store it.
-  if (m_selector && m_selector->ready()) {
-    m_jsonLocation = m_selector->result();
-    m_selector.reset();
-  }
-}