[wpiutil] Add high speed data logging

wpiutil/src/main/native/include/wpi/DataLog.h

@@ -0,0 +1,680 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#pragma once
+
+#include <stdint.h>
+
+#include <functional>
+#include <initializer_list>
+#include <memory>
+#include <string>
+#include <string_view>
+#include <thread>
+#include <vector>
+
+#include "wpi/DenseMap.h"
+#include "wpi/StringMap.h"
+#include "wpi/condition_variable.h"
+#include "wpi/mutex.h"
+#include "wpi/span.h"
+
+namespace wpi {
+class Logger;
+}  // namespace wpi
+
+namespace wpi::log {
+
+namespace impl {
+
+enum ControlRecordType {
+  kControlStart = 0,
+  kControlFinish,
+  kControlSetMetadata
+};
+
+}  // namespace impl
+
+/**
+ * A data log. The log file is created immediately upon construction with a
+ * temporary filename.  The file may be renamed at any time using the
+ * SetFilename() function.
+ *
+ * The lifetime of the data log object must be longer than any data log entry
+ * objects that refer to it.
+ *
+ * The data log is periodically flushed to disk.  It can also be explicitly
+ * flushed to disk by using the Flush() function.
+ */
+class DataLog final {
+ public:
+  /**
+   * Construct a new Data Log.  The log will be initially created with a
+   * temporary filename.
+   *
+   * @param dir directory to store the log
+   * @param filename filename to use; if none provided, a random filename is
+   *                 generated of the form "wpilog_{}.wpilog"
+   * @param period time between automatic flushes to disk, in seconds;
+   *               this is a time/storage tradeoff
+   * @param extraHeader extra header data
+   */
+  explicit DataLog(std::string_view dir = "", std::string_view filename = "",
+                   double period = 0.25, std::string_view extraHeader = "");
+
+  /**
+   * Construct a new Data Log.  The log will be initially created with a
+   * temporary filename.
+   *
+   * @param msglog message logger (will be called from separate thread)
+   * @param dir directory to store the log
+   * @param filename filename to use; if none provided, a random filename is
+   *                 generated of the form "wpilog_{}.wpilog"
+   * @param period time between automatic flushes to disk, in seconds;
+   *               this is a time/storage tradeoff
+   * @param extraHeader extra header data
+   */
+  explicit DataLog(wpi::Logger& msglog, std::string_view dir = "",
+                   std::string_view filename = "", double period = 0.25,
+                   std::string_view extraHeader = "");
+
+  /**
+   * Construct a new Data Log that passes its output to the provided function
+   * rather than a file.  The write function will be called on a separate
+   * background thread and may block.  The write function is called with an
+   * empty data array when the thread is terminating.
+   *
+   * @param write write function
+   * @param period time between automatic calls to write, in seconds;
+   *               this is a time/storage tradeoff
+   * @param extraHeader extra header data
+   */
+  explicit DataLog(std::function<void(wpi::span<const uint8_t> data)> write,
+                   double period = 0.25, std::string_view extraHeader = "");
+
+  /**
+   * Construct a new Data Log that passes its output to the provided function
+   * rather than a file.  The write function will be called on a separate
+   * background thread and may block.  The write function is called with an
+   * empty data array when the thread is terminating.
+   *
+   * @param msglog message logger (will be called from separate thread)
+   * @param write write function
+   * @param period time between automatic calls to write, in seconds;
+   *               this is a time/storage tradeoff
+   * @param extraHeader extra header data
+   */
+  explicit DataLog(wpi::Logger& msglog,
+                   std::function<void(wpi::span<const uint8_t> data)> write,
+                   double period = 0.25, std::string_view extraHeader = "");
+
+  ~DataLog();
+  DataLog(const DataLog&) = delete;
+  DataLog& operator=(const DataLog&) = delete;
+  DataLog(DataLog&&) = delete;
+  DataLog& operator=(const DataLog&&) = delete;
+
+  /**
+   * Change log filename.
+   *
+   * @param filename filename
+   */
+  void SetFilename(std::string_view filename);
+
+  /**
+   * Explicitly flushes the log data to disk.
+   */
+  void Flush();
+
+  /**
+   * Pauses appending of data records to the log.  While paused, no data records
+   * are saved (e.g. AppendX is a no-op).  Has no effect on entry starts /
+   * finishes / metadata changes.
+   */
+  void Pause();
+
+  /**
+   * Resumes appending of data records to the log.
+   */
+  void Resume();
+
+  /**
+   * Start an entry.  Duplicate names are allowed (with the same type), and
+   * result in the same index being returned (Start/Finish are reference
+   * counted).  A duplicate name with a different type will result in an error
+   * message being printed to the console and 0 being returned (which will be
+   * ignored by the Append functions).
+   *
+   * @param name Name
+   * @param type Data type
+   * @param metadata Initial metadata (e.g. data properties)
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   *
+   * @return Entry index
+   */
+  int Start(std::string_view name, std::string_view type,
+            std::string_view metadata = {}, int64_t timestamp = 0);
+
+  /**
+   * Finish an entry.
+   *
+   * @param entry Entry index
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void Finish(int entry, int64_t timestamp = 0);
+
+  /**
+   * Updates the metadata for an entry.
+   *
+   * @param entry Entry index
+   * @param metadata New metadata for the entry
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void SetMetadata(int entry, std::string_view metadata, int64_t timestamp = 0);
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param entry Entry index, as returned by Start()
+   * @param data Data to record
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void AppendRaw(int entry, wpi::span<const uint8_t> data, int64_t timestamp);
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param entry Entry index, as returned by Start()
+   * @param data Data to record
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void AppendRaw2(int entry, wpi::span<const wpi::span<const uint8_t>> data,
+                  int64_t timestamp);
+
+  void AppendBoolean(int entry, bool value, int64_t timestamp);
+  void AppendInteger(int entry, int64_t value, int64_t timestamp);
+  void AppendFloat(int entry, float value, int64_t timestamp);
+  void AppendDouble(int entry, double value, int64_t timestamp);
+  void AppendString(int entry, std::string_view value, int64_t timestamp);
+  void AppendBooleanArray(int entry, wpi::span<const bool> arr,
+                          int64_t timestamp);
+  void AppendBooleanArray(int entry, wpi::span<const int> arr,
+                          int64_t timestamp);
+  void AppendBooleanArray(int entry, wpi::span<const uint8_t> arr,
+                          int64_t timestamp);
+  void AppendIntegerArray(int entry, wpi::span<const int64_t> arr,
+                          int64_t timestamp);
+  void AppendFloatArray(int entry, wpi::span<const float> arr,
+                        int64_t timestamp);
+  void AppendDoubleArray(int entry, wpi::span<const double> arr,
+                         int64_t timestamp);
+  void AppendStringArray(int entry, wpi::span<const std::string> arr,
+                         int64_t timestamp);
+  void AppendStringArray(int entry, wpi::span<const std::string_view> arr,
+                         int64_t timestamp);
+
+ private:
+  void WriterThreadMain(std::string_view dir);
+  void WriterThreadMain(
+      std::function<void(wpi::span<const uint8_t> data)> write);
+
+  // must be called with m_mutex held
+  uint8_t* StartRecord(uint32_t entry, uint64_t timestamp, uint32_t payloadSize,
+                       size_t reserveSize);
+  uint8_t* Reserve(size_t size);
+  void AppendImpl(wpi::span<const uint8_t> data);
+  void AppendStringImpl(std::string_view str);
+
+  wpi::Logger& m_msglog;
+  mutable wpi::mutex m_mutex;
+  wpi::condition_variable m_cond;
+  bool m_active{true};
+  bool m_doFlush{false};
+  bool m_paused{false};
+  double m_period;
+  std::string m_extraHeader;
+  std::string m_newFilename;
+  class Buffer;
+  std::vector<Buffer> m_free;
+  std::vector<Buffer> m_outgoing;
+  struct EntryInfo {
+    std::string type;
+    int id{0};
+  };
+  wpi::StringMap<EntryInfo> m_entries;
+  wpi::DenseMap<int, unsigned int> m_entryCounts;
+  int m_lastId = 0;
+  std::thread m_thread;
+};
+
+/**
+ * Log entry base class.
+ */
+class DataLogEntry {
+ protected:
+  DataLogEntry() = default;
+  DataLogEntry(DataLog& log, std::string_view name, std::string_view type,
+               std::string_view metadata = {}, int64_t timestamp = 0)
+      : m_log{&log}, m_entry{log.Start(name, type, metadata, timestamp)} {}
+
+ public:
+  DataLogEntry(const DataLogEntry&) = delete;
+  DataLogEntry& operator=(const DataLogEntry&) = delete;
+
+  DataLogEntry(DataLogEntry&& rhs) : m_log{rhs.m_log}, m_entry{rhs.m_entry} {
+    rhs.m_log = nullptr;
+  }
+  DataLogEntry& operator=(DataLogEntry&& rhs) {
+    if (m_log) {
+      m_log->Finish(m_entry);
+    }
+    m_log = rhs.m_log;
+    rhs.m_log = nullptr;
+    m_entry = rhs.m_entry;
+    return *this;
+  }
+
+  explicit operator bool() const { return m_log != nullptr; }
+
+  /**
+   * Updates the metadata for the entry.
+   *
+   * @param metadata New metadata for the entry
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void SetMetadata(std::string_view metadata, int64_t timestamp = 0) {
+    m_log->SetMetadata(m_entry, metadata, timestamp);
+  }
+
+  /**
+   * Finishes the entry.
+   *
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void Finish(int64_t timestamp = 0) { m_log->Finish(m_entry, timestamp); }
+
+ protected:
+  DataLog* m_log = nullptr;
+  int m_entry = 0;
+};
+
+/**
+ * Log arbitrary byte data.
+ */
+class RawLogEntry : public DataLogEntry {
+ public:
+  static constexpr std::string_view kDataType = "raw";
+
+  RawLogEntry() = default;
+  RawLogEntry(DataLog& log, std::string_view name, int64_t timestamp = 0)
+      : RawLogEntry{log, name, {}, kDataType, timestamp} {}
+  RawLogEntry(DataLog& log, std::string_view name, std::string_view metadata,
+              int64_t timestamp = 0)
+      : RawLogEntry{log, name, metadata, kDataType, timestamp} {}
+  RawLogEntry(DataLog& log, std::string_view name, std::string_view metadata,
+              std::string_view type, int64_t timestamp = 0)
+      : DataLogEntry{log, name, type, metadata, timestamp} {}
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param data Data to record
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void Append(wpi::span<const uint8_t> data, int64_t timestamp = 0) {
+    m_log->AppendRaw(m_entry, data, timestamp);
+  }
+};
+
+/**
+ * Log boolean values.
+ */
+class BooleanLogEntry : public DataLogEntry {
+ public:
+  static constexpr std::string_view kDataType = "boolean";
+
+  BooleanLogEntry() = default;
+  BooleanLogEntry(DataLog& log, std::string_view name, int64_t timestamp = 0)
+      : BooleanLogEntry{log, name, {}, timestamp} {}
+  BooleanLogEntry(DataLog& log, std::string_view name,
+                  std::string_view metadata, int64_t timestamp = 0)
+      : DataLogEntry{log, name, kDataType, metadata, timestamp} {}
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param value Value to record
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void Append(bool value, int64_t timestamp = 0) {
+    m_log->AppendBoolean(m_entry, value, timestamp);
+  }
+};
+
+/**
+ * Log integer values.
+ */
+class IntegerLogEntry : public DataLogEntry {
+ public:
+  static constexpr std::string_view kDataType = "int64";
+
+  IntegerLogEntry() = default;
+  IntegerLogEntry(DataLog& log, std::string_view name, int64_t timestamp = 0)
+      : IntegerLogEntry{log, name, {}, timestamp} {}
+  IntegerLogEntry(DataLog& log, std::string_view name,
+                  std::string_view metadata, int64_t timestamp = 0)
+      : DataLogEntry{log, name, kDataType, metadata, timestamp} {}
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param value Value to record
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void Append(int64_t value, int64_t timestamp = 0) {
+    m_log->AppendInteger(m_entry, value, timestamp);
+  }
+};
+
+/**
+ * Log float values.
+ */
+class FloatLogEntry : public DataLogEntry {
+ public:
+  static constexpr std::string_view kDataType = "float";
+
+  FloatLogEntry() = default;
+  FloatLogEntry(DataLog& log, std::string_view name, int64_t timestamp = 0)
+      : FloatLogEntry{log, name, {}, timestamp} {}
+  FloatLogEntry(DataLog& log, std::string_view name, std::string_view metadata,
+                int64_t timestamp = 0)
+      : DataLogEntry{log, name, kDataType, metadata, timestamp} {}
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param value Value to record
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void Append(float value, int64_t timestamp = 0) {
+    m_log->AppendFloat(m_entry, value, timestamp);
+  }
+};
+
+/**
+ * Log double values.
+ */
+class DoubleLogEntry : public DataLogEntry {
+ public:
+  static constexpr std::string_view kDataType = "double";
+
+  DoubleLogEntry() = default;
+  DoubleLogEntry(DataLog& log, std::string_view name, int64_t timestamp = 0)
+      : DoubleLogEntry{log, name, {}, timestamp} {}
+  DoubleLogEntry(DataLog& log, std::string_view name, std::string_view metadata,
+                 int64_t timestamp = 0)
+      : DataLogEntry{log, name, kDataType, metadata, timestamp} {}
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param value Value to record
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void Append(double value, int64_t timestamp = 0) {
+    m_log->AppendDouble(m_entry, value, timestamp);
+  }
+};
+
+/**
+ * Log string values.
+ */
+class StringLogEntry : public DataLogEntry {
+ public:
+  static constexpr const char* kDataType = "string";
+
+  StringLogEntry() = default;
+  StringLogEntry(DataLog& log, std::string_view name, int64_t timestamp = 0)
+      : StringLogEntry{log, name, {}, kDataType, timestamp} {}
+  StringLogEntry(DataLog& log, std::string_view name, std::string_view metadata,
+                 int64_t timestamp = 0)
+      : StringLogEntry{log, name, metadata, kDataType, timestamp} {}
+  StringLogEntry(DataLog& log, std::string_view name, std::string_view metadata,
+                 std::string_view type, int64_t timestamp = 0)
+      : DataLogEntry{log, name, type, metadata, timestamp} {}
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param value Value to record
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void Append(std::string_view value, int64_t timestamp = 0) {
+    m_log->AppendString(m_entry, value, timestamp);
+  }
+};
+
+/**
+ * Log array of boolean values.
+ */
+class BooleanArrayLogEntry : public DataLogEntry {
+ public:
+  static constexpr const char* kDataType = "boolean[]";
+
+  BooleanArrayLogEntry() = default;
+  BooleanArrayLogEntry(DataLog& log, std::string_view name,
+                       int64_t timestamp = 0)
+      : BooleanArrayLogEntry{log, name, {}, timestamp} {}
+  BooleanArrayLogEntry(DataLog& log, std::string_view name,
+                       std::string_view metadata, int64_t timestamp = 0)
+      : DataLogEntry{log, name, kDataType, metadata, timestamp} {}
+
+  /**
+   * Appends a record to the log.  For find functions to work, timestamp
+   * must be monotonically increasing.
+   *
+   * @param arr Values to record
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void Append(wpi::span<const bool> arr, int64_t timestamp = 0) {
+    m_log->AppendBooleanArray(m_entry, arr, timestamp);
+  }
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param arr Values to record
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void Append(std::initializer_list<bool> arr, int64_t timestamp = 0) {
+    Append(wpi::span{arr.begin(), arr.end()}, timestamp);
+  }
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param arr Values to record
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void Append(wpi::span<const int> arr, int64_t timestamp = 0) {
+    m_log->AppendBooleanArray(m_entry, arr, timestamp);
+  }
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param arr Values to record
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void Append(std::initializer_list<int> arr, int64_t timestamp = 0) {
+    Append(wpi::span{arr.begin(), arr.end()}, timestamp);
+  }
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param arr Values to record
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void Append(wpi::span<const uint8_t> arr, int64_t timestamp = 0) {
+    m_log->AppendBooleanArray(m_entry, arr, timestamp);
+  }
+};
+
+/**
+ * Log array of integer values.
+ */
+class IntegerArrayLogEntry : public DataLogEntry {
+ public:
+  static constexpr const char* kDataType = "int64[]";
+
+  IntegerArrayLogEntry() = default;
+  IntegerArrayLogEntry(DataLog& log, std::string_view name,
+                       int64_t timestamp = 0)
+      : IntegerArrayLogEntry{log, name, {}, timestamp} {}
+  IntegerArrayLogEntry(DataLog& log, std::string_view name,
+                       std::string_view metadata, int64_t timestamp = 0)
+      : DataLogEntry{log, name, kDataType, metadata, timestamp} {}
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param arr Values to record
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void Append(wpi::span<const int64_t> arr, int64_t timestamp = 0) {
+    m_log->AppendIntegerArray(m_entry, arr, timestamp);
+  }
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param arr Values to record
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void Append(std::initializer_list<int64_t> arr, int64_t timestamp = 0) {
+    Append({arr.begin(), arr.end()}, timestamp);
+  }
+};
+
+/**
+ * Log array of float values.
+ */
+class FloatArrayLogEntry : public DataLogEntry {
+ public:
+  static constexpr const char* kDataType = "float[]";
+
+  FloatArrayLogEntry() = default;
+  FloatArrayLogEntry(DataLog& log, std::string_view name, int64_t timestamp = 0)
+      : FloatArrayLogEntry{log, name, {}, timestamp} {}
+  FloatArrayLogEntry(DataLog& log, std::string_view name,
+                     std::string_view metadata, int64_t timestamp = 0)
+      : DataLogEntry{log, name, kDataType, metadata, timestamp} {}
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param arr Values to record
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void Append(wpi::span<const float> arr, int64_t timestamp = 0) {
+    m_log->AppendFloatArray(m_entry, arr, timestamp);
+  }
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param arr Values to record
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void Append(std::initializer_list<float> arr, int64_t timestamp = 0) {
+    Append({arr.begin(), arr.end()}, timestamp);
+  }
+};
+
+/**
+ * Log array of double values.
+ */
+class DoubleArrayLogEntry : public DataLogEntry {
+ public:
+  static constexpr const char* kDataType = "double[]";
+
+  DoubleArrayLogEntry() = default;
+  DoubleArrayLogEntry(DataLog& log, std::string_view name,
+                      int64_t timestamp = 0)
+      : DoubleArrayLogEntry{log, name, {}, timestamp} {}
+  DoubleArrayLogEntry(DataLog& log, std::string_view name,
+                      std::string_view metadata, int64_t timestamp = 0)
+      : DataLogEntry{log, name, kDataType, metadata, timestamp} {}
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param arr Values to record
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void Append(wpi::span<const double> arr, int64_t timestamp = 0) {
+    m_log->AppendDoubleArray(m_entry, arr, timestamp);
+  }
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param arr Values to record
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void Append(std::initializer_list<double> arr, int64_t timestamp = 0) {
+    Append({arr.begin(), arr.end()}, timestamp);
+  }
+};
+
+/**
+ * Log array of string values.
+ */
+class StringArrayLogEntry : public DataLogEntry {
+ public:
+  static constexpr const char* kDataType = "string[]";
+
+  StringArrayLogEntry() = default;
+  StringArrayLogEntry(DataLog& log, std::string_view name,
+                      int64_t timestamp = 0)
+      : StringArrayLogEntry{log, name, {}, timestamp} {}
+  StringArrayLogEntry(DataLog& log, std::string_view name,
+                      std::string_view metadata, int64_t timestamp = 0)
+      : DataLogEntry{log, name, kDataType, metadata, timestamp} {}
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param arr Values to record
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void Append(wpi::span<const std::string> arr, int64_t timestamp = 0) {
+    m_log->AppendStringArray(m_entry, arr, timestamp);
+  }
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param arr Values to record
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void Append(wpi::span<const std::string_view> arr, int64_t timestamp = 0) {
+    m_log->AppendStringArray(m_entry, arr, timestamp);
+  }
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param arr Values to record
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void Append(std::initializer_list<std::string_view> arr,
+              int64_t timestamp = 0) {
+    Append(wpi::span<const std::string_view>{arr.begin(), arr.end()},
+           timestamp);
+  }
+};
+
+}  // namespace wpi::log

wpiutil/src/main/native/include/wpi/DataLogReader.h

@@ -0,0 +1,369 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#pragma once
+
+#include <stdint.h>
+
+#include <iterator>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "wpi/MemoryBuffer.h"
+#include "wpi/span.h"
+
+namespace wpi::log {
+
+/**
+ * Data contained in a start control record as created by DataLog::Start() when
+ * writing the log. This can be read by calling DataLogRecord::GetStartData().
+ */
+struct StartRecordData {
+  /** Entry ID; this will be used for this entry in future records. */
+  int entry;
+
+  /** Entry name. */
+  std::string_view name;
+
+  /** Type of the stored data for this entry, as a string, e.g. "double". */
+  std::string_view type;
+
+  /** Initial metadata. */
+  std::string_view metadata;
+};
+
+/**
+ * Data contained in a set metadata control record as created by
+ * DataLog::SetMetadata(). This can be read by calling
+ * DataLogRecord::GetSetMetadataData().
+ */
+struct MetadataRecordData {
+  /** Entry ID. */
+  int entry;
+
+  /** New metadata for the entry. */
+  std::string_view metadata;
+};
+
+/**
+ * A record in the data log. May represent either a control record (entry == 0)
+ * or a data record. Used only for reading (e.g. with DataLogReader).
+ */
+class DataLogRecord {
+ public:
+  DataLogRecord() = default;
+  DataLogRecord(int entry, int64_t timestamp, wpi::span<const uint8_t> data)
+      : m_timestamp{timestamp}, m_data{data}, m_entry{entry} {}
+
+  /**
+   * Gets the entry ID.
+   *
+   * @return entry ID
+   */
+  int GetEntry() const { return m_entry; }
+
+  /**
+   * Gets the record timestamp.
+   *
+   * @return Timestamp, in integer microseconds
+   */
+  int64_t GetTimestamp() const { return m_timestamp; }
+
+  /**
+   * Gets the size of the raw data.
+   *
+   * @return size
+   */
+  size_t GetSize() const { return m_data.size(); }
+
+  /**
+   * Gets the raw data. Use the GetX functions to decode based on the data type
+   * in the entry's start record.
+   */
+  wpi::span<const uint8_t> GetRaw() const { return m_data; }
+
+  /**
+   * Returns true if the record is a control record.
+   *
+   * @return True if control record, false if normal data record.
+   */
+  bool IsControl() const { return m_entry == 0; }
+
+  /**
+   * Returns true if the record is a start control record. Use GetStartData()
+   * to decode the contents.
+   *
+   * @return True if start control record, false otherwise.
+   */
+  bool IsStart() const;
+
+  /**
+   * Returns true if the record is a finish control record. Use GetFinishEntry()
+   * to decode the contents.
+   *
+   * @return True if finish control record, false otherwise.
+   */
+  bool IsFinish() const;
+
+  /**
+   * Returns true if the record is a set metadata control record. Use
+   * GetSetMetadataData() to decode the contents.
+   *
+   * @return True if set metadata control record, false otherwise.
+   */
+  bool IsSetMetadata() const;
+
+  /**
+   * Decodes a start control record.
+   *
+   * @param[out] out start record decoded data (if successful)
+   * @return True on success, false on error
+   */
+  bool GetStartData(StartRecordData* out) const;
+
+  /**
+   * Decodes a finish control record.
+   *
+   * @param[out] out finish record entry ID (if successful)
+   * @return True on success, false on error
+   */
+  bool GetFinishEntry(int* out) const;
+
+  /**
+   * Decodes a set metadata control record.
+   *
+   * @param[out] out set metadata record decoded data (if successful)
+   * @return True on success, false on error
+   */
+  bool GetSetMetadataData(MetadataRecordData* out) const;
+
+  /**
+   * Decodes a data record as a boolean. Note if the data type (as indicated in
+   * the corresponding start control record for this entry) is not "boolean",
+   * invalid results may be returned.
+   *
+   * @param[out] value boolean value (if successful)
+   * @return True on success, false on error
+   */
+  bool GetBoolean(bool* value) const;
+
+  /**
+   * Decodes a data record as an integer. Note if the data type (as indicated in
+   * the corresponding start control record for this entry) is not "int64",
+   * invalid results may be returned.
+   *
+   * @param[out] value integer value (if successful)
+   * @return True on success, false on error
+   */
+  bool GetInteger(int64_t* value) const;
+
+  /**
+   * Decodes a data record as a float. Note if the data type (as indicated in
+   * the corresponding start control record for this entry) is not "float",
+   * invalid results may be returned.
+   *
+   * @param[out] value float value (if successful)
+   * @return True on success, false on error
+   */
+  bool GetFloat(float* value) const;
+
+  /**
+   * Decodes a data record as a double. Note if the data type (as indicated in
+   * the corresponding start control record for this entry) is not "double",
+   * invalid results may be returned.
+   *
+   * @param[out] value double value (if successful)
+   * @return True on success, false on error
+   */
+  bool GetDouble(double* value) const;
+
+  /**
+   * Decodes a data record as a string. Note if the data type (as indicated in
+   * the corresponding start control record for this entry) is not "string",
+   * invalid results may be returned.
+   *
+   * @param[out] value string value
+   * @return True (never fails)
+   */
+  bool GetString(std::string_view* value) const;
+
+  /**
+   * Decodes a data record as a boolean array. Note if the data type (as
+   * indicated in the corresponding start control record for this entry) is not
+   * "boolean[]", invalid results may be returned.
+   *
+   * @param[out] arr boolean array
+   * @return True (never fails)
+   */
+  bool GetBooleanArray(std::vector<int>* arr) const;
+
+  /**
+   * Decodes a data record as an integer array. Note if the data type (as
+   * indicated in the corresponding start control record for this entry) is not
+   * "int64[]", invalid results may be returned.
+   *
+   * @param[out] arr integer array (if successful)
+   * @return True on success, false on error
+   */
+  bool GetIntegerArray(std::vector<int64_t>* arr) const;
+
+  /**
+   * Decodes a data record as a float array. Note if the data type (as
+   * indicated in the corresponding start control record for this entry) is not
+   * "float[]", invalid results may be returned.
+   *
+   * @param[out] arr float array (if successful)
+   * @return True on success, false on error
+   */
+  bool GetFloatArray(std::vector<float>* arr) const;
+
+  /**
+   * Decodes a data record as a double array. Note if the data type (as
+   * indicated in the corresponding start control record for this entry) is not
+   * "double[]", invalid results may be returned.
+   *
+   * @param[out] arr double array (if successful)
+   * @return True on success, false on error
+   */
+  bool GetDoubleArray(std::vector<double>* arr) const;
+
+  /**
+   * Decodes a data record as a string array. Note if the data type (as
+   * indicated in the corresponding start control record for this entry) is not
+   * "string[]", invalid results may be returned.
+   *
+   * @param[out] arr string array (if successful)
+   * @return True on success, false on error
+   */
+  bool GetStringArray(std::vector<std::string_view>* arr) const;
+
+ private:
+  int64_t m_timestamp{0};
+  wpi::span<const uint8_t> m_data;
+  int m_entry{-1};
+};
+
+class DataLogReader;
+
+/** DataLogReader iterator. */
+class DataLogIterator {
+ public:
+  using iterator_category = std::forward_iterator_tag;
+  using value_type = DataLogRecord;
+  using pointer = const value_type*;
+  using reference = const value_type&;
+
+  DataLogIterator(const DataLogReader* reader, size_t pos)
+      : m_reader{reader}, m_pos{pos} {}
+
+  bool operator==(const DataLogIterator& oth) const {
+    return m_reader == oth.m_reader && m_pos == oth.m_pos;
+  }
+  bool operator!=(const DataLogIterator& oth) const {
+    return !this->operator==(oth);
+  }
+
+  bool operator<(const DataLogIterator& oth) const { return m_pos < oth.m_pos; }
+  bool operator>(const DataLogIterator& oth) const {
+    return !this->operator<(oth) && !this->operator==(oth);
+  }
+  bool operator<=(const DataLogIterator& oth) const {
+    return !this->operator>(oth);
+  }
+  bool operator>=(const DataLogIterator& oth) const {
+    return !this->operator<(oth);
+  }
+
+  DataLogIterator& operator++();
+
+  DataLogIterator operator++(int) {
+    DataLogIterator tmp = *this;
+    ++*this;
+    return tmp;
+  }
+
+  reference operator*() const;
+
+  pointer operator->() const { return &this->operator*(); }
+
+ private:
+  const DataLogReader* m_reader;
+  size_t m_pos;
+  mutable bool m_valid = false;
+  mutable DataLogRecord m_value;
+};
+
+/** Data log reader (reads logs written by the DataLog class). */
+class DataLogReader {
+  friend class DataLogIterator;
+
+ public:
+  using iterator = DataLogIterator;
+
+  /** Constructs from a memory buffer. */
+  explicit DataLogReader(std::unique_ptr<MemoryBuffer> buffer);
+
+  /** Returns true if the data log is valid (e.g. has a valid header). */
+  explicit operator bool() const { return IsValid(); }
+
+  /** Returns true if the data log is valid (e.g. has a valid header). */
+  bool IsValid() const;
+
+  /**
+   * Gets the data log version. Returns 0 if data log is invalid.
+   *
+   * @return Version number; most significant byte is major, least significant
+   *         is minor (so version 1.0 will be 0x0100)
+   */
+  uint16_t GetVersion() const;
+
+  /**
+   * Gets the extra header data.
+   *
+   * @return Extra header data
+   */
+  std::string_view GetExtraHeader() const;
+
+  /**
+   * Gets the buffer identifier, typically the filename.
+   *
+   * @return Identifier string
+   */
+  std::string_view GetBufferIdentifier() const {
+    return m_buf ? m_buf->GetBufferIdentifier() : "Invalid";
+  }
+
+  /** Returns iterator to first record. */
+  iterator begin() const;
+
+  /** Returns end iterator. */
+  iterator end() const { return DataLogIterator{this, SIZE_MAX}; }
+
+ private:
+  std::unique_ptr<MemoryBuffer> m_buf;
+
+  bool GetRecord(size_t* pos, DataLogRecord* out) const;
+  bool GetNextRecord(size_t* pos) const;
+};
+
+inline DataLogIterator& DataLogIterator::operator++() {
+  if (!m_reader->GetNextRecord(&m_pos)) {
+    m_pos = SIZE_MAX;
+  }
+  m_valid = false;
+  return *this;
+}
+
+inline DataLogIterator::reference DataLogIterator::operator*() const {
+  if (!m_valid) {
+    size_t pos = m_pos;
+    if (m_reader->GetRecord(&pos, &m_value)) {
+      m_valid = true;
+    }
+  }
+  return m_value;
+}
+
+}  // namespace wpi::log