[wpiutil, ntcore] Add structured data support (#5391)

This adds support for two serialization formats for complex data types:

- Protobuf for complex objects with variable length internals that need forward and backward wire compatibility (lower speed, more flexible)
- Raw struct (ByteBuffer-style) for fixed-length objects (higher speed, less flexible)

Deserialization can be done either by creating a new object (for immutable objects) or overwriting the contents of an existing object (for mutable objects).

Implementing classes should provide inner classes that implement the Protobuf or Struct interface (in Java) or specialize the wpi::Protobuf or wpi::Struct struct (in C++). It is possible for classes to implement both. If the class itself does not implement serialization, it's possible for third parties/users to provide an implementation instead.

Uses the Google protobuf implementation for C++ and the QuickBuffers alternative protobuf implementation for Java.

wpiutil/src/main/java/edu/wpi/first/util/datalog/DataLog.java

@@ -4,7 +4,14 @@
 
 package edu.wpi.first.util.datalog;
 
+import edu.wpi.first.util.WPIUtilJNI;
+import edu.wpi.first.util.protobuf.Protobuf;
+import edu.wpi.first.util.struct.Struct;
 import java.nio.ByteBuffer;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 
 /**
  * A data log. The log file is created immediately upon construction with a temporary filename. The
@@ -104,6 +111,130 @@ public final class DataLog implements AutoCloseable {
     DataLogJNI.resume(m_impl);
   }
 
+  /**
+   * Returns whether there is a data schema already registered with the given name.
+   *
+   * @param name Name (the string passed as the data type for records using this schema)
+   * @return True if schema already registered
+   */
+  public boolean hasSchema(String name) {
+    return m_schemaSet.contains(name);
+  }
+
+  /**
+   * Registers a data schema. Data schemas provide information for how a certain data type string
+   * can be decoded. The type string of a data schema indicates the type of the schema itself (e.g.
+   * "protobuf" for protobuf schemas, "struct" for struct schemas, etc). In the data log, schemas
+   * are saved just like normal records, with the name being generated from the provided name:
+   * "/.schema/name". Duplicate calls to this function with the same name are silently ignored.
+   *
+   * @param name Name (the string passed as the data type for records using this schema)
+   * @param type Type of schema (e.g. "protobuf", "struct", etc)
+   * @param schema Schema data
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  public void addSchema(String name, String type, byte[] schema, long timestamp) {
+    if (!m_schemaSet.add(name)) {
+      return;
+    }
+    DataLogJNI.addSchema(m_impl, name, type, schema, timestamp);
+  }
+
+  /**
+   * Registers a data schema. Data schemas provide information for how a certain data type string
+   * can be decoded. The type string of a data schema indicates the type of the schema itself (e.g.
+   * "protobuf" for protobuf schemas, "struct" for struct schemas, etc). In the data log, schemas
+   * are saved just like normal records, with the name being generated from the provided name:
+   * "/.schema/name". Duplicate calls to this function with the same name are silently ignored.
+   *
+   * @param name Name (the string passed as the data type for records using this schema)
+   * @param type Type of schema (e.g. "protobuf", "struct", etc)
+   * @param schema Schema data
+   */
+  public void addSchema(String name, String type, byte[] schema) {
+    addSchema(name, type, schema, 0);
+  }
+
+  /**
+   * Registers a data schema. Data schemas provide information for how a certain data type string
+   * can be decoded. The type string of a data schema indicates the type of the schema itself (e.g.
+   * "protobuf" for protobuf schemas, "struct" for struct schemas, etc). In the data log, schemas
+   * are saved just like normal records, with the name being generated from the provided name:
+   * "/.schema/name". Duplicate calls to this function with the same name are silently ignored.
+   *
+   * @param name Name (the string passed as the data type for records using this schema)
+   * @param type Type of schema (e.g. "protobuf", "struct", etc)
+   * @param schema Schema data
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  public void addSchema(String name, String type, String schema, long timestamp) {
+    if (!m_schemaSet.add(name)) {
+      return;
+    }
+    DataLogJNI.addSchemaString(m_impl, name, type, schema, timestamp);
+  }
+
+  /**
+   * Registers a data schema. Data schemas provide information for how a certain data type string
+   * can be decoded. The type string of a data schema indicates the type of the schema itself (e.g.
+   * "protobuf" for protobuf schemas, "struct" for struct schemas, etc). In the data log, schemas
+   * are saved just like normal records, with the name being generated from the provided name:
+   * "/.schema/name". Duplicate calls to this function with the same name are silently ignored.
+   *
+   * @param name Name (the string passed as the data type for records using this schema)
+   * @param type Type of schema (e.g. "protobuf", "struct", etc)
+   * @param schema Schema data
+   */
+  public void addSchema(String name, String type, String schema) {
+    addSchema(name, type, schema, 0);
+  }
+
+  /**
+   * Registers a protobuf schema. Duplicate calls to this function with the same name are silently
+   * ignored.
+   *
+   * @param proto protobuf serialization object
+   * @param timestamp Time stamp (0 to indicate now)
+   */
+  public void addSchema(Protobuf<?, ?> proto, long timestamp) {
+    final long actualTimestamp = timestamp == 0 ? WPIUtilJNI.now() : timestamp;
+    proto.forEachDescriptor(
+        this::hasSchema,
+        (typeString, schema) ->
+            addSchema(typeString, "proto:FileDescriptorProto", schema, actualTimestamp));
+  }
+
+  /**
+   * Registers a protobuf schema. Duplicate calls to this function with the same name are silently
+   * ignored.
+   *
+   * @param proto protobuf serialization object
+   */
+  public void addSchema(Protobuf<?, ?> proto) {
+    addSchema(proto, 0);
+  }
+
+  /**
+   * Registers a struct schema. Duplicate calls to this function with the same name are silently
+   * ignored.
+   *
+   * @param struct struct serialization object
+   * @param timestamp Time stamp (0 to indicate now)
+   */
+  public void addSchema(Struct<?> struct, long timestamp) {
+    addSchemaImpl(struct, timestamp == 0 ? WPIUtilJNI.now() : timestamp, new HashSet<>());
+  }
+
+  /**
+   * Registers a struct schema. Duplicate calls to this function with the same name are silently
+   * ignored.
+   *
+   * @param struct struct serialization object
+   */
+  public void addSchema(Struct<?> struct) {
+    addSchema(struct, 0);
+  }
+
   /**
    * Start an entry. Duplicate names are allowed (with the same type), and result in the same index
    * being returned (start/finish are reference counted). A duplicate name with a different type
@@ -358,5 +489,22 @@ public final class DataLog implements AutoCloseable {
     return m_impl;
   }
 
+  private void addSchemaImpl(Struct<?> struct, long timestamp, Set<String> seen) {
+    String typeString = struct.getTypeString();
+    if (hasSchema(typeString)) {
+      return;
+    }
+    if (!seen.add(typeString)) {
+      throw new UnsupportedOperationException(typeString + ": circular reference with " + seen);
+    }
+    addSchema(typeString, "structschema", struct.getSchema(), timestamp);
+    for (Struct<?> inner : struct.getNested()) {
+      addSchemaImpl(inner, timestamp, seen);
+    }
+    seen.remove(typeString);
+  }
+
   private long m_impl;
+  private final ConcurrentMap<String, Integer> m_schemaMap = new ConcurrentHashMap<>();
+  private final Set<String> m_schemaSet = m_schemaMap.keySet();
 }

wpiutil/src/main/java/edu/wpi/first/util/datalog/DataLogJNI.java

@@ -18,6 +18,11 @@ public class DataLogJNI extends WPIUtilJNI {
 
   static native void resume(long impl);
 
+  static native void addSchema(long impl, String name, String type, byte[] schema, long timestamp);
+
+  static native void addSchemaString(
+      long impl, String name, String type, String schema, long timestamp);
+
   static native int start(long impl, String name, String type, String metadata, long timestamp);
 
   static native void finish(long impl, int entry, long timestamp);

wpiutil/src/main/java/edu/wpi/first/util/datalog/ProtobufLogEntry.java

@@ -0,0 +1,117 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+package edu.wpi.first.util.datalog;
+
+import edu.wpi.first.util.protobuf.Protobuf;
+import edu.wpi.first.util.protobuf.ProtobufBuffer;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import us.hebi.quickbuf.ProtoMessage;
+
+/**
+ * Log protobuf-encoded values.
+ *
+ * @param <T> value class
+ */
+public final class ProtobufLogEntry<T> extends DataLogEntry {
+  private ProtobufLogEntry(
+      DataLog log, String name, Protobuf<T, ?> proto, String metadata, long timestamp) {
+    super(log, name, proto.getTypeString(), metadata, timestamp);
+    m_buf = ProtobufBuffer.create(proto);
+    log.addSchema(proto, timestamp);
+  }
+
+  /**
+   * Creates a protobuf-encoded log entry.
+   *
+   * @param <T> value class (inferred from proto)
+   * @param <MessageType> protobuf message type (inferred from proto)
+   * @param log datalog
+   * @param name name of the entry
+   * @param proto protobuf serialization implementation
+   * @param metadata metadata
+   * @param timestamp entry creation timestamp (0=now)
+   * @return ProtobufLogEntry
+   */
+  public static <T, MessageType extends ProtoMessage<?>> ProtobufLogEntry<T> create(
+      DataLog log, String name, Protobuf<T, MessageType> proto, String metadata, long timestamp) {
+    return new ProtobufLogEntry<T>(log, name, proto, metadata, timestamp);
+  }
+
+  /**
+   * Creates a protobuf-encoded log entry.
+   *
+   * @param <T> value class (inferred from proto)
+   * @param <MessageType> protobuf message type (inferred from proto)
+   * @param log datalog
+   * @param name name of the entry
+   * @param proto protobuf serialization implementation
+   * @param metadata metadata
+   * @return ProtobufLogEntry
+   */
+  public static <T, MessageType extends ProtoMessage<?>> ProtobufLogEntry<T> create(
+      DataLog log, String name, Protobuf<T, MessageType> proto, String metadata) {
+    return create(log, name, proto, metadata, 0);
+  }
+
+  /**
+   * Creates a protobuf-encoded log entry.
+   *
+   * @param <T> value class (inferred from proto)
+   * @param <MessageType> protobuf message type (inferred from proto)
+   * @param log datalog
+   * @param name name of the entry
+   * @param proto protobuf serialization implementation
+   * @param timestamp entry creation timestamp (0=now)
+   * @return ProtobufLogEntry
+   */
+  public static <T, MessageType extends ProtoMessage<?>> ProtobufLogEntry<T> create(
+      DataLog log, String name, Protobuf<T, MessageType> proto, long timestamp) {
+    return create(log, name, proto, "", timestamp);
+  }
+
+  /**
+   * Creates a protobuf-encoded log entry.
+   *
+   * @param <T> value class (inferred from proto)
+   * @param <MessageType> protobuf message type (inferred from proto)
+   * @param log datalog
+   * @param name name of the entry
+   * @param proto protobuf serialization implementation
+   * @return ProtobufLogEntry
+   */
+  public static <T, MessageType extends ProtoMessage<?>> ProtobufLogEntry<T> create(
+      DataLog log, String name, Protobuf<T, MessageType> proto) {
+    return create(log, name, proto, 0);
+  }
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param value Value to record
+   * @param timestamp Time stamp (0 to indicate now)
+   */
+  public void append(T value, long timestamp) {
+    try {
+      synchronized (m_buf) {
+        ByteBuffer bb = m_buf.write(value);
+        m_log.appendRaw(m_entry, bb, 0, bb.position(), timestamp);
+      }
+    } catch (IOException e) {
+      // ignore
+    }
+  }
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param value Value to record
+   */
+  public void append(T value) {
+    append(value, 0);
+  }
+
+  private final ProtobufBuffer<T, ?> m_buf;
+}

wpiutil/src/main/java/edu/wpi/first/util/datalog/StructArrayLogEntry.java

@@ -0,0 +1,140 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+package edu.wpi.first.util.datalog;
+
+import edu.wpi.first.util.struct.Struct;
+import edu.wpi.first.util.struct.StructBuffer;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+
+/**
+ * Log struct-encoded array values.
+ *
+ * @param <T> value class
+ */
+public final class StructArrayLogEntry<T> extends DataLogEntry {
+  private StructArrayLogEntry(
+      DataLog log, String name, Struct<T> struct, String metadata, long timestamp) {
+    super(log, name, struct.getTypeString() + "[]", metadata, timestamp);
+    m_buf = StructBuffer.create(struct);
+    log.addSchema(struct, timestamp);
+  }
+
+  /**
+   * Creates a struct-encoded array log entry.
+   *
+   * @param <T> value class (inferred from struct)
+   * @param log datalog
+   * @param name name of the entry
+   * @param struct struct serialization implementation
+   * @param metadata metadata
+   * @param timestamp entry creation timestamp (0=now)
+   * @return StructArrayLogEntry
+   */
+  public static <T> StructArrayLogEntry<T> create(
+      DataLog log, String name, Struct<T> struct, String metadata, long timestamp) {
+    return new StructArrayLogEntry<T>(log, name, struct, metadata, timestamp);
+  }
+
+  /**
+   * Creates a struct-encoded array log entry.
+   *
+   * @param <T> value class (inferred from struct)
+   * @param log datalog
+   * @param name name of the entry
+   * @param struct struct serialization implementation
+   * @param metadata metadata
+   * @return StructArrayLogEntry
+   */
+  public static <T> StructArrayLogEntry<T> create(
+      DataLog log, String name, Struct<T> struct, String metadata) {
+    return create(log, name, struct, metadata, 0);
+  }
+
+  /**
+   * Creates a struct-encoded array log entry.
+   *
+   * @param <T> value class (inferred from struct)
+   * @param log datalog
+   * @param name name of the entry
+   * @param struct struct serialization implementation
+   * @param timestamp entry creation timestamp (0=now)
+   * @return StructArrayLogEntry
+   */
+  public static <T> StructArrayLogEntry<T> create(
+      DataLog log, String name, Struct<T> struct, long timestamp) {
+    return create(log, name, struct, "", timestamp);
+  }
+
+  /**
+   * Creates a struct-encoded array log entry.
+   *
+   * @param <T> value class (inferred from struct)
+   * @param log datalog
+   * @param name name of the entry
+   * @param struct struct serialization implementation
+   * @return StructArrayLogEntry
+   */
+  public static <T> StructArrayLogEntry<T> create(DataLog log, String name, Struct<T> struct) {
+    return create(log, name, struct, 0);
+  }
+
+  /**
+   * Ensures sufficient buffer space is available for the given number of elements.
+   *
+   * @param nelem number of elements
+   */
+  public void reserve(int nelem) {
+    synchronized (m_buf) {
+      m_buf.reserve(nelem);
+    }
+  }
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param value Value to record
+   * @param timestamp Time stamp (0 to indicate now)
+   */
+  public void append(T[] value, long timestamp) {
+    synchronized (this) {
+      ByteBuffer bb = m_buf.writeArray(value);
+      m_log.appendRaw(m_entry, bb, 0, bb.position(), timestamp);
+    }
+  }
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param value Value to record
+   */
+  public void append(T[] value) {
+    append(value, 0);
+  }
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param value Value to record
+   * @param timestamp Time stamp (0 to indicate now)
+   */
+  public void append(Collection<T> value, long timestamp) {
+    synchronized (m_buf) {
+      ByteBuffer bb = m_buf.writeArray(value);
+      m_log.appendRaw(m_entry, bb, 0, bb.position(), timestamp);
+    }
+  }
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param value Value to record
+   */
+  public void append(Collection<T> value) {
+    append(value, 0);
+  }
+
+  private final StructBuffer<T> m_buf;
+}

wpiutil/src/main/java/edu/wpi/first/util/datalog/StructLogEntry.java

@@ -0,0 +1,106 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+package edu.wpi.first.util.datalog;
+
+import edu.wpi.first.util.struct.Struct;
+import edu.wpi.first.util.struct.StructBuffer;
+import java.nio.ByteBuffer;
+
+/**
+ * Log struct-encoded values.
+ *
+ * @param <T> value class
+ */
+public final class StructLogEntry<T> extends DataLogEntry {
+  private StructLogEntry(
+      DataLog log, String name, Struct<T> struct, String metadata, long timestamp) {
+    super(log, name, struct.getTypeString(), metadata, timestamp);
+    m_buf = StructBuffer.create(struct);
+    log.addSchema(struct, timestamp);
+  }
+
+  /**
+   * Creates a struct-encoded log entry.
+   *
+   * @param <T> value class (inferred from struct)
+   * @param log datalog
+   * @param name name of the entry
+   * @param struct struct serialization implementation
+   * @param metadata metadata
+   * @param timestamp entry creation timestamp (0=now)
+   * @return StructLogEntry
+   */
+  public static <T> StructLogEntry<T> create(
+      DataLog log, String name, Struct<T> struct, String metadata, long timestamp) {
+    return new StructLogEntry<T>(log, name, struct, metadata, timestamp);
+  }
+
+  /**
+   * Creates a struct-encoded log entry.
+   *
+   * @param <T> value class (inferred from struct)
+   * @param log datalog
+   * @param name name of the entry
+   * @param struct struct serialization implementation
+   * @param metadata metadata
+   * @return StructLogEntry
+   */
+  public static <T> StructLogEntry<T> create(
+      DataLog log, String name, Struct<T> struct, String metadata) {
+    return create(log, name, struct, metadata, 0);
+  }
+
+  /**
+   * Creates a struct-encoded log entry.
+   *
+   * @param <T> value class (inferred from struct)
+   * @param log datalog
+   * @param name name of the entry
+   * @param struct struct serialization implementation
+   * @param timestamp entry creation timestamp (0=now)
+   * @return StructLogEntry
+   */
+  public static <T> StructLogEntry<T> create(
+      DataLog log, String name, Struct<T> struct, long timestamp) {
+    return create(log, name, struct, "", timestamp);
+  }
+
+  /**
+   * Creates a struct-encoded log entry.
+   *
+   * @param <T> value class (inferred from struct)
+   * @param log datalog
+   * @param name name of the entry
+   * @param struct struct serialization implementation
+   * @return StructLogEntry
+   */
+  public static <T> StructLogEntry<T> create(DataLog log, String name, Struct<T> struct) {
+    return create(log, name, struct, 0);
+  }
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param value Value to record
+   * @param timestamp Time stamp (0 to indicate now)
+   */
+  public void append(T value, long timestamp) {
+    synchronized (m_buf) {
+      ByteBuffer bb = m_buf.write(value);
+      m_log.appendRaw(m_entry, bb, 0, bb.position(), timestamp);
+    }
+  }
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param value Value to record
+   */
+  public void append(T value) {
+    append(value, 0);
+  }
+
+  private final StructBuffer<T> m_buf;
+}

wpiutil/src/main/java/edu/wpi/first/util/protobuf/Protobuf.java

@@ -0,0 +1,121 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+package edu.wpi.first.util.protobuf;
+
+import java.util.function.BiConsumer;
+import java.util.function.Predicate;
+import us.hebi.quickbuf.Descriptors.Descriptor;
+import us.hebi.quickbuf.Descriptors.FileDescriptor;
+import us.hebi.quickbuf.ProtoMessage;
+
+/**
+ * Interface for Protobuf serialization.
+ *
+ * <p>This is designed for serialization of more complex data structures including forward/backwards
+ * compatibility and repeated/nested/variable length members, etc. Serialization and deserialization
+ * code is auto-generated from .proto interface descriptions (the MessageType generic parameter).
+ *
+ * <p>Idiomatically, classes that support protobuf serialization should provide a static final
+ * member named "proto" that provides an instance of an implementation of this interface.
+ *
+ * @param <T> object type
+ * @param <MessageType> protobuf message type
+ */
+public interface Protobuf<T, MessageType extends ProtoMessage<?>> {
+  /**
+   * Gets the Class object for the stored value.
+   *
+   * @return Class
+   */
+  Class<T> getTypeClass();
+
+  /**
+   * Gets the type string (e.g. for NetworkTables). This should be globally unique and start with
+   * "proto:".
+   *
+   * @return type string
+   */
+  default String getTypeString() {
+    return "proto:" + getDescriptor().getFullName();
+  }
+
+  /**
+   * Gets the protobuf descriptor.
+   *
+   * @return descriptor
+   */
+  Descriptor getDescriptor();
+
+  /**
+   * Gets the list of protobuf types referenced by this protobuf.
+   *
+   * @return list of protobuf types
+   */
+  default Protobuf<?, ?>[] getNested() {
+    return new Protobuf<?, ?>[] {};
+  }
+
+  /**
+   * Creates protobuf message.
+   *
+   * @return protobuf message
+   */
+  MessageType createMessage();
+
+  /**
+   * Deserializes an object from a protobuf message.
+   *
+   * @param msg protobuf message
+   * @return New object
+   */
+  T unpack(MessageType msg);
+
+  /**
+   * Copies the object contents into a protobuf message. Implementations should call either
+   * msg.setMember(member) or member.copyToProto(msg.getMutableMember()) for each member.
+   *
+   * @param msg protobuf message
+   * @param value object to serialize
+   */
+  void pack(MessageType msg, T value);
+
+  /**
+   * Updates the object contents from a protobuf message. Implementations should call
+   * msg.getMember(member), MemberClass.makeFromProto(msg.getMember()), or
+   * member.updateFromProto(msg.getMember()) for each member.
+   *
+   * <p>Immutable classes cannot and should not implement this function. The default implementation
+   * throws UnsupportedOperationException.
+   *
+   * @param out object to update
+   * @param msg protobuf message
+   * @throws UnsupportedOperationException if the object is immutable
+   */
+  default void unpackInto(T out, MessageType msg) {
+    throw new UnsupportedOperationException("object does not support unpackInto");
+  }
+
+  /**
+   * Loops over all protobuf descriptors including nested/referenced descriptors.
+   *
+   * @param exists function that returns false if fn should be called for the given type string
+   * @param fn function to call for each descriptor
+   */
+  default void forEachDescriptor(Predicate<String> exists, BiConsumer<String, byte[]> fn) {
+    forEachDescriptorImpl(getDescriptor().getFile(), exists, fn);
+  }
+
+  private static void forEachDescriptorImpl(
+      FileDescriptor desc, Predicate<String> exists, BiConsumer<String, byte[]> fn) {
+    String name = "proto:" + desc.getFullName();
+    if (exists.test(name)) {
+      return;
+    }
+    for (FileDescriptor dep : desc.getDependencies()) {
+      forEachDescriptorImpl(dep, exists, fn);
+    }
+    fn.accept(name, desc.toProtoBytes());
+  }
+}

wpiutil/src/main/java/edu/wpi/first/util/protobuf/ProtobufBuffer.java

@@ -0,0 +1,164 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+package edu.wpi.first.util.protobuf;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import us.hebi.quickbuf.ProtoMessage;
+import us.hebi.quickbuf.ProtoSink;
+import us.hebi.quickbuf.ProtoSource;
+
+/**
+ * Reusable buffer for serialization/deserialization to/from a protobuf.
+ *
+ * @param <T> object type
+ * @param <MessageType> protobuf message type
+ */
+public final class ProtobufBuffer<T, MessageType extends ProtoMessage<?>> {
+  private ProtobufBuffer(Protobuf<T, MessageType> proto) {
+    m_buf = ByteBuffer.allocateDirect(1024);
+    m_sink = ProtoSink.newDirectSink();
+    m_sink.setOutput(m_buf);
+    m_source = ProtoSource.newDirectSource();
+    m_msg = proto.createMessage();
+    m_proto = proto;
+  }
+
+  public static <T, MessageType extends ProtoMessage<?>> ProtobufBuffer<T, MessageType> create(
+      Protobuf<T, MessageType> proto) {
+    return new ProtobufBuffer<T, MessageType>(proto);
+  }
+
+  /**
+   * Gets the protobuf object of the stored type.
+   *
+   * @return protobuf object
+   */
+  public Protobuf<T, MessageType> getProto() {
+    return m_proto;
+  }
+
+  /**
+   * Gets the type string.
+   *
+   * @return type string
+   */
+  public String getTypeString() {
+    return m_proto.getTypeString();
+  }
+
+  /**
+   * Serializes a value to a ByteBuffer. The returned ByteBuffer is a direct byte buffer with the
+   * position set to the end of the serialized data.
+   *
+   * @param value value
+   * @return byte buffer
+   * @throws IOException if serialization failed
+   */
+  public ByteBuffer write(T value) throws IOException {
+    m_msg.clearQuick();
+    m_proto.pack(m_msg, value);
+    int size = m_msg.getSerializedSize();
+    if (size < m_buf.capacity()) {
+      m_buf = ByteBuffer.allocateDirect(size * 2);
+      m_sink.setOutput(m_buf);
+    }
+    m_sink.reset();
+    m_msg.writeTo(m_sink);
+    m_buf.position(m_sink.getTotalBytesWritten());
+    return m_buf;
+  }
+
+  /**
+   * Deserializes a value from a byte array, creating a new object.
+   *
+   * @param buf byte array
+   * @param start starting location within byte array
+   * @param len length of serialized data
+   * @return new object
+   * @throws IOException if deserialization failed
+   */
+  public T read(byte[] buf, int start, int len) throws IOException {
+    m_msg.clearQuick();
+    m_source.setInput(buf, start, len);
+    m_msg.mergeFrom(m_source);
+    return m_proto.unpack(m_msg);
+  }
+
+  /**
+   * Deserializes a value from a byte array, creating a new object.
+   *
+   * @param buf byte array
+   * @return new object
+   * @throws IOException if deserialization failed
+   */
+  public T read(byte[] buf) throws IOException {
+    return read(buf, 0, buf.length);
+  }
+
+  /**
+   * Deserializes a value from a ByteBuffer, creating a new object.
+   *
+   * @param buf byte buffer
+   * @return new object
+   * @throws IOException if deserialization failed
+   */
+  public T read(ByteBuffer buf) throws IOException {
+    m_msg.clearQuick();
+    m_source.setInput(buf);
+    m_msg.mergeFrom(m_source);
+    return m_proto.unpack(m_msg);
+  }
+
+  /**
+   * Deserializes a value from a byte array into a mutable object.
+   *
+   * @param out object (will be updated with deserialized contents)
+   * @param buf byte array
+   * @param start starting location within byte array
+   * @param len length of serialized data
+   * @throws IOException if deserialization failed
+   * @throws UnsupportedOperationException if the object is immutable
+   */
+  public void readInto(T out, byte[] buf, int start, int len) throws IOException {
+    m_msg.clearQuick();
+    m_source.setInput(buf, start, len);
+    m_msg.mergeFrom(m_source);
+    m_proto.unpackInto(out, m_msg);
+  }
+
+  /**
+   * Deserializes a value from a byte array into a mutable object.
+   *
+   * @param out object (will be updated with deserialized contents)
+   * @param buf byte array
+   * @throws IOException if deserialization failed
+   * @throws UnsupportedOperationException if the object is immutable
+   */
+  public void readInto(T out, byte[] buf) throws IOException {
+    readInto(out, buf, 0, buf.length);
+  }
+
+  /**
+   * Deserializes a value from a ByteBuffer into a mutable object.
+   *
+   * @param out object (will be updated with deserialized contents)
+   * @param buf byte buffer
+   * @throws IOException if deserialization failed
+   * @throws UnsupportedOperationException if the object is immutable
+   */
+  public void readInto(T out, ByteBuffer buf) throws IOException {
+    m_msg.clearQuick();
+    m_source.setInput(buf);
+    m_msg.mergeFrom(m_source);
+    m_proto.unpackInto(out, m_msg);
+  }
+
+  private ByteBuffer m_buf;
+  private final ProtoSink m_sink;
+  private final ProtoSource m_source;
+  private final MessageType m_msg;
+  private final Protobuf<T, MessageType> m_proto;
+}

wpiutil/src/main/java/edu/wpi/first/util/struct/BadSchemaException.java

@@ -0,0 +1,43 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+package edu.wpi.first.util.struct;
+
+public class BadSchemaException extends Exception {
+  private final String m_field;
+
+  public BadSchemaException(String s) {
+    super(s);
+    m_field = "";
+  }
+
+  public BadSchemaException(String message, Throwable cause) {
+    super(message, cause);
+    m_field = "";
+  }
+
+  public BadSchemaException(Throwable cause) {
+    super(cause);
+    m_field = "";
+  }
+
+  public BadSchemaException(String field, String s) {
+    super(s);
+    m_field = field;
+  }
+
+  public BadSchemaException(String field, String message, Throwable cause) {
+    super(message, cause);
+    m_field = field;
+  }
+
+  public String getField() {
+    return m_field;
+  }
+
+  @Override
+  public String toString() {
+    return m_field.isEmpty() ? getMessage() : "field " + m_field + ": " + getMessage();
+  }
+}

wpiutil/src/main/java/edu/wpi/first/util/struct/DynamicStruct.java

@@ -0,0 +1,632 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+package edu.wpi.first.util.struct;
+
+import java.nio.BufferUnderflowException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.ReadOnlyBufferException;
+import java.nio.charset.StandardCharsets;
+
+/** Dynamic (run-time) access to a serialized raw struct. */
+public final class DynamicStruct {
+  private DynamicStruct(StructDescriptor desc, ByteBuffer data) {
+    m_desc = desc;
+    m_data = data.order(ByteOrder.LITTLE_ENDIAN);
+  }
+
+  /**
+   * Constructs a new dynamic struct object with internal storage. The descriptor must be valid. The
+   * internal storage is allocated using ByteBuffer.allocate().
+   *
+   * @param desc struct descriptor
+   * @return dynamic struct object
+   * @throws IllegalStateException if struct descriptor is invalid
+   */
+  public static DynamicStruct allocate(StructDescriptor desc) {
+    return new DynamicStruct(desc, ByteBuffer.allocate(desc.getSize()));
+  }
+
+  /**
+   * Constructs a new dynamic struct object with internal storage. The descriptor must be valid. The
+   * internal storage is allocated using ByteBuffer.allocateDirect().
+   *
+   * @param desc struct descriptor
+   * @return dynamic struct object
+   * @throws IllegalStateException if struct descriptor is invalid
+   */
+  public static DynamicStruct allocateDirect(StructDescriptor desc) {
+    return new DynamicStruct(desc, ByteBuffer.allocateDirect(desc.getSize()));
+  }
+
+  /**
+   * Constructs a new dynamic struct object. Note: the passed data buffer is not copied.
+   * Modifications to the passed buffer will be reflected in the struct and vice-versa.
+   *
+   * @param desc struct descriptor
+   * @param data byte buffer containing serialized data starting at current position
+   * @return dynamic struct object
+   */
+  public static DynamicStruct wrap(StructDescriptor desc, ByteBuffer data) {
+    return new DynamicStruct(desc, data.slice());
+  }
+
+  /**
+   * Gets the struct descriptor.
+   *
+   * @return struct descriptor
+   */
+  public StructDescriptor getDescriptor() {
+    return m_desc;
+  }
+
+  /**
+   * Gets the serialized backing data buffer.
+   *
+   * @return data buffer
+   */
+  public ByteBuffer getBuffer() {
+    return m_data.duplicate().position(0);
+  }
+
+  /**
+   * Overwrites the entire serialized struct by copying data from a byte array.
+   *
+   * @param data replacement data for the struct
+   * @throws BufferUnderflowException if data is smaller than the struct size
+   * @throws ReadOnlyBufferException if the underlying buffer is read-only
+   * @throws IllegalStateException if struct descriptor is invalid
+   */
+  public void setData(byte[] data) {
+    if (data.length < m_desc.getSize()) {
+      throw new BufferUnderflowException();
+    }
+    m_data.position(0).put(data);
+  }
+
+  /**
+   * Overwrites the entire serialized struct by copying data from a byte buffer.
+   *
+   * @param data replacement data for the struct; copy starts from current position
+   * @throws BufferUnderflowException if remaining data is smaller than the struct size
+   * @throws ReadOnlyBufferException if the underlying buffer is read-only
+   * @throws IllegalStateException if struct descriptor is invalid
+   */
+  public void setData(ByteBuffer data) {
+    if (data.remaining() < m_desc.getSize()) {
+      throw new BufferUnderflowException();
+    }
+    int oldLimit = data.limit();
+    m_data.position(0).put(data.limit(m_desc.getSize()));
+    data.limit(oldLimit);
+  }
+
+  /**
+   * Gets a struct field descriptor by name.
+   *
+   * @param name field name
+   * @return field descriptor, or null if no field with that name exists
+   */
+  public StructFieldDescriptor findField(String name) {
+    return m_desc.findFieldByName(name);
+  }
+
+  /**
+   * Gets the value of a boolean field.
+   *
+   * @param field field descriptor
+   * @param arrIndex array index (must be less than field array size)
+   * @return boolean field value
+   * @throws UnsupportedOperationException if field is not bool type
+   * @throws IllegalArgumentException if field is not a member of this struct
+   * @throws IllegalStateException if struct descriptor is invalid
+   * @throws ArrayIndexOutOfBoundsException if array index is out of bounds
+   */
+  public boolean getBoolField(StructFieldDescriptor field, int arrIndex) {
+    if (field.getType() != StructFieldType.kBool) {
+      throw new UnsupportedOperationException("field is not bool type");
+    }
+    return getFieldImpl(field, arrIndex) != 0;
+  }
+
+  /**
+   * Gets the value of a boolean field.
+   *
+   * @param field field descriptor
+   * @return boolean field value
+   * @throws UnsupportedOperationException if field is not bool type
+   * @throws IllegalArgumentException if field is not a member of this struct
+   * @throws IllegalStateException if struct descriptor is invalid
+   */
+  public boolean getBoolField(StructFieldDescriptor field) {
+    return getBoolField(field, 0);
+  }
+
+  /**
+   * Sets the value of a boolean field.
+   *
+   * @param field field descriptor
+   * @param value boolean value
+   * @param arrIndex array index (must be less than field array size)
+   * @throws UnsupportedOperationException if field is not bool type
+   * @throws IllegalArgumentException if field is not a member of this struct
+   * @throws IllegalStateException if struct descriptor is invalid
+   * @throws ArrayIndexOutOfBoundsException if array index is out of bounds
+   * @throws ReadOnlyBufferException if the underlying buffer is read-only
+   */
+  public void setBoolField(StructFieldDescriptor field, boolean value, int arrIndex) {
+    if (field.getType() != StructFieldType.kBool) {
+      throw new UnsupportedOperationException("field is not bool type");
+    }
+    setFieldImpl(field, value ? 1 : 0, arrIndex);
+  }
+
+  /**
+   * Sets the value of a boolean field.
+   *
+   * @param field field descriptor
+   * @param value boolean value
+   * @throws UnsupportedOperationException if field is not bool type
+   * @throws IllegalArgumentException if field is not a member of this struct
+   * @throws IllegalStateException if struct descriptor is invalid
+   * @throws ReadOnlyBufferException if the underlying buffer is read-only
+   */
+  public void setBoolField(StructFieldDescriptor field, boolean value) {
+    setBoolField(field, value, 0);
+  }
+
+  /**
+   * Gets the value of an integer field.
+   *
+   * @param field field descriptor
+   * @param arrIndex array index (must be less than field array size)
+   * @return integer field value
+   * @throws UnsupportedOperationException if field is not integer type
+   * @throws IllegalArgumentException if field is not a member of this struct
+   * @throws IllegalStateException if struct descriptor is invalid
+   * @throws ArrayIndexOutOfBoundsException if array index is out of bounds
+   */
+  public long getIntField(StructFieldDescriptor field, int arrIndex) {
+    if (!field.isInt() && !field.isUint()) {
+      throw new UnsupportedOperationException("field is not integer type");
+    }
+    return getFieldImpl(field, arrIndex);
+  }
+
+  /**
+   * Gets the value of an integer field.
+   *
+   * @param field field descriptor
+   * @return integer field value
+   * @throws UnsupportedOperationException if field is not integer type
+   * @throws IllegalArgumentException if field is not a member of this struct
+   * @throws IllegalStateException if struct descriptor is invalid
+   */
+  public long getIntField(StructFieldDescriptor field) {
+    return getIntField(field, 0);
+  }
+
+  /**
+   * Sets the value of an integer field.
+   *
+   * @param field field descriptor
+   * @param value integer value
+   * @param arrIndex array index (must be less than field array size)
+   * @throws UnsupportedOperationException if field is not integer type
+   * @throws IllegalArgumentException if field is not a member of this struct
+   * @throws IllegalStateException if struct descriptor is invalid
+   * @throws ArrayIndexOutOfBoundsException if array index is out of bounds
+   * @throws ReadOnlyBufferException if the underlying buffer is read-only
+   */
+  public void setIntField(StructFieldDescriptor field, long value, int arrIndex) {
+    if (!field.isInt() && !field.isUint()) {
+      throw new UnsupportedOperationException("field is not integer type");
+    }
+    setFieldImpl(field, value, arrIndex);
+  }
+
+  /**
+   * Sets the value of an integer field.
+   *
+   * @param field field descriptor
+   * @param value integer value
+   * @throws UnsupportedOperationException if field is not integer type
+   * @throws IllegalArgumentException if field is not a member of this struct
+   * @throws IllegalStateException if struct descriptor is invalid
+   * @throws ReadOnlyBufferException if the underlying buffer is read-only
+   */
+  public void setIntField(StructFieldDescriptor field, long value) {
+    setIntField(field, value, 0);
+  }
+
+  /**
+   * Gets the value of a float field.
+   *
+   * @param field field descriptor
+   * @param arrIndex array index (must be less than field array size)
+   * @return float field value
+   * @throws UnsupportedOperationException if field is not float type
+   * @throws IllegalArgumentException if field is not a member of this struct
+   * @throws IllegalStateException if struct descriptor is invalid
+   * @throws ArrayIndexOutOfBoundsException if array index is out of bounds
+   */
+  public float getFloatField(StructFieldDescriptor field, int arrIndex) {
+    if (field.getType() != StructFieldType.kFloat) {
+      throw new UnsupportedOperationException("field is not float type");
+    }
+    return Float.intBitsToFloat((int) getFieldImpl(field, arrIndex));
+  }
+
+  /**
+   * Gets the value of a float field.
+   *
+   * @param field field descriptor
+   * @return float field value
+   * @throws UnsupportedOperationException if field is not float type
+   * @throws IllegalArgumentException if field is not a member of this struct
+   * @throws IllegalStateException if struct descriptor is invalid
+   */
+  public float getFloatField(StructFieldDescriptor field) {
+    return getFloatField(field, 0);
+  }
+
+  /**
+   * Sets the value of a float field.
+   *
+   * @param field field descriptor
+   * @param value float value
+   * @param arrIndex array index (must be less than field array size)
+   * @throws UnsupportedOperationException if field is not float type
+   * @throws IllegalArgumentException if field is not a member of this struct
+   * @throws IllegalStateException if struct descriptor is invalid
+   * @throws ArrayIndexOutOfBoundsException if array index is out of bounds
+   * @throws ReadOnlyBufferException if the underlying buffer is read-only
+   */
+  public void setFloatField(StructFieldDescriptor field, float value, int arrIndex) {
+    if (field.getType() != StructFieldType.kFloat) {
+      throw new UnsupportedOperationException("field is not float type");
+    }
+    setFieldImpl(field, Float.floatToIntBits(value), arrIndex);
+  }
+
+  /**
+   * Sets the value of a float field.
+   *
+   * @param field field descriptor
+   * @param value float value
+   * @throws UnsupportedOperationException if field is not float type
+   * @throws IllegalArgumentException if field is not a member of this struct
+   * @throws IllegalStateException if struct descriptor is invalid
+   * @throws ReadOnlyBufferException if the underlying buffer is read-only
+   */
+  public void setFloatField(StructFieldDescriptor field, float value) {
+    setFloatField(field, value, 0);
+  }
+
+  /**
+   * Gets the value of a double field.
+   *
+   * @param field field descriptor
+   * @param arrIndex array index (must be less than field array size)
+   * @return double field value
+   * @throws UnsupportedOperationException if field is not double type
+   * @throws IllegalArgumentException if field is not a member of this struct
+   * @throws IllegalStateException if struct descriptor is invalid
+   * @throws ArrayIndexOutOfBoundsException if array index is out of bounds
+   */
+  public double getDoubleField(StructFieldDescriptor field, int arrIndex) {
+    if (field.getType() != StructFieldType.kDouble) {
+      throw new UnsupportedOperationException("field is not double type");
+    }
+    return Double.longBitsToDouble(getFieldImpl(field, arrIndex));
+  }
+
+  /**
+   * Gets the value of a double field.
+   *
+   * @param field field descriptor
+   * @return double field value
+   * @throws UnsupportedOperationException if field is not double type
+   * @throws IllegalArgumentException if field is not a member of this struct
+   * @throws IllegalStateException if struct descriptor is invalid
+   */
+  public double getDoubleField(StructFieldDescriptor field) {
+    return getDoubleField(field, 0);
+  }
+
+  /**
+   * Sets the value of a double field.
+   *
+   * @param field field descriptor
+   * @param value double value
+   * @param arrIndex array index (must be less than field array size)
+   * @throws UnsupportedOperationException if field is not double type
+   * @throws IllegalArgumentException if field is not a member of this struct
+   * @throws IllegalStateException if struct descriptor is invalid
+   * @throws ArrayIndexOutOfBoundsException if array index is out of bounds
+   * @throws ReadOnlyBufferException if the underlying buffer is read-only
+   */
+  public void setDoubleField(StructFieldDescriptor field, double value, int arrIndex) {
+    if (field.getType() != StructFieldType.kDouble) {
+      throw new UnsupportedOperationException("field is not double type");
+    }
+    setFieldImpl(field, Double.doubleToLongBits(value), arrIndex);
+  }
+
+  /**
+   * Sets the value of a double field.
+   *
+   * @param field field descriptor
+   * @param value double value
+   * @throws UnsupportedOperationException if field is not double type
+   * @throws IllegalArgumentException if field is not a member of this struct
+   * @throws IllegalStateException if struct descriptor is invalid
+   * @throws ReadOnlyBufferException if the underlying buffer is read-only
+   */
+  public void setDoubleField(StructFieldDescriptor field, double value) {
+    setDoubleField(field, value, 0);
+  }
+
+  /**
+   * Gets the value of a character or character array field.
+   *
+   * @param field field descriptor
+   * @return field value
+   * @throws UnsupportedOperationException if field is not char type
+   * @throws IllegalArgumentException if field is not a member of this struct
+   * @throws IllegalStateException if struct descriptor is invalid
+   */
+  public String getStringField(StructFieldDescriptor field) {
+    if (field.getType() != StructFieldType.kChar) {
+      throw new UnsupportedOperationException("field is not char type");
+    }
+    if (!field.getParent().equals(m_desc)) {
+      throw new IllegalArgumentException("field is not part of this struct");
+    }
+    if (!m_desc.isValid()) {
+      throw new IllegalStateException("struct descriptor is not valid");
+    }
+    byte[] bytes = new byte[field.m_arraySize];
+    m_data.position(field.m_offset).get(bytes, 0, field.m_arraySize);
+    return new String(bytes, StandardCharsets.UTF_8);
+  }
+
+  /**
+   * Sets the value of a character or character array field.
+   *
+   * @param field field descriptor
+   * @param value field value
+   * @throws UnsupportedOperationException if field is not char type
+   * @throws IllegalArgumentException if field is not a member of this struct
+   * @throws IllegalStateException if struct descriptor is invalid
+   */
+  public void setStringField(StructFieldDescriptor field, String value) {
+    if (field.getType() != StructFieldType.kChar) {
+      throw new UnsupportedOperationException("field is not char type");
+    }
+    if (!field.getParent().equals(m_desc)) {
+      throw new IllegalArgumentException("field is not part of this struct");
+    }
+    if (!m_desc.isValid()) {
+      throw new IllegalStateException("struct descriptor is not valid");
+    }
+    ByteBuffer bb = StandardCharsets.UTF_8.encode(value);
+    int len = Math.min(bb.remaining(), field.m_arraySize);
+    m_data.position(field.m_offset).put(bb.limit(len));
+    for (int i = len; i < field.m_arraySize; i++) {
+      m_data.put((byte) 0);
+    }
+  }
+
+  /**
+   * Gets the value of a struct field.
+   *
+   * @param field field descriptor
+   * @param arrIndex array index (must be less than field array size)
+   * @return field value
+   * @throws UnsupportedOperationException if field is not of struct type
+   * @throws IllegalArgumentException if field is not a member of this struct
+   * @throws IllegalStateException if struct descriptor is invalid
+   * @throws ArrayIndexOutOfBoundsException if array index is out of bounds
+   */
+  public DynamicStruct getStructField(StructFieldDescriptor field, int arrIndex) {
+    if (field.getType() != StructFieldType.kStruct) {
+      throw new UnsupportedOperationException("field is not struct type");
+    }
+    if (!field.getParent().equals(m_desc)) {
+      throw new IllegalArgumentException("field is not part of this struct");
+    }
+    if (!m_desc.isValid()) {
+      throw new IllegalStateException("struct descriptor is not valid");
+    }
+    if (arrIndex < 0 || arrIndex >= field.m_arraySize) {
+      throw new ArrayIndexOutOfBoundsException(
+          "arrIndex (" + arrIndex + ") is larger than array size (" + field.m_arraySize + ")");
+    }
+    StructDescriptor struct = field.getStruct();
+    return wrap(struct, m_data.position(field.m_offset + arrIndex * struct.m_size));
+  }
+
+  /**
+   * Gets the value of a struct field.
+   *
+   * @param field field descriptor
+   * @return field value
+   * @throws UnsupportedOperationException if field is not of struct type
+   * @throws IllegalArgumentException if field is not a member of this struct
+   * @throws IllegalStateException if struct descriptor is invalid
+   */
+  public DynamicStruct getStructField(StructFieldDescriptor field) {
+    return getStructField(field, 0);
+  }
+
+  /**
+   * Sets the value of a struct field.
+   *
+   * @param field field descriptor
+   * @param value struct value
+   * @param arrIndex array index (must be less than field array size)
+   * @throws UnsupportedOperationException if field is not struct type
+   * @throws IllegalArgumentException if field is not a member of this struct
+   * @throws IllegalStateException if struct descriptor is invalid
+   * @throws ArrayIndexOutOfBoundsException if array index is out of bounds
+   * @throws ReadOnlyBufferException if the underlying buffer is read-only
+   */
+  public void setStructField(StructFieldDescriptor field, DynamicStruct value, int arrIndex) {
+    if (field.getType() != StructFieldType.kStruct) {
+      throw new UnsupportedOperationException("field is not struct type");
+    }
+    if (!field.getParent().equals(m_desc)) {
+      throw new IllegalArgumentException("field is not part of this struct");
+    }
+    if (!m_desc.isValid()) {
+      throw new IllegalStateException("struct descriptor is not valid");
+    }
+    StructDescriptor struct = field.getStruct();
+    if (!value.getDescriptor().equals(struct)) {
+      throw new IllegalArgumentException("value's struct type does not match field struct type");
+    }
+    if (!value.getDescriptor().isValid()) {
+      throw new IllegalStateException("value's struct descriptor is not valid");
+    }
+    if (arrIndex < 0 || arrIndex >= field.m_arraySize) {
+      throw new ArrayIndexOutOfBoundsException(
+          "arrIndex (" + arrIndex + ") is larger than array size (" + field.m_arraySize + ")");
+    }
+    m_data
+        .position(field.m_offset + arrIndex * struct.m_size)
+        .put(value.m_data.position(0).limit(value.getDescriptor().getSize()));
+  }
+
+  /**
+   * Sets the value of a struct field.
+   *
+   * @param field field descriptor
+   * @param value struct value
+   * @throws UnsupportedOperationException if field is not struct type
+   * @throws IllegalArgumentException if field is not a member of this struct
+   * @throws IllegalStateException if struct descriptor is invalid
+   * @throws ReadOnlyBufferException if the underlying buffer is read-only
+   */
+  public void setStructField(StructFieldDescriptor field, DynamicStruct value) {
+    setStructField(field, value, 0);
+  }
+
+  private long getFieldImpl(StructFieldDescriptor field, int arrIndex) {
+    if (!field.getParent().equals(m_desc)) {
+      throw new IllegalArgumentException("field is not part of this struct");
+    }
+    if (!m_desc.isValid()) {
+      throw new IllegalStateException("struct descriptor is not valid");
+    }
+    if (arrIndex < 0 || arrIndex >= field.m_arraySize) {
+      throw new ArrayIndexOutOfBoundsException(
+          "arrIndex (" + arrIndex + ") is larger than array size (" + field.m_arraySize + ")");
+    }
+
+    long val;
+    switch (field.m_size) {
+      case 1:
+        val = m_data.get(field.m_offset + arrIndex);
+        break;
+      case 2:
+        val = m_data.getShort(field.m_offset + arrIndex * 2);
+        break;
+      case 4:
+        val = m_data.getInt(field.m_offset + arrIndex * 4);
+        break;
+      case 8:
+        val = m_data.getLong(field.m_offset + arrIndex * 8);
+        break;
+      default:
+        throw new IllegalStateException("invalid field size");
+    }
+
+    if (field.isUint() || field.getType() == StructFieldType.kBool) {
+      // for unsigned fields, we can simply logical shift and mask
+      return (val >>> field.m_bitShift) & field.getBitMask();
+    } else {
+      // to get sign extension, shift so the sign bit within the bitfield goes to the long's sign
+      // bit (also clearing all higher bits), then shift back down (also clearing all lower bits);
+      // since upper and lower bits are cleared with the shifts, the bitmask is unnecessary
+      return (val << (64 - field.m_bitShift - field.getBitWidth())) >> (64 - field.getBitWidth());
+    }
+  }
+
+  private void setFieldImpl(StructFieldDescriptor field, long value, int arrIndex) {
+    if (!field.getParent().equals(m_desc)) {
+      throw new IllegalArgumentException("field is not part of this struct");
+    }
+    if (!m_desc.isValid()) {
+      throw new IllegalStateException("struct descriptor is not valid");
+    }
+    if (arrIndex < 0 || arrIndex >= field.m_arraySize) {
+      throw new ArrayIndexOutOfBoundsException(
+          "arrIndex (" + arrIndex + ") is larger than array size (" + field.m_arraySize + ")");
+    }
+
+    // common case is no bit shift and no masking
+    if (!field.isBitField()) {
+      switch (field.m_size) {
+        case 1:
+          m_data.put(field.m_offset + arrIndex, (byte) value);
+          break;
+        case 2:
+          m_data.putShort(field.m_offset + arrIndex * 2, (short) value);
+          break;
+        case 4:
+          m_data.putInt(field.m_offset + arrIndex * 4, (int) value);
+          break;
+        case 8:
+          m_data.putLong(field.m_offset + arrIndex * 8, value);
+          break;
+        default:
+          throw new IllegalStateException("invalid field size");
+      }
+      return;
+    }
+
+    // handle bit shifting and masking into current value
+    switch (field.m_size) {
+      case 1:
+        {
+          byte val = m_data.get(field.m_offset + arrIndex);
+          val &= ~(field.getBitMask() << field.m_bitShift);
+          val |= (value & field.getBitMask()) << field.m_bitShift;
+          m_data.put(field.m_offset + arrIndex, val);
+          break;
+        }
+      case 2:
+        {
+          short val = m_data.getShort(field.m_offset + arrIndex * 2);
+          val &= ~(field.getBitMask() << field.m_bitShift);
+          val |= (value & field.getBitMask()) << field.m_bitShift;
+          m_data.putShort(field.m_offset + arrIndex * 2, val);
+          break;
+        }
+      case 4:
+        {
+          int val = m_data.getInt(field.m_offset + arrIndex * 4);
+          val &= ~(field.getBitMask() << field.m_bitShift);
+          val |= (value & field.getBitMask()) << field.m_bitShift;
+          m_data.putInt(field.m_offset + arrIndex * 4, val);
+          break;
+        }
+      case 8:
+        {
+          long val = m_data.getLong(field.m_offset + arrIndex * 8);
+          val &= ~(field.getBitMask() << field.m_bitShift);
+          val |= (value & field.getBitMask()) << field.m_bitShift;
+          m_data.putLong(field.m_offset + arrIndex * 8, val);
+          break;
+        }
+      default:
+        throw new IllegalStateException("invalid field size");
+    }
+  }
+
+  private final StructDescriptor m_desc;
+  private final ByteBuffer m_data;
+}

wpiutil/src/main/java/edu/wpi/first/util/struct/Struct.java

@@ -0,0 +1,116 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+package edu.wpi.first.util.struct;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Interface for raw struct serialization.
+ *
+ * <p>This is designed for serializing small fixed-size data structures in the fastest and most
+ * compact means possible. Serialization consists of making relative put() calls to a ByteBuffer and
+ * deserialization consists of making relative get() calls from a ByteBuffer.
+ *
+ * <p>Idiomatically, classes that support raw struct serialization should provide a static final
+ * member named "struct" that provides an instance of an implementation of this interface.
+ *
+ * @param <T> object type
+ */
+public interface Struct<T> {
+  /** Serialized size of a "bool" value. */
+  int kSizeBool = 1;
+
+  /** Serialized size of an "int8" or "uint8" value. */
+  int kSizeInt8 = 1;
+
+  /** Serialized size of an "int16" or "uint16" value. */
+  int kSizeInt16 = 2;
+
+  /** Serialized size of an "int32" or "uint32" value. */
+  int kSizeInt32 = 4;
+
+  /** Serialized size of an "int64" or "uint64" value. */
+  int kSizeInt64 = 8;
+
+  /** Serialized size of an "float" or "float32" value. */
+  int kSizeFloat = 4;
+
+  /** Serialized size of an "double" or "float64" value. */
+  int kSizeDouble = 8;
+
+  /**
+   * Gets the Class object for the stored value.
+   *
+   * @return Class
+   */
+  Class<T> getTypeClass();
+
+  /**
+   * Gets the type string (e.g. for NetworkTables). This should be globally unique and start with
+   * "struct:".
+   *
+   * @return type string
+   */
+  String getTypeString();
+
+  /**
+   * Gets the serialized size (in bytes). This should always be a constant.
+   *
+   * @return serialized size
+   */
+  int getSize();
+
+  /**
+   * Gets the schema.
+   *
+   * @return schema
+   */
+  String getSchema();
+
+  /**
+   * Gets the list of struct types referenced by this struct.
+   *
+   * @return list of struct types
+   */
+  default Struct<?>[] getNested() {
+    return new Struct<?>[] {};
+  }
+
+  /**
+   * Deserializes an object from a raw struct serialized ByteBuffer starting at the current
+   * position. Will increment the ByteBuffer position by getStructSize() bytes. Will not otherwise
+   * modify the ByteBuffer (e.g. byte order will not be changed).
+   *
+   * @param bb ByteBuffer
+   * @return New object
+   */
+  T unpack(ByteBuffer bb);
+
+  /**
+   * Puts object contents to a ByteBuffer starting at the current position. Will increment the
+   * ByteBuffer position by getStructSize() bytes. Will not otherwise modify the ByteBuffer (e.g.
+   * byte order will not be changed).
+   *
+   * @param bb ByteBuffer
+   * @param value object to serialize
+   */
+  void pack(ByteBuffer bb, T value);
+
+  /**
+   * Updates object contents from a raw struct serialized ByteBuffer starting at the current
+   * position. Will increment the ByteBuffer position by getStructSize() bytes. Will not otherwise
+   * modify the ByteBuffer (e.g. byte order will not be changed).
+   *
+   * <p>Immutable classes cannot and should not implement this function. The default implementation
+   * throws UnsupportedOperationException.
+   *
+   * @param out object to update
+   * @param bb ByteBuffer
+   * @throws UnsupportedOperationException if the object is immutable
+   */
+  default void unpackInto(T out, ByteBuffer bb) {
+    throw new UnsupportedOperationException("object does not support unpackInto");
+  }
+}

wpiutil/src/main/java/edu/wpi/first/util/struct/StructBuffer.java

@@ -0,0 +1,224 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+package edu.wpi.first.util.struct;
+
+import java.lang.reflect.Array;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.Collection;
+
+/**
+ * Reusable buffer for serialization/deserialization to/from a raw struct.
+ *
+ * @param <T> object type
+ */
+public final class StructBuffer<T> {
+  private StructBuffer(Struct<T> struct) {
+    m_structSize = struct.getSize();
+    m_buf = ByteBuffer.allocateDirect(m_structSize).order(ByteOrder.LITTLE_ENDIAN);
+    m_struct = struct;
+  }
+
+  public static <T> StructBuffer<T> create(Struct<T> struct) {
+    return new StructBuffer<T>(struct);
+  }
+
+  /**
+   * Gets the struct object of the stored type.
+   *
+   * @return struct object
+   */
+  public Struct<T> getStruct() {
+    return m_struct;
+  }
+
+  /**
+   * Gets the type string.
+   *
+   * @return type string
+   */
+  public String getTypeString() {
+    return m_struct.getTypeString();
+  }
+
+  /**
+   * Ensures sufficient buffer space is available for the given number of elements.
+   *
+   * @param nelem number of elements
+   */
+  public void reserve(int nelem) {
+    if ((nelem * m_structSize) > m_buf.capacity()) {
+      m_buf = ByteBuffer.allocateDirect(nelem * m_structSize).order(ByteOrder.LITTLE_ENDIAN);
+    }
+  }
+
+  /**
+   * Serializes a value to a ByteBuffer. The returned ByteBuffer is a direct byte buffer with the
+   * position set to the end of the serialized data.
+   *
+   * @param value value
+   * @return byte buffer
+   */
+  public ByteBuffer write(T value) {
+    m_buf.position(0);
+    m_struct.pack(m_buf, value);
+    return m_buf;
+  }
+
+  /**
+   * Deserializes a value from a byte array, creating a new object.
+   *
+   * @param buf byte array
+   * @param start starting location within byte array
+   * @param len length of serialized data
+   * @return new object
+   */
+  public T read(byte[] buf, int start, int len) {
+    return read(ByteBuffer.wrap(buf, start, len));
+  }
+
+  /**
+   * Deserializes a value from a byte array, creating a new object.
+   *
+   * @param buf byte array
+   * @return new object
+   */
+  public T read(byte[] buf) {
+    return read(buf, 0, buf.length);
+  }
+
+  /**
+   * Deserializes a value from a ByteBuffer, creating a new object.
+   *
+   * @param buf byte buffer
+   * @return new object
+   */
+  public T read(ByteBuffer buf) {
+    buf.order(ByteOrder.LITTLE_ENDIAN);
+    return m_struct.unpack(buf);
+  }
+
+  /**
+   * Deserializes a value from a byte array into a mutable object.
+   *
+   * @param out object (will be updated with deserialized contents)
+   * @param buf byte array
+   * @param start starting location within byte array
+   * @param len length of serialized data
+   * @throws UnsupportedOperationException if T is immutable
+   */
+  public void readInto(T out, byte[] buf, int start, int len) {
+    readInto(out, ByteBuffer.wrap(buf, start, len));
+  }
+
+  /**
+   * Deserializes a value from a byte array into a mutable object.
+   *
+   * @param out object (will be updated with deserialized contents)
+   * @param buf byte array
+   * @throws UnsupportedOperationException if T is immutable
+   */
+  public void readInto(T out, byte[] buf) {
+    readInto(out, buf, 0, buf.length);
+  }
+
+  /**
+   * Deserializes a value from a ByteBuffer into a mutable object.
+   *
+   * @param out object (will be updated with deserialized contents)
+   * @param buf byte buffer
+   * @throws UnsupportedOperationException if T is immutable
+   */
+  public void readInto(T out, ByteBuffer buf) {
+    m_struct.unpackInto(out, buf);
+  }
+
+  /**
+   * Serializes a collection of values to a ByteBuffer. The returned ByteBuffer is a direct byte
+   * buffer with the position set to the end of the serialized data.
+   *
+   * @param values values
+   * @return byte buffer
+   */
+  public ByteBuffer writeArray(Collection<T> values) {
+    m_buf.position(0);
+    if ((values.size() * m_structSize) > m_buf.capacity()) {
+      m_buf =
+          ByteBuffer.allocateDirect(values.size() * m_structSize * 2)
+              .order(ByteOrder.LITTLE_ENDIAN);
+    }
+    for (T v : values) {
+      m_struct.pack(m_buf, v);
+    }
+    return m_buf;
+  }
+
+  /**
+   * Serializes an array of values to a ByteBuffer. The returned ByteBuffer is a direct byte buffer
+   * with the position set to the end of the serialized data.
+   *
+   * @param values values
+   * @return byte buffer
+   */
+  public ByteBuffer writeArray(T[] values) {
+    m_buf.position(0);
+    if ((values.length * m_structSize) > m_buf.capacity()) {
+      m_buf =
+          ByteBuffer.allocateDirect(values.length * m_structSize * 2)
+              .order(ByteOrder.LITTLE_ENDIAN);
+    }
+    for (T v : values) {
+      m_struct.pack(m_buf, v);
+    }
+    return m_buf;
+  }
+
+  /**
+   * Deserializes an array of values from a byte array, creating an array of new objects.
+   *
+   * @param buf byte array
+   * @param start starting location within byte array
+   * @param len length of serialized data
+   * @return new object array
+   */
+  public T[] readArray(byte[] buf, int start, int len) {
+    return readArray(ByteBuffer.wrap(buf, start, len));
+  }
+
+  /**
+   * Deserializes an array of values from a byte array, creating an array of new objects.
+   *
+   * @param buf byte array
+   * @return new object array
+   */
+  public T[] readArray(byte[] buf) {
+    return readArray(buf, 0, buf.length);
+  }
+
+  /**
+   * Deserializes an array of values from a ByteBuffer, creating an array of new objects.
+   *
+   * @param buf byte buffer
+   * @return new object array
+   */
+  public T[] readArray(ByteBuffer buf) {
+    buf.order(ByteOrder.LITTLE_ENDIAN);
+    int len = buf.limit() - buf.position();
+    if ((len % m_structSize) != 0) {
+      throw new RuntimeException("buffer size not a multiple of struct size");
+    }
+    int nelem = len / m_structSize;
+    @SuppressWarnings("unchecked")
+    T[] arr = (T[]) Array.newInstance(m_struct.getClass(), nelem);
+    for (int i = 0; i < nelem; i++) {
+      arr[i] = m_struct.unpack(buf);
+    }
+    return arr;
+  }
+
+  private ByteBuffer m_buf;
+  private final Struct<T> m_struct;
+  private final int m_structSize;
+}

wpiutil/src/main/java/edu/wpi/first/util/struct/StructDescriptor.java

@@ -0,0 +1,156 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+package edu.wpi.first.util.struct;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Stack;
+
+/** Raw struct dynamic struct descriptor. */
+public class StructDescriptor {
+  StructDescriptor(String name) {
+    m_name = name;
+  }
+
+  /**
+   * Gets the struct name.
+   *
+   * @return name
+   */
+  public String getName() {
+    return m_name;
+  }
+
+  /**
+   * Gets the struct schema.
+   *
+   * @return schema
+   */
+  public String getSchema() {
+    return m_schema;
+  }
+
+  /**
+   * Returns whether the struct is valid (e.g. the struct is fully defined and field offsets
+   * computed).
+   *
+   * @return true if valid
+   */
+  public boolean isValid() {
+    return m_valid;
+  }
+
+  /**
+   * Returns the struct size, in bytes. Not valid unless IsValid() is true.
+   *
+   * @return size in bytes
+   * @throws IllegalStateException if descriptor is invalid
+   */
+  public int getSize() {
+    if (!m_valid) {
+      throw new IllegalStateException("descriptor is invalid");
+    }
+    return m_size;
+  }
+
+  /**
+   * Gets a field descriptor by name. Note the field cannot be accessed until the struct is valid.
+   *
+   * @param name field name
+   * @return field descriptor, or nullptr if not found
+   */
+  public StructFieldDescriptor findFieldByName(String name) {
+    return m_fieldsByName.get(name);
+  }
+
+  /**
+   * Gets all field descriptors. Note fields cannot be accessed until the struct is valid.
+   *
+   * @return field descriptors
+   */
+  public List<StructFieldDescriptor> getFields() {
+    return m_fields;
+  }
+
+  boolean checkCircular(Stack<StructDescriptor> stack) {
+    stack.push(this);
+    for (StructDescriptor ref : m_references) {
+      if (stack.contains(ref)) {
+        return false;
+      }
+      if (!ref.checkCircular(stack)) {
+        return false;
+      }
+    }
+    stack.pop();
+    return true;
+  }
+
+  void calculateOffsets(Stack<StructDescriptor> stack) {
+    int offset = 0;
+    int shift = 0;
+    int prevBitfieldSize = 0;
+    for (StructFieldDescriptor field : m_fields) {
+      if (!field.isBitField()) {
+        shift = 0; // reset shift on non-bitfield element
+        offset += prevBitfieldSize; // finish bitfield if active
+        prevBitfieldSize = 0; // previous is now not bitfield
+        field.m_offset = offset;
+        StructDescriptor struct = field.getStruct();
+        if (struct != null) {
+          if (!struct.isValid()) {
+            m_valid = false;
+            return;
+          }
+          field.m_size = struct.m_size;
+        }
+        offset += field.m_size * field.m_arraySize;
+      } else {
+        int bitWidth = field.getBitWidth();
+        if (field.getType() == StructFieldType.kBool
+            && prevBitfieldSize != 0
+            && (shift + 1) <= (prevBitfieldSize * 8)) {
+          // bool takes on size of preceding bitfield type (if it fits)
+          field.m_size = prevBitfieldSize;
+        } else if (field.m_size != prevBitfieldSize || (shift + bitWidth) > (field.m_size * 8)) {
+          shift = 0;
+          offset += prevBitfieldSize;
+        }
+        prevBitfieldSize = field.m_size;
+        field.m_offset = offset;
+        field.m_bitShift = shift;
+        shift += bitWidth;
+      }
+    }
+
+    // update struct size
+    m_size = offset + prevBitfieldSize;
+    m_valid = true;
+
+    // now that we're valid, referring types may be too
+    stack.push(this);
+    for (StructDescriptor ref : m_references) {
+      if (stack.contains(ref)) {
+        throw new IllegalStateException(
+            "internal error (inconsistent data): circular struct reference between "
+                + m_name
+                + " and "
+                + ref.m_name);
+      }
+      ref.calculateOffsets(stack);
+    }
+    stack.pop();
+  }
+
+  private final String m_name;
+  String m_schema;
+  final List<StructDescriptor> m_references = new ArrayList<>();
+  final List<StructFieldDescriptor> m_fields = new ArrayList<>();
+  final Map<String, StructFieldDescriptor> m_fieldsByName = new HashMap<>();
+  int m_size;
+  boolean m_valid;
+}

wpiutil/src/main/java/edu/wpi/first/util/struct/StructDescriptorDatabase.java

@@ -0,0 +1,148 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+package edu.wpi.first.util.struct;
+
+import edu.wpi.first.util.struct.parser.ParseException;
+import edu.wpi.first.util.struct.parser.ParsedDeclaration;
+import edu.wpi.first.util.struct.parser.ParsedSchema;
+import edu.wpi.first.util.struct.parser.Parser;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Stack;
+
+/** Database of raw struct dynamic descriptors. */
+public class StructDescriptorDatabase {
+  /**
+   * Adds a structure schema to the database. If the struct references other structs that have not
+   * yet been added, it will not be valid until those structs are also added.
+   *
+   * @param name structure name
+   * @param schema structure schema
+   * @return Added struct dynamic descriptor
+   * @throws BadSchemaException if schema invalid
+   */
+  public StructDescriptor add(String name, String schema) throws BadSchemaException {
+    Parser parser = new Parser(schema);
+    ParsedSchema parsed;
+    try {
+      parsed = parser.parse();
+    } catch (ParseException e) {
+      throw new BadSchemaException("parse error", e);
+    }
+
+    // turn parsed schema into descriptors
+    StructDescriptor theStruct = m_structs.computeIfAbsent(name, k -> new StructDescriptor(k));
+    theStruct.m_schema = schema;
+    theStruct.m_fields.clear();
+    boolean isValid = true;
+    for (ParsedDeclaration decl : parsed.declarations) {
+      StructFieldType type = StructFieldType.fromString(decl.typeString);
+      int size = type.size;
+
+      // bitfield checks
+      if (decl.bitWidth != 0) {
+        // only integer or boolean types are allowed
+        if (!type.isInt && !type.isUint && type != StructFieldType.kBool) {
+          throw new BadSchemaException(
+              decl.name, "type " + decl.typeString + " cannot be bitfield");
+        }
+
+        // bit width cannot be larger than field size
+        if (decl.bitWidth > (size * 8)) {
+          throw new BadSchemaException(
+              decl.name, "bit width " + decl.bitWidth + " exceeds type size");
+        }
+
+        // bit width must be 1 for booleans
+        if (type == StructFieldType.kBool && decl.bitWidth != 1) {
+          throw new BadSchemaException(decl.name, "bit width must be 1 for bool type");
+        }
+
+        // cannot combine array and bitfield (shouldn't parse, but double-check)
+        if (decl.arraySize > 1) {
+          throw new BadSchemaException(decl.name, "cannot combine array and bitfield");
+        }
+      }
+
+      // struct handling
+      StructDescriptor structDesc = null;
+      if (type == StructFieldType.kStruct) {
+        // recursive definitions are not allowed
+        if (decl.typeString.equals(name)) {
+          throw new BadSchemaException(decl.name, "recursive struct reference");
+        }
+
+        // cross-reference struct, creating a placeholder if necessary
+        StructDescriptor aStruct =
+            m_structs.computeIfAbsent(decl.typeString, k -> new StructDescriptor(k));
+
+        // if the struct isn't valid, we can't be valid either
+        if (aStruct.isValid()) {
+          size = aStruct.getSize();
+        } else {
+          isValid = false;
+        }
+
+        // add to cross-references for when the struct does become valid
+        aStruct.m_references.add(theStruct);
+        structDesc = aStruct;
+      }
+
+      // create field
+      StructFieldDescriptor fieldDesc =
+          new StructFieldDescriptor(
+              theStruct,
+              decl.name,
+              type,
+              size,
+              decl.arraySize,
+              decl.bitWidth,
+              decl.enumValues,
+              structDesc);
+      if (theStruct.m_fieldsByName.put(decl.name, fieldDesc) != null) {
+        throw new BadSchemaException(decl.name, "duplicate field name");
+      }
+
+      theStruct.m_fields.add(fieldDesc);
+    }
+
+    theStruct.m_valid = isValid;
+    Stack<StructDescriptor> stack = new Stack<>();
+    if (isValid) {
+      // we have all the info needed, so calculate field offset & shift
+      theStruct.calculateOffsets(stack);
+    } else {
+      // check for circular reference
+      if (!theStruct.checkCircular(stack)) {
+        StringBuilder builder = new StringBuilder();
+        builder.append("circular struct reference: ");
+        boolean first = true;
+        for (StructDescriptor elem : stack) {
+          if (!first) {
+            builder.append(" <- ");
+          } else {
+            first = false;
+          }
+          builder.append(elem.getName());
+        }
+        throw new BadSchemaException(builder.toString());
+      }
+    }
+
+    return theStruct;
+  }
+
+  /**
+   * Finds a structure in the database by name.
+   *
+   * @param name structure name
+   * @return struct descriptor, or null if not found
+   */
+  public StructDescriptor find(String name) {
+    return m_structs.get(name);
+  }
+
+  private final Map<String, StructDescriptor> m_structs = new HashMap<>();
+}

wpiutil/src/main/java/edu/wpi/first/util/struct/StructFieldDescriptor.java

@@ -0,0 +1,241 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+package edu.wpi.first.util.struct;
+
+import java.util.Map;
+
+/** Raw struct dynamic field descriptor. */
+public class StructFieldDescriptor {
+  private static int toBitWidth(int size, int bitWidth) {
+    if (bitWidth == 0) {
+      return size * 8;
+    } else {
+      return bitWidth;
+    }
+  }
+
+  private static long toBitMask(int size, int bitWidth) {
+    if (size == 0) {
+      return 0;
+    } else {
+      return -1L >>> (64 - toBitWidth(size, bitWidth));
+    }
+  }
+
+  // does not fill in offset, shift
+  StructFieldDescriptor(
+      StructDescriptor parent,
+      String name,
+      StructFieldType type,
+      int size,
+      int arraySize,
+      int bitWidth,
+      Map<String, Long> enumValues,
+      StructDescriptor structDesc) {
+    m_parent = parent;
+    m_name = name;
+    m_size = size;
+    m_arraySize = arraySize;
+    m_enum = enumValues;
+    m_struct = structDesc;
+    m_bitMask = toBitMask(size, bitWidth);
+    m_type = type;
+    m_bitWidth = toBitWidth(size, bitWidth);
+  }
+
+  /**
+   * Gets the dynamic struct this field is contained in.
+   *
+   * @return struct descriptor
+   */
+  public StructDescriptor getParent() {
+    return m_parent;
+  }
+
+  /**
+   * Gets the field name.
+   *
+   * @return field name
+   */
+  public String getName() {
+    return m_name;
+  }
+
+  /**
+   * Gets the field type.
+   *
+   * @return field type
+   */
+  public StructFieldType getType() {
+    return m_type;
+  }
+
+  /**
+   * Returns whether the field type is a signed integer.
+   *
+   * @return true if signed integer, false otherwise
+   */
+  public boolean isInt() {
+    return m_type.isInt;
+  }
+
+  /**
+   * Returns whether the field type is an unsigned integer.
+   *
+   * @return true if unsigned integer, false otherwise
+   */
+  public boolean isUint() {
+    return m_type.isUint;
+  }
+
+  /**
+   * Gets the underlying storage size of the field, in bytes.
+   *
+   * @return number of bytes
+   */
+  public int getSize() {
+    return m_size;
+  }
+
+  /**
+   * Gets the storage offset of the field, in bytes.
+   *
+   * @return number of bytes from the start of the struct
+   */
+  public int getOffset() {
+    return m_offset;
+  }
+
+  /**
+   * Gets the bit width of the field, in bits.
+   *
+   * @return number of bits
+   */
+  public int getBitWidth() {
+    return m_bitWidth == 0 ? m_size * 8 : m_bitWidth;
+  }
+
+  /**
+   * Gets the bit mask for the field. The mask is always the least significant bits (it is not
+   * shifted).
+   *
+   * @return bit mask
+   */
+  public long getBitMask() {
+    return m_bitMask;
+  }
+
+  /**
+   * Gets the bit shift for the field (LSB=0).
+   *
+   * @return number of bits
+   */
+  public int getBitShift() {
+    return m_bitShift;
+  }
+
+  /**
+   * Returns whether the field is an array.
+   *
+   * @return true if array
+   */
+  public boolean isArray() {
+    return m_arraySize > 1;
+  }
+
+  /**
+   * Gets the array size. Returns 1 if non-array.
+   *
+   * @return number of elements
+   */
+  public int getArraySize() {
+    return m_arraySize;
+  }
+
+  /**
+   * Returns whether the field has enumerated values.
+   *
+   * @return true if there are enumerated values
+   */
+  public boolean hasEnum() {
+    return m_enum != null;
+  }
+
+  /**
+   * Gets the enumerated values.
+   *
+   * @return set of enumerated values
+   */
+  public Map<String, Long> getEnumValues() {
+    return m_enum;
+  }
+
+  /**
+   * Gets the struct descriptor for a struct data type.
+   *
+   * @return struct descriptor; returns null for non-struct
+   */
+  public StructDescriptor getStruct() {
+    return m_struct;
+  }
+
+  /**
+   * Gets the minimum unsigned integer value that can be stored in this field.
+   *
+   * @return minimum value
+   */
+  public long getUintMin() {
+    return 0;
+  }
+
+  /**
+   * Gets the maximum unsigned integer value that can be stored in this field. Note this is not the
+   * actual maximum for uint64 (due to Java lacking support for 64-bit unsigned integers).
+   *
+   * @return maximum value
+   */
+  public long getUintMax() {
+    return m_bitMask;
+  }
+
+  /**
+   * Gets the minimum signed integer value that can be stored in this field.
+   *
+   * @return minimum value
+   */
+  public long getIntMin() {
+    return (-(m_bitMask >> 1)) - 1;
+  }
+
+  /**
+   * Gets the maximum signed integer value that can be stored in this field.
+   *
+   * @return maximum value
+   */
+  public long getIntMax() {
+    return m_bitMask >> 1;
+  }
+
+  /**
+   * Returns whether the field is a bitfield.
+   *
+   * @return true if bitfield
+   */
+  public boolean isBitField() {
+    return m_bitShift != 0 || m_bitWidth != (m_size * 8);
+  }
+
+  private final StructDescriptor m_parent;
+  private final String m_name;
+  int m_size;
+  int m_offset;
+  final int m_arraySize; // 1 for non-arrays
+  private final Map<String, Long> m_enum;
+  private final StructDescriptor m_struct; // null for non-structs
+  private final long m_bitMask;
+  private final StructFieldType m_type;
+  private final int m_bitWidth;
+  int m_bitShift;
+}

wpiutil/src/main/java/edu/wpi/first/util/struct/StructFieldType.java

@@ -0,0 +1,67 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+package edu.wpi.first.util.struct;
+
+/** Known data types for raw struct dynamic fields (see StructFieldDescriptor). */
+public enum StructFieldType {
+  kBool("bool", false, false, 1),
+  kChar("char", false, false, 1),
+  kInt8("int8", true, false, 1),
+  kInt16("int16", true, false, 2),
+  kInt32("int32", true, false, 4),
+  kInt64("int64", true, false, 8),
+  kUint8("uint8", false, true, 1),
+  kUint16("uint16", false, true, 2),
+  kUint32("uint32", false, true, 4),
+  kUint64("uint64", false, true, 8),
+  kFloat("float", false, false, 4),
+  kDouble("double", false, false, 8),
+  kStruct("struct", false, false, 0);
+
+  @SuppressWarnings("MemberName")
+  public final String name;
+
+  @SuppressWarnings("MemberName")
+  public final boolean isInt;
+
+  @SuppressWarnings("MemberName")
+  public final boolean isUint;
+
+  @SuppressWarnings("MemberName")
+  public final int size;
+
+  StructFieldType(String name, boolean isInt, boolean isUint, int size) {
+    this.name = name;
+    this.isInt = isInt;
+    this.isUint = isUint;
+    this.size = size;
+  }
+
+  @Override
+  public String toString() {
+    return name;
+  }
+
+  /**
+   * Get field type from string.
+   *
+   * @param str string
+   * @return field type
+   */
+  public static StructFieldType fromString(String str) {
+    for (StructFieldType type : StructFieldType.values()) {
+      if (type.name.equals(str)) {
+        return type;
+      }
+    }
+    if ("float32".equals(str)) {
+      return kFloat;
+    } else if ("float64".equals(str)) {
+      return kDouble;
+    } else {
+      return kStruct;
+    }
+  }
+}

wpiutil/src/main/java/edu/wpi/first/util/struct/parser/Lexer.java

@@ -0,0 +1,132 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+package edu.wpi.first.util.struct.parser;
+
+/** Raw struct schema lexer. */
+public class Lexer {
+  /**
+   * Construct a raw struct schema lexer.
+   *
+   * @param in schema
+   */
+  public Lexer(String in) {
+    m_in = in;
+  }
+
+  /**
+   * Gets the next token.
+   *
+   * @return Token kind; the token text can be retrieved using getTokenText()
+   */
+  public TokenKind scan() {
+    // skip whitespace
+    do {
+      get();
+    } while (m_current == ' ' || m_current == '\t' || m_current == '\n' || m_current == '\r');
+    m_tokenStart = m_pos - 1;
+
+    switch (m_current) {
+      case '[':
+        return TokenKind.kLeftBracket;
+      case ']':
+        return TokenKind.kRightBracket;
+      case '{':
+        return TokenKind.kLeftBrace;
+      case '}':
+        return TokenKind.kRightBrace;
+      case ':':
+        return TokenKind.kColon;
+      case ';':
+        return TokenKind.kSemicolon;
+      case ',':
+        return TokenKind.kComma;
+      case '=':
+        return TokenKind.kEquals;
+      case '-':
+      case '0':
+      case '1':
+      case '2':
+      case '3':
+      case '4':
+      case '5':
+      case '6':
+      case '7':
+      case '8':
+      case '9':
+        return scanInteger();
+      case '\0':
+        return TokenKind.kEndOfInput;
+      default:
+        if (Character.isLetter(m_current) || m_current == '_') {
+          return scanIdentifier();
+        }
+        return TokenKind.kUnknown;
+    }
+  }
+
+  /**
+   * Gets the text of the last lexed token.
+   *
+   * @return token text
+   */
+  public String getTokenText() {
+    if (m_tokenStart >= m_in.length()) {
+      return "";
+    }
+    return m_in.substring(m_tokenStart, m_pos);
+  }
+
+  /**
+   * Gets the starting position of the last lexed token.
+   *
+   * @return position (0 = first character)
+   */
+  public int getPosition() {
+    return m_tokenStart;
+  }
+
+  private TokenKind scanInteger() {
+    do {
+      get();
+    } while (Character.isDigit(m_current));
+    unget();
+    return TokenKind.kInteger;
+  }
+
+  private TokenKind scanIdentifier() {
+    do {
+      get();
+    } while (Character.isLetterOrDigit(m_current) || m_current == '_');
+    unget();
+    return TokenKind.kIdentifier;
+  }
+
+  private void get() {
+    if (m_pos < m_in.length()) {
+      m_current = m_in.charAt(m_pos);
+    } else {
+      m_current = '\0';
+    }
+    ++m_pos;
+  }
+
+  private void unget() {
+    if (m_pos > 0) {
+      m_pos--;
+      if (m_pos < m_in.length()) {
+        m_current = m_in.charAt(m_pos);
+      } else {
+        m_current = '\0';
+      }
+    } else {
+      m_current = '\0';
+    }
+  }
+
+  final String m_in;
+  char m_current;
+  int m_tokenStart;
+  int m_pos;
+}

wpiutil/src/main/java/edu/wpi/first/util/struct/parser/ParseException.java

@@ -0,0 +1,33 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+package edu.wpi.first.util.struct.parser;
+
+public class ParseException extends Exception {
+  private final int m_pos;
+
+  public ParseException(int pos, String s) {
+    super(s);
+    m_pos = pos;
+  }
+
+  public ParseException(int pos, String message, Throwable cause) {
+    super(message, cause);
+    m_pos = pos;
+  }
+
+  public ParseException(int pos, Throwable cause) {
+    super(cause);
+    m_pos = pos;
+  }
+
+  public int getPosition() {
+    return m_pos;
+  }
+
+  @Override
+  public String toString() {
+    return m_pos + ": " + getMessage();
+  }
+}

wpiutil/src/main/java/edu/wpi/first/util/struct/parser/ParsedDeclaration.java

@@ -0,0 +1,25 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+package edu.wpi.first.util.struct.parser;
+
+import java.util.Map;
+
+/** Raw struct schema declaration. */
+public class ParsedDeclaration {
+  @SuppressWarnings("MemberName")
+  public String typeString;
+
+  @SuppressWarnings("MemberName")
+  public String name;
+
+  @SuppressWarnings("MemberName")
+  public Map<String, Long> enumValues;
+
+  @SuppressWarnings("MemberName")
+  public int arraySize = 1;
+
+  @SuppressWarnings("MemberName")
+  public int bitWidth;
+}

wpiutil/src/main/java/edu/wpi/first/util/struct/parser/ParsedSchema.java

@@ -0,0 +1,14 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+package edu.wpi.first.util.struct.parser;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/** Raw struct schema. */
+public class ParsedSchema {
+  @SuppressWarnings("MemberName")
+  public List<ParsedDeclaration> declarations = new ArrayList<>();
+}

wpiutil/src/main/java/edu/wpi/first/util/struct/parser/Parser.java

@@ -0,0 +1,157 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+package edu.wpi.first.util.struct.parser;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/** Raw struct schema parser. */
+public class Parser {
+  /**
+   * Construct a raw struct schema parser.
+   *
+   * @param in schema
+   */
+  public Parser(String in) {
+    m_lexer = new Lexer(in);
+  }
+
+  /**
+   * Parses the schema.
+   *
+   * @return parsed schema object
+   * @throws ParseException on parse error
+   */
+  public ParsedSchema parse() throws ParseException {
+    ParsedSchema schema = new ParsedSchema();
+    do {
+      getNextToken();
+      if (m_token == TokenKind.kSemicolon) {
+        continue;
+      }
+      if (m_token == TokenKind.kEndOfInput) {
+        break;
+      }
+      schema.declarations.add(parseDeclaration());
+    } while (m_token != TokenKind.kEndOfInput);
+    return schema;
+  }
+
+  private ParsedDeclaration parseDeclaration() throws ParseException {
+    ParsedDeclaration decl = new ParsedDeclaration();
+
+    // optional enum specification
+    if (m_token == TokenKind.kIdentifier && "enum".equals(m_lexer.getTokenText())) {
+      getNextToken();
+      expect(TokenKind.kLeftBrace);
+      decl.enumValues = parseEnum();
+      getNextToken();
+    } else if (m_token == TokenKind.kLeftBrace) {
+      decl.enumValues = parseEnum();
+      getNextToken();
+    }
+
+    // type name
+    expect(TokenKind.kIdentifier);
+    decl.typeString = m_lexer.getTokenText();
+    getNextToken();
+
+    // identifier name
+    expect(TokenKind.kIdentifier);
+    decl.name = m_lexer.getTokenText();
+    getNextToken();
+
+    // array or bit field
+    if (m_token == TokenKind.kLeftBracket) {
+      getNextToken();
+      expect(TokenKind.kInteger);
+      String valueStr = m_lexer.getTokenText();
+      int value;
+      try {
+        value = Integer.parseInt(valueStr);
+      } catch (NumberFormatException e) {
+        value = 0;
+      }
+      if (value > 0) {
+        decl.arraySize = value;
+      } else {
+        throw new ParseException(
+            m_lexer.m_pos, "array size '" + valueStr + "' is not a positive integer");
+      }
+      getNextToken();
+      expect(TokenKind.kRightBracket);
+      getNextToken();
+    } else if (m_token == TokenKind.kColon) {
+      getNextToken();
+      expect(TokenKind.kInteger);
+      String valueStr = m_lexer.getTokenText();
+      int value;
+      try {
+        value = Integer.parseInt(valueStr);
+      } catch (NumberFormatException e) {
+        value = 0;
+      }
+      if (value > 0) {
+        decl.bitWidth = value;
+      } else {
+        throw new ParseException(
+            m_lexer.m_pos, "bitfield width '" + valueStr + "' is not a positive integer");
+      }
+      getNextToken();
+    }
+
+    // declaration must end with EOF or semicolon
+    if (m_token != TokenKind.kEndOfInput) {
+      expect(TokenKind.kSemicolon);
+    }
+
+    return decl;
+  }
+
+  private Map<String, Long> parseEnum() throws ParseException {
+    Map<String, Long> map = new HashMap<>();
+
+    // we start with current = '{'
+    getNextToken();
+    while (m_token != TokenKind.kRightBrace) {
+      expect(TokenKind.kIdentifier);
+      final String name = m_lexer.getTokenText();
+      getNextToken();
+      expect(TokenKind.kEquals);
+      getNextToken();
+      expect(TokenKind.kInteger);
+      String valueStr = m_lexer.getTokenText();
+      long value;
+      try {
+        value = Long.parseLong(valueStr);
+      } catch (NumberFormatException e) {
+        throw new ParseException(m_lexer.m_pos, "could not parse enum value '" + valueStr + "'");
+      }
+      map.put(name, value);
+      getNextToken();
+      if (m_token == TokenKind.kRightBrace) {
+        break;
+      }
+      expect(TokenKind.kComma);
+      getNextToken();
+    }
+    return map;
+  }
+
+  private TokenKind getNextToken() {
+    m_token = m_lexer.scan();
+    return m_token;
+  }
+
+  private void expect(TokenKind kind) throws ParseException {
+    if (m_token != kind) {
+      throw new ParseException(
+          m_lexer.m_pos, "expected " + kind + ", got '" + m_lexer.getTokenText() + "'");
+    }
+  }
+
+  final Lexer m_lexer;
+  TokenKind m_token;
+}

wpiutil/src/main/java/edu/wpi/first/util/struct/parser/TokenKind.java

@@ -0,0 +1,32 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+package edu.wpi.first.util.struct.parser;
+
+/** A lexed raw struct schema token. */
+public enum TokenKind {
+  kUnknown("unknown"),
+  kInteger("integer"),
+  kIdentifier("identifier"),
+  kLeftBracket("'['"),
+  kRightBracket("']'"),
+  kLeftBrace("'{'"),
+  kRightBrace("'}'"),
+  kColon("':'"),
+  kSemicolon("';'"),
+  kComma("','"),
+  kEquals("'='"),
+  kEndOfInput("<EOF>");
+
+  private final String m_name;
+
+  TokenKind(String name) {
+    this.m_name = name;
+  }
+
+  @Override
+  public String toString() {
+    return m_name;
+  }
+}

wpiutil/src/main/native/cpp/DataLog.cpp

@@ -31,6 +31,7 @@
 #include "wpi/Endian.h"
 #include "wpi/Logger.h"
 #include "wpi/MathExtras.h"
+#include "wpi/SmallString.h"
 #include "wpi/fs.h"
 #include "wpi/timestamp.h"
 
@@ -211,6 +212,33 @@ void DataLog::Resume() {
   m_paused = false;
 }
 
+bool DataLog::HasSchema(std::string_view name) const {
+  std::scoped_lock lock{m_mutex};
+  wpi::SmallString<128> fullName{"/.schema/"};
+  fullName += name;
+  auto it = m_entries.find(fullName);
+  return it != m_entries.end();
+}
+
+void DataLog::AddSchema(std::string_view name, std::string_view type,
+                        std::span<const uint8_t> schema, int64_t timestamp) {
+  std::scoped_lock lock{m_mutex};
+  wpi::SmallString<128> fullName{"/.schema/"};
+  fullName += name;
+  auto& entryInfo = m_entries[fullName];
+  if (entryInfo.id != 0) {
+    return;  // don't add duplicates
+  }
+  int entry = StartImpl(fullName, type, {}, timestamp);
+
+  // inline AppendRaw() without releasing lock
+  if (entry <= 0) {
+    [[unlikely]] return;  // should never happen, but check anyway
+  }
+  StartRecord(entry, timestamp, schema.size(), 0);
+  AppendImpl(schema);
+}
+
 static void WriteToFile(fs::file_t f, std::span<const uint8_t> data,
                         std::string_view filename, wpi::Logger& msglog) {
   do {
@@ -484,6 +512,11 @@ void DataLog::WriterThreadMain(
 int DataLog::Start(std::string_view name, std::string_view type,
                    std::string_view metadata, int64_t timestamp) {
   std::scoped_lock lock{m_mutex};
+  return StartImpl(name, type, metadata, timestamp);
+}
+
+int DataLog::StartImpl(std::string_view name, std::string_view type,
+                       std::string_view metadata, int64_t timestamp) {
   auto& entryInfo = m_entries[name];
   if (entryInfo.id == 0) {
     entryInfo.id = ++m_lastId;

wpiutil/src/main/native/cpp/jni/DataLogJNI.cpp

@@ -111,6 +111,47 @@ Java_edu_wpi_first_util_datalog_DataLogJNI_resume
   reinterpret_cast<DataLog*>(impl)->Resume();
 }
 
+/*
+ * Class:     edu_wpi_first_util_datalog_DataLogJNI
+ * Method:    addSchema
+ * Signature: (JLjava/lang/String;Ljava/lang/String;[BJ)V
+ */
+JNIEXPORT void JNICALL
+Java_edu_wpi_first_util_datalog_DataLogJNI_addSchema
+  (JNIEnv* env, jclass, jlong impl, jstring name, jstring type,
+   jbyteArray schema, jlong timestamp)
+{
+  if (impl == 0) {
+    wpi::ThrowNullPointerException(env, "impl is null");
+    return;
+  }
+  reinterpret_cast<DataLog*>(impl)->AddSchema(
+      JStringRef{env, name}, JStringRef{env, type},
+      JSpan<const jbyte>{env, schema}.uarray(), timestamp);
+}
+
+/*
+ * Class:     edu_wpi_first_util_datalog_DataLogJNI
+ * Method:    addSchemaString
+ * Signature: (JLjava/lang/String;Ljava/lang/String;Ljava/lang/String;J)V
+ */
+JNIEXPORT void JNICALL
+Java_edu_wpi_first_util_datalog_DataLogJNI_addSchemaString
+  (JNIEnv* env, jclass, jlong impl, jstring name, jstring type, jstring schema,
+   jlong timestamp)
+{
+  if (impl == 0) {
+    wpi::ThrowNullPointerException(env, "impl is null");
+    return;
+  }
+  JStringRef schemaStr{env, schema};
+  std::string_view schemaView = schemaStr.str();
+  reinterpret_cast<DataLog*>(impl)->AddSchema(
+      JStringRef{env, name}, JStringRef{env, type},
+      {reinterpret_cast<const uint8_t*>(schemaView.data()), schemaView.size()},
+      timestamp);
+}
+
 /*
  * Class:     edu_wpi_first_util_datalog_DataLogJNI
  * Method:    start

wpiutil/src/main/native/cpp/protobuf/Protobuf.cpp

@@ -0,0 +1,194 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#include "wpi/protobuf/Protobuf.h"
+
+#include <fmt/format.h>
+#include <google/protobuf/descriptor.h>
+#include <google/protobuf/descriptor.pb.h>
+#include <google/protobuf/io/zero_copy_stream.h>
+#include <google/protobuf/message.h>
+
+#include "wpi/SmallVector.h"
+
+using namespace wpi;
+
+using google::protobuf::Arena;
+using google::protobuf::FileDescriptor;
+using google::protobuf::FileDescriptorProto;
+
+namespace {
+class VectorOutputStream final
+    : public google::protobuf::io::ZeroCopyOutputStream {
+ public:
+  // Create a StringOutputStream which appends bytes to the given string.
+  // The string remains property of the caller, but it is mutated in arbitrary
+  // ways and MUST NOT be accessed in any way until you're done with the
+  // stream. Either be sure there's no further usage, or (safest) destroy the
+  // stream before using the contents.
+  //
+  // Hint:  If you call target->reserve(n) before creating the stream,
+  //   the first call to Next() will return at least n bytes of buffer
+  //   space.
+  explicit VectorOutputStream(std::vector<uint8_t>& target) : target_{target} {}
+  VectorOutputStream(const VectorOutputStream&) = delete;
+  ~VectorOutputStream() override = default;
+
+  VectorOutputStream& operator=(const VectorOutputStream&) = delete;
+
+  // implements ZeroCopyOutputStream ---------------------------------
+  bool Next(void** data, int* size) override;
+  void BackUp(int count) override { target_.resize(target_.size() - count); }
+  int64_t ByteCount() const override { return target_.size(); }
+
+ private:
+  static constexpr size_t kMinimumSize = 16;
+
+  std::vector<uint8_t>& target_;
+};
+
+class SmallVectorOutputStream final
+    : public google::protobuf::io::ZeroCopyOutputStream {
+ public:
+  // Create a StringOutputStream which appends bytes to the given string.
+  // The string remains property of the caller, but it is mutated in arbitrary
+  // ways and MUST NOT be accessed in any way until you're done with the
+  // stream. Either be sure there's no further usage, or (safest) destroy the
+  // stream before using the contents.
+  //
+  // Hint:  If you call target->reserve(n) before creating the stream,
+  //   the first call to Next() will return at least n bytes of buffer
+  //   space.
+  explicit SmallVectorOutputStream(wpi::SmallVectorImpl<uint8_t>& target)
+      : target_{target} {
+    target.resize(0);
+  }
+  SmallVectorOutputStream(const SmallVectorOutputStream&) = delete;
+  ~SmallVectorOutputStream() override = default;
+
+  SmallVectorOutputStream& operator=(const SmallVectorOutputStream&) = delete;
+
+  // implements ZeroCopyOutputStream ---------------------------------
+  bool Next(void** data, int* size) override;
+  void BackUp(int count) override { target_.resize(target_.size() - count); }
+  int64_t ByteCount() const override { return target_.size(); }
+
+ private:
+  static constexpr size_t kMinimumSize = 16;
+
+  wpi::SmallVectorImpl<uint8_t>& target_;
+};
+}  // namespace
+
+bool VectorOutputStream::Next(void** data, int* size) {
+  size_t old_size = target_.size();
+
+  // Grow the string.
+  size_t new_size;
+  if (old_size < target_.capacity()) {
+    // Resize to match its capacity, since we can get away
+    // without a memory allocation this way.
+    new_size = target_.capacity();
+  } else {
+    // Size has reached capacity, try to double it.
+    new_size = old_size * 2;
+  }
+  // Avoid integer overflow in returned '*size'.
+  new_size = (std::min)(new_size, old_size + (std::numeric_limits<int>::max)());
+  // Increase the size, also make sure that it is at least kMinimumSize.
+  target_.resize((std::max)(new_size, kMinimumSize));
+
+  *data = target_.data() + old_size;
+  *size = target_.size() - old_size;
+  return true;
+}
+
+bool SmallVectorOutputStream::Next(void** data, int* size) {
+  size_t old_size = target_.size();
+
+  // Grow the string.
+  size_t new_size;
+  if (old_size < target_.capacity()) {
+    // Resize to match its capacity, since we can get away
+    // without a memory allocation this way.
+    new_size = target_.capacity();
+  } else {
+    // Size has reached capacity, try to double it.
+    new_size = old_size * 2;
+  }
+  // Avoid integer overflow in returned '*size'.
+  new_size = (std::min)(new_size, old_size + (std::numeric_limits<int>::max)());
+  // Increase the size, also make sure that it is at least kMinimumSize.
+  target_.resize_for_overwrite((std::max)(new_size, kMinimumSize));
+
+  *data = target_.data() + old_size;
+  *size = target_.size() - old_size;
+  return true;
+}
+
+void detail::DeleteProtobuf(google::protobuf::Message* msg) {
+  if (msg && !msg->GetArena()) {
+    delete msg;
+  }
+}
+
+bool detail::ParseProtobuf(google::protobuf::Message* msg,
+                           std::span<const uint8_t> data) {
+  return msg->ParseFromArray(data.data(), data.size());
+}
+
+bool detail::SerializeProtobuf(wpi::SmallVectorImpl<uint8_t>& out,
+                               const google::protobuf::Message& msg) {
+  SmallVectorOutputStream stream{out};
+  return msg.SerializeToZeroCopyStream(&stream);
+}
+
+bool detail::SerializeProtobuf(std::vector<uint8_t>& out,
+                               const google::protobuf::Message& msg) {
+  VectorOutputStream stream{out};
+  return msg.SerializeToZeroCopyStream(&stream);
+}
+
+std::string detail::GetTypeString(const google::protobuf::Message& msg) {
+  return fmt::format("proto:{}", msg.GetDescriptor()->full_name());
+}
+
+static void ForEachProtobufDescriptorImpl(
+    const FileDescriptor* desc,
+    function_ref<bool(std::string_view typeString)> exists,
+    function_ref<void(std::string_view typeString,
+                      std::span<const uint8_t> schema)>
+        fn,
+    Arena* arena, FileDescriptorProto** descproto) {
+  std::string name = fmt::format("proto:{}", desc->name());
+  if (exists(name)) {
+    return;
+  }
+  for (int i = 0, ndep = desc->dependency_count(); i < ndep; ++i) {
+    ForEachProtobufDescriptorImpl(desc->dependency(i), exists, fn, arena,
+                                  descproto);
+  }
+  if (!*descproto) {
+    *descproto = Arena::CreateMessage<FileDescriptorProto>(arena);
+  }
+  (*descproto)->Clear();
+  desc->CopyTo(*descproto);
+  SmallVector<uint8_t, 128> buf;
+  detail::SerializeProtobuf(buf, **descproto);
+  fn(name, buf);
+}
+
+void detail::ForEachProtobufDescriptor(
+    const google::protobuf::Message& msg,
+    function_ref<bool(std::string_view filename)> exists,
+    function_ref<void(std::string_view filename,
+                      std::span<const uint8_t> descriptor)>
+        fn) {
+  FileDescriptorProto* descproto = nullptr;
+  ForEachProtobufDescriptorImpl(msg.GetDescriptor()->file(), exists, fn,
+                                msg.GetArena(), &descproto);
+  if (descproto && !msg.GetArena()) {
+    delete descproto;
+  }
+}

wpiutil/src/main/native/cpp/protobuf/ProtobufMessageDatabase.cpp

@@ -0,0 +1,119 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#include "wpi/protobuf/ProtobufMessageDatabase.h"
+
+#include <google/protobuf/descriptor.h>
+
+using namespace wpi;
+
+using google::protobuf::Arena;
+using google::protobuf::FileDescriptorProto;
+using google::protobuf::Message;
+
+void ProtobufMessageDatabase::Add(std::string_view filename,
+                                  std::span<const uint8_t> data) {
+  auto& file = m_files[filename];
+  if (file.complete) {
+    file.complete = false;
+
+    // rebuild the pool EXCEPT for this descriptor
+    m_pool = std::make_unique<google::protobuf::DescriptorPool>();
+
+    for (auto&& p : m_files) {
+      p.second.inPool = false;
+    }
+    for (auto&& p : m_files) {
+      if (p.second.complete && !p.second.inPool) {
+        Rebuild(p.second);
+      }
+    }
+
+    // clear messages and reset factory; Find() will recreate as needed
+    m_msgs.clear();
+    m_factory = std::make_unique<google::protobuf::DynamicMessageFactory>();
+  }
+
+  if (!file.proto) {
+    file.proto = std::unique_ptr<FileDescriptorProto>{
+        Arena::CreateMessage<FileDescriptorProto>(nullptr)};
+  } else {
+    // replacing an existing one; remove any previously existing refs
+    for (auto&& dep : file.proto->dependency()) {
+      auto& depFile = m_files[dep];
+      std::erase(depFile.uses, filename);
+    }
+    file.proto->Clear();
+  }
+
+  // parse data
+  if (!file.proto->ParseFromArray(data.data(), data.size())) {
+    return;
+  }
+
+  Build(filename, file);
+}
+
+Message* ProtobufMessageDatabase::Find(std::string_view name) const {
+  // cached
+  auto& msg = m_msgs[name];
+  if (msg) {
+    return msg.get();
+  }
+
+  // need to create it
+  auto desc = m_pool->FindMessageTypeByName(std::string{name});
+  if (!desc) {
+    return nullptr;
+  }
+  msg = std::unique_ptr<Message>{m_factory->GetPrototype(desc)->New(nullptr)};
+  return msg.get();
+}
+
+void ProtobufMessageDatabase::Build(std::string_view filename,
+                                    ProtoFile& file) {
+  if (file.complete) {
+    return;
+  }
+  // are all of the dependencies complete?
+  bool complete = true;
+  for (auto&& dep : file.proto->dependency()) {
+    auto& depFile = m_files[dep];
+    if (!depFile.complete) {
+      complete = false;
+    }
+    depFile.uses.emplace_back(filename);
+  }
+  if (!complete) {
+    return;
+  }
+
+  // add to pool
+  if (!m_pool->BuildFile(*file.proto)) {
+    return;
+  }
+  file.inPool = true;
+  file.complete = true;
+
+  // recursively validate all uses
+  for (auto&& use : file.uses) {
+    Build(use, m_files[use]);
+  }
+}
+
+bool ProtobufMessageDatabase::Rebuild(ProtoFile& file) {
+  for (auto&& dep : file.proto->dependency()) {
+    auto& depFile = m_files[dep];
+    if (!depFile.inPool) {
+      if (!Rebuild(depFile)) {
+        return false;
+      }
+    }
+  }
+  if (!m_pool->BuildFile(*file.proto)) {
+    return false;
+  }
+  file.inPool = true;
+  return true;
+}

wpiutil/src/main/native/cpp/struct/DynamicStruct.cpp

@@ -0,0 +1,444 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#include "wpi/struct/DynamicStruct.h"
+
+#include <algorithm>
+
+#include <fmt/format.h>
+
+#include "wpi/Endian.h"
+#include "wpi/SmallString.h"
+#include "wpi/SmallVector.h"
+#include "wpi/raw_ostream.h"
+#include "wpi/struct/SchemaParser.h"
+
+using namespace wpi;
+
+static size_t TypeToSize(StructFieldType type) {
+  switch (type) {
+    case StructFieldType::kBool:
+    case StructFieldType::kChar:
+    case StructFieldType::kInt8:
+    case StructFieldType::kUint8:
+      return 1;
+    case StructFieldType::kInt16:
+    case StructFieldType::kUint16:
+      return 2;
+    case StructFieldType::kInt32:
+    case StructFieldType::kUint32:
+    case StructFieldType::kFloat:
+      return 4;
+    case StructFieldType::kInt64:
+    case StructFieldType::kUint64:
+    case StructFieldType::kDouble:
+      return 8;
+    default:
+      return 0;
+  }
+}
+
+static StructFieldType TypeStringToType(std::string_view str) {
+  if (str == "bool") {
+    return StructFieldType::kBool;
+  } else if (str == "char") {
+    return StructFieldType::kChar;
+  } else if (str == "int8") {
+    return StructFieldType::kInt8;
+  } else if (str == "int16") {
+    return StructFieldType::kInt16;
+  } else if (str == "int32") {
+    return StructFieldType::kInt32;
+  } else if (str == "int64") {
+    return StructFieldType::kInt64;
+  } else if (str == "uint8") {
+    return StructFieldType::kUint8;
+  } else if (str == "uint16") {
+    return StructFieldType::kUint16;
+  } else if (str == "uint32") {
+    return StructFieldType::kUint32;
+  } else if (str == "uint64") {
+    return StructFieldType::kUint64;
+  } else if (str == "float" || str == "float32") {
+    return StructFieldType::kFloat;
+  } else if (str == "double" || str == "float64") {
+    return StructFieldType::kDouble;
+  } else {
+    return StructFieldType::kStruct;
+  }
+}
+
+static inline unsigned int ToBitWidth(size_t size, unsigned int bitWidth) {
+  if (bitWidth == 0) {
+    return size * 8;
+  } else {
+    return bitWidth;
+  }
+}
+
+static inline uint64_t ToBitMask(size_t size, unsigned int bitWidth) {
+  if (size == 0) {
+    return 0;
+  } else {
+    return UINT64_MAX >> (64 - ToBitWidth(size, bitWidth));
+  }
+}
+
+StructFieldDescriptor::StructFieldDescriptor(
+    const StructDescriptor* parent, std::string_view name, StructFieldType type,
+    size_t size, size_t arraySize, unsigned int bitWidth, EnumValues enumValues,
+    const StructDescriptor* structDesc, const private_init&)
+    : m_parent{parent},
+      m_name{name},
+      m_size{size},
+      m_arraySize{arraySize},
+      m_enum{std::move(enumValues)},
+      m_struct{structDesc},
+      m_bitMask{ToBitMask(size, bitWidth)},
+      m_type{type},
+      m_bitWidth{ToBitWidth(size, bitWidth)} {}
+
+const StructFieldDescriptor* StructDescriptor::FindFieldByName(
+    std::string_view name) const {
+  auto it = m_fieldsByName.find(name);
+  if (it == m_fieldsByName.end()) {
+    return nullptr;
+  }
+  return &m_fields[it->second];
+}
+
+bool StructDescriptor::CheckCircular(
+    wpi::SmallVectorImpl<const StructDescriptor*>& stack) const {
+  stack.emplace_back(this);
+  for (auto&& ref : m_references) {
+    if (std::find(stack.begin(), stack.end(), ref) != stack.end()) {
+      [[unlikely]] return false;
+    }
+    if (!ref->CheckCircular(stack)) {
+      [[unlikely]] return false;
+    }
+  }
+  stack.pop_back();
+  return true;
+}
+
+std::string StructDescriptor::CalculateOffsets(
+    wpi::SmallVectorImpl<const StructDescriptor*>& stack) {
+  size_t offset = 0;
+  unsigned int shift = 0;
+  size_t prevBitfieldSize = 0;
+  for (auto&& field : m_fields) {
+    if (!field.IsBitField()) {
+      [[likely]] shift = 0;        // reset shift on non-bitfield element
+      offset += prevBitfieldSize;  // finish bitfield if active
+      prevBitfieldSize = 0;        // previous is now not bitfield
+      field.m_offset = offset;
+      if (field.m_struct) {
+        if (!field.m_struct->IsValid()) {
+          m_valid = false;
+          [[unlikely]] return {};
+        }
+        field.m_size = field.m_struct->m_size;
+      }
+      offset += field.m_size * field.m_arraySize;
+    } else {
+      if (field.m_type == StructFieldType::kBool && prevBitfieldSize != 0 &&
+          (shift + 1) <= (prevBitfieldSize * 8)) {
+        // bool takes on size of preceding bitfield type (if it fits)
+        field.m_size = prevBitfieldSize;
+      } else if (field.m_size != prevBitfieldSize ||
+                 (shift + field.m_bitWidth) > (field.m_size * 8)) {
+        shift = 0;
+        offset += prevBitfieldSize;
+      }
+      prevBitfieldSize = field.m_size;
+      field.m_offset = offset;
+      field.m_bitShift = shift;
+      shift += field.m_bitWidth;
+    }
+  }
+
+  // update struct size
+  m_size = offset + prevBitfieldSize;
+  m_valid = true;
+
+  // now that we're valid, referring types may be too
+  stack.emplace_back(this);
+  for (auto&& ref : m_references) {
+    if (std::find(stack.begin(), stack.end(), ref) != stack.end()) {
+      [[unlikely]] return fmt::format(
+          "internal error (inconsistent data): circular struct reference "
+          "between {} and {}",
+          m_name, ref->m_name);
+    }
+    auto err = ref->CalculateOffsets(stack);
+    if (!err.empty()) {
+      [[unlikely]] return err;
+    }
+  }
+  stack.pop_back();
+  return {};
+}
+
+const StructDescriptor* StructDescriptorDatabase::Add(std::string_view name,
+                                                      std::string_view schema,
+                                                      std::string* err) {
+  structparser::Parser parser{schema};
+  structparser::ParsedSchema parsed;
+  if (!parser.Parse(&parsed)) {
+    *err = fmt::format("parse error: {}", parser.GetError());
+    [[unlikely]] return nullptr;
+  }
+
+  // turn parsed schema into descriptors
+  auto& theStruct = m_structs[name];
+  if (!theStruct) {
+    theStruct = std::make_unique<StructDescriptor>(
+        name, StructDescriptor::private_init{});
+  }
+  theStruct->m_schema = schema;
+  theStruct->m_fields.clear();
+  theStruct->m_fields.reserve(parsed.declarations.size());
+  bool isValid = true;
+  for (auto&& decl : parsed.declarations) {
+    auto type = TypeStringToType(decl.typeString);
+    size_t size = TypeToSize(type);
+
+    // bitfield checks
+    if (decl.bitWidth != 0) {
+      // only integer or boolean types are allowed
+      if (type == StructFieldType::kChar || type == StructFieldType::kFloat ||
+          type == StructFieldType::kDouble ||
+          type == StructFieldType::kStruct) {
+        *err = fmt::format("field {}: type {} cannot be bitfield", decl.name,
+                           decl.typeString);
+        [[unlikely]] return nullptr;
+      }
+
+      // bit width cannot be larger than field size
+      if (decl.bitWidth > (size * 8)) {
+        *err = fmt::format("field {}: bit width {} exceeds type size",
+                           decl.name, decl.bitWidth);
+        [[unlikely]] return nullptr;
+      }
+
+      // bit width must be 1 for booleans
+      if (type == StructFieldType::kBool && decl.bitWidth != 1) {
+        *err = fmt::format("field {}: bit width must be 1 for bool type",
+                           decl.name);
+        [[unlikely]] return nullptr;
+      }
+
+      // cannot combine array and bitfield (shouldn't parse, but double-check)
+      if (decl.arraySize > 1) {
+        *err = fmt::format("field {}: cannot combine array and bitfield",
+                           decl.name);
+        [[unlikely]] return nullptr;
+      }
+    }
+
+    // struct handling
+    const StructDescriptor* structDesc = nullptr;
+    if (type == StructFieldType::kStruct) {
+      // recursive definitions are not allowed
+      if (decl.typeString == name) {
+        *err = fmt::format("field {}: recursive struct reference", decl.name);
+        [[unlikely]] return nullptr;
+      }
+
+      // cross-reference struct, creating a placeholder if necessary
+      auto& aStruct = m_structs[decl.typeString];
+      if (!aStruct) {
+        aStruct = std::make_unique<StructDescriptor>(
+            decl.typeString, StructDescriptor::private_init{});
+      }
+
+      // if the struct isn't valid, we can't be valid either
+      if (aStruct->IsValid()) {
+        size = aStruct->GetSize();
+      } else {
+        isValid = false;
+      }
+
+      // add to cross-references for when the struct does become valid
+      aStruct->m_references.emplace_back(theStruct.get());
+      structDesc = aStruct.get();
+    }
+
+    // create field
+    if (!theStruct->m_fieldsByName
+             .insert({decl.name, theStruct->m_fields.size()})
+             .second) {
+      *err = fmt::format("duplicate field {}", decl.name);
+      [[unlikely]] return nullptr;
+    }
+
+    theStruct->m_fields.emplace_back(theStruct.get(), decl.name, type, size,
+                                     decl.arraySize, decl.bitWidth,
+                                     std::move(decl.enumValues), structDesc,
+                                     StructFieldDescriptor::private_init{});
+  }
+
+  theStruct->m_valid = isValid;
+  if (isValid) {
+    // we have all the info needed, so calculate field offset & shift
+    wpi::SmallVector<const StructDescriptor*, 16> stack;
+    auto err2 = theStruct->CalculateOffsets(stack);
+    if (!err2.empty()) {
+      *err = std::move(err2);
+      [[unlikely]] return nullptr;
+    }
+  } else {
+    // check for circular reference
+    wpi::SmallVector<const StructDescriptor*, 16> stack;
+    if (!theStruct->CheckCircular(stack)) {
+      wpi::SmallString<128> buf;
+      wpi::raw_svector_ostream os{buf};
+      for (auto&& elem : stack) {
+        if (!buf.empty()) {
+          os << " <- ";
+        }
+        os << elem->GetName();
+      }
+      *err = fmt::format("circular struct reference: {}", os.str());
+      [[unlikely]] return nullptr;
+    }
+  }
+
+  return theStruct.get();
+}
+
+const StructDescriptor* StructDescriptorDatabase::Find(
+    std::string_view name) const {
+  auto it = m_structs.find(name);
+  if (it == m_structs.end()) {
+    return nullptr;
+  }
+  return it->second.get();
+}
+
+uint64_t DynamicStruct::GetFieldImpl(const StructFieldDescriptor* field,
+                                     size_t arrIndex) const {
+  assert(field->m_parent == m_desc);
+  assert(m_desc->IsValid());
+  assert(arrIndex < field->m_arraySize);
+  uint64_t val;
+  switch (field->m_size) {
+    case 1:
+      val = m_data[field->m_offset + arrIndex];
+      break;
+    case 2:
+      val = support::endian::read16le(&m_data[field->m_offset + arrIndex * 2]);
+      break;
+    case 4:
+      val = support::endian::read32le(&m_data[field->m_offset + arrIndex * 4]);
+      break;
+    case 8:
+      val = support::endian::read64le(&m_data[field->m_offset + arrIndex * 8]);
+      break;
+    default:
+      assert(false && "invalid field size");
+      return 0;
+  }
+  return (val >> field->m_bitShift) & field->m_bitMask;
+}
+
+void MutableDynamicStruct::SetData(std::span<const uint8_t> data) {
+  assert(data.size() >= m_desc->GetSize());
+  std::copy(data.begin(), data.begin() + m_desc->GetSize(), m_data.begin());
+}
+
+void MutableDynamicStruct::SetStringField(const StructFieldDescriptor* field,
+                                          std::string_view value) {
+  assert(field->m_type == StructFieldType::kChar);
+  assert(field->m_parent == m_desc);
+  assert(m_desc->IsValid());
+  size_t len = (std::min)(field->m_arraySize, value.size());
+  std::copy(value.begin(), value.begin() + len,
+            reinterpret_cast<char*>(&m_data[field->m_offset]));
+  std::fill(&m_data[field->m_offset + len],
+            &m_data[field->m_offset + field->m_arraySize], 0);
+}
+
+void MutableDynamicStruct::SetStructField(const StructFieldDescriptor* field,
+                                          const DynamicStruct& value,
+                                          size_t arrIndex) {
+  assert(field->m_type == StructFieldType::kStruct);
+  assert(field->m_parent == m_desc);
+  assert(m_desc->IsValid());
+  assert(value.GetDescriptor() == field->m_struct);
+  assert(value.GetDescriptor()->IsValid());
+  assert(arrIndex < field->m_arraySize);
+  auto source = value.GetData();
+  size_t len = field->m_struct->GetSize();
+  std::copy(source.begin(), source.begin() + len,
+            m_data.begin() + field->m_offset + arrIndex * len);
+}
+
+void MutableDynamicStruct::SetFieldImpl(const StructFieldDescriptor* field,
+                                        uint64_t value, size_t arrIndex) {
+  assert(field->m_parent == m_desc);
+  assert(m_desc->IsValid());
+  assert(arrIndex < field->m_arraySize);
+
+  // common case is no bit shift and no masking
+  if (!field->IsBitField()) {
+    switch (field->m_size) {
+      case 1:
+        m_data[field->m_offset + arrIndex] = value;
+        break;
+      case 2:
+        support::endian::write16le(&m_data[field->m_offset + arrIndex * 2],
+                                   value);
+        break;
+      case 4:
+        support::endian::write32le(&m_data[field->m_offset + arrIndex * 4],
+                                   value);
+        break;
+      case 8:
+        support::endian::write64le(&m_data[field->m_offset + arrIndex * 8],
+                                   value);
+        break;
+      default:
+        assert(false && "invalid field size");
+    }
+    return;
+  }
+
+  // handle bit shifting and masking into current value
+  switch (field->m_size) {
+    case 1: {
+      uint8_t* data = &m_data[field->m_offset + arrIndex];
+      *data &= ~(field->m_bitMask << field->m_bitShift);
+      *data |= (value & field->m_bitMask) << field->m_bitShift;
+      break;
+    }
+    case 2: {
+      uint8_t* data = &m_data[field->m_offset + arrIndex * 2];
+      uint16_t val = support::endian::read16le(data);
+      val &= ~(field->m_bitMask << field->m_bitShift);
+      val |= (value & field->m_bitMask) << field->m_bitShift;
+      support::endian::write16le(data, val);
+      break;
+    }
+    case 4: {
+      uint8_t* data = &m_data[field->m_offset + arrIndex * 4];
+      uint32_t val = support::endian::read32le(data);
+      val &= ~(field->m_bitMask << field->m_bitShift);
+      val |= (value & field->m_bitMask) << field->m_bitShift;
+      support::endian::write32le(data, val);
+      break;
+    }
+    case 8: {
+      uint8_t* data = &m_data[field->m_offset + arrIndex * 8];
+      uint64_t val = support::endian::read64le(data);
+      val &= ~(field->m_bitMask << field->m_bitShift);
+      val |= (value & field->m_bitMask) << field->m_bitShift;
+      support::endian::write64le(data, val);
+      break;
+    }
+    default:
+      assert(false && "invalid field size");
+  }
+}

wpiutil/src/main/native/cpp/struct/SchemaParser.cpp

@@ -0,0 +1,238 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#include "wpi/struct/SchemaParser.h"
+
+#include <fmt/format.h>
+
+#include "wpi/StringExtras.h"
+
+using namespace wpi::structparser;
+
+std::string_view wpi::structparser::ToString(Token::Kind kind) {
+  switch (kind) {
+    case Token::kInteger:
+      return "integer";
+    case Token::kIdentifier:
+      return "identifier";
+    case Token::kLeftBracket:
+      return "'['";
+    case Token::kRightBracket:
+      return "']'";
+    case Token::kLeftBrace:
+      return "'{'";
+    case Token::kRightBrace:
+      return "'}'";
+    case Token::kColon:
+      return "':'";
+    case Token::kSemicolon:
+      return "';'";
+    case Token::kComma:
+      return "','";
+    case Token::kEquals:
+      return "'='";
+    case Token::kEndOfInput:
+      return "<EOF>";
+    default:
+      return "unknown";
+  }
+}
+
+Token Lexer::Scan() {
+  // skip whitespace
+  do {
+    Get();
+  } while (m_current == ' ' || m_current == '\t' || m_current == '\n' ||
+           m_current == '\r');
+  m_tokenStart = m_pos - 1;
+
+  switch (m_current) {
+    case '[':
+      return MakeToken(Token::kLeftBracket);
+    case ']':
+      return MakeToken(Token::kRightBracket);
+    case '{':
+      return MakeToken(Token::kLeftBrace);
+    case '}':
+      return MakeToken(Token::kRightBrace);
+    case ':':
+      return MakeToken(Token::kColon);
+    case ';':
+      return MakeToken(Token::kSemicolon);
+    case ',':
+      return MakeToken(Token::kComma);
+    case '=':
+      return MakeToken(Token::kEquals);
+    case '-':
+    case '0':
+    case '1':
+    case '2':
+    case '3':
+    case '4':
+    case '5':
+    case '6':
+    case '7':
+    case '8':
+    case '9':
+      return ScanInteger();
+    case -1:
+      return {Token::kEndOfInput, {}};
+    default:
+      if (isAlpha(m_current) || m_current == '_') {
+        [[likely]] return ScanIdentifier();
+      }
+      return MakeToken(Token::kUnknown);
+  }
+}
+
+Token Lexer::ScanInteger() {
+  do {
+    Get();
+  } while (isDigit(m_current));
+  Unget();
+  return MakeToken(Token::kInteger);
+}
+
+Token Lexer::ScanIdentifier() {
+  do {
+    Get();
+  } while (isAlnum(m_current) || m_current == '_');
+  Unget();
+  return MakeToken(Token::kIdentifier);
+}
+
+void Parser::FailExpect(Token::Kind desired) {
+  Fail(fmt::format("expected {}, got '{}'", ToString(desired), m_token.text));
+}
+
+void Parser::Fail(std::string_view msg) {
+  m_error = fmt::format("{}: {}", m_lexer.GetPosition(), msg);
+}
+
+bool Parser::Parse(ParsedSchema* out) {
+  do {
+    GetNextToken();
+    if (m_token.Is(Token::kSemicolon)) {
+      continue;
+    }
+    if (m_token.Is(Token::kEndOfInput)) {
+      break;
+    }
+    if (!ParseDeclaration(&out->declarations.emplace_back())) {
+      [[unlikely]] return false;
+    }
+  } while (m_token.kind != Token::kEndOfInput);
+  return true;
+}
+
+bool Parser::ParseDeclaration(ParsedDeclaration* out) {
+  // optional enum specification
+  if (m_token.Is(Token::kIdentifier) && m_token.text == "enum") {
+    GetNextToken();
+    if (!Expect(Token::kLeftBrace)) {
+      [[unlikely]] return false;
+    }
+    if (!ParseEnum(&out->enumValues)) {
+      [[unlikely]] return false;
+    }
+    GetNextToken();
+  } else if (m_token.Is(Token::kLeftBrace)) {
+    if (!ParseEnum(&out->enumValues)) {
+      [[unlikely]] return false;
+    }
+    GetNextToken();
+  }
+
+  // type name
+  if (!Expect(Token::kIdentifier)) {
+    [[unlikely]] return false;
+  }
+  out->typeString = m_token.text;
+  GetNextToken();
+
+  // identifier name
+  if (!Expect(Token::kIdentifier)) {
+    [[unlikely]] return false;
+  }
+  out->name = m_token.text;
+  GetNextToken();
+
+  // array or bit field
+  if (m_token.Is(Token::kLeftBracket)) {
+    GetNextToken();
+    if (!Expect(Token::kInteger)) {
+      [[unlikely]] return false;
+    }
+    auto val = parse_integer<uint64_t>(m_token.text, 10);
+    if (val && *val > 0) {
+      out->arraySize = *val;
+    } else {
+      Fail(fmt::format("array size '{}' is not a positive integer",
+                       m_token.text));
+      [[unlikely]] return false;
+    }
+    GetNextToken();
+    if (!Expect(Token::kRightBracket)) {
+      [[unlikely]] return false;
+    }
+    GetNextToken();
+  } else if (m_token.Is(Token::kColon)) {
+    GetNextToken();
+    if (!Expect(Token::kInteger)) {
+      [[unlikely]] return false;
+    }
+    auto val = parse_integer<unsigned int>(m_token.text, 10);
+    if (val && *val > 0) {
+      out->bitWidth = *val;
+    } else {
+      Fail(fmt::format("bitfield width '{}' is not a positive integer",
+                       m_token.text));
+      [[unlikely]] return false;
+    }
+    GetNextToken();
+  }
+
+  // declaration must end with EOF or semicolon
+  if (m_token.Is(Token::kEndOfInput)) {
+    return true;
+  }
+  return Expect(Token::kSemicolon);
+}
+
+bool Parser::ParseEnum(EnumValues* out) {
+  // we start with current = '{'
+  GetNextToken();
+  while (!m_token.Is(Token::kRightBrace)) {
+    if (!Expect(Token::kIdentifier)) {
+      [[unlikely]] return false;
+    }
+    std::string name;
+    name = m_token.text;
+    GetNextToken();
+    if (!Expect(Token::kEquals)) {
+      [[unlikely]] return false;
+    }
+    GetNextToken();
+    if (!Expect(Token::kInteger)) {
+      [[unlikely]] return false;
+    }
+    int64_t value;
+    if (auto val = parse_integer<int64_t>(m_token.text, 10)) {
+      value = *val;
+    } else {
+      Fail(fmt::format("could not parse enum value '{}'", m_token.text));
+      [[unlikely]] return false;
+    }
+    out->emplace_back(std::move(name), value);
+    GetNextToken();
+    if (m_token.Is(Token::kRightBrace)) {
+      break;
+    }
+    if (!Expect(Token::kComma)) {
+      [[unlikely]] return false;
+    }
+    GetNextToken();
+  }
+  return true;
+}

wpiutil/src/main/native/include/wpi/DataLog.h

@@ -7,19 +7,27 @@
 #include <stdint.h>
 
 #ifdef __cplusplus
+#include <concepts>
 #include <functional>
 #include <initializer_list>
 #include <memory>
+#include <ranges>
 #include <span>
 #include <string>
 #include <string_view>
 #include <thread>
+#include <utility>
 #include <vector>
+#include <version>
 
 #include "wpi/DenseMap.h"
+#include "wpi/SmallVector.h"
 #include "wpi/StringMap.h"
 #include "wpi/condition_variable.h"
 #include "wpi/mutex.h"
+#include "wpi/protobuf/Protobuf.h"
+#include "wpi/struct/Struct.h"
+#include "wpi/timestamp.h"
 #endif  // __cplusplus
 
 /**
@@ -170,6 +178,95 @@ class DataLog final {
    */
   void Resume();
 
+  /**
+   * Returns whether there is a data schema already registered with the given
+   * name.
+   *
+   * @param name Name (the string passed as the data type for records using this
+   *             schema)
+   * @return True if schema already registered
+   */
+  bool HasSchema(std::string_view name) const;
+
+  /**
+   * Registers a data schema.  Data schemas provide information for how a
+   * certain data type string can be decoded.  The type string of a data schema
+   * indicates the type of the schema itself (e.g. "protobuf" for protobuf
+   * schemas, "struct" for struct schemas, etc). In the data log, schemas are
+   * saved just like normal records, with the name being generated from the
+   * provided name: "/.schema/<name>".  Duplicate calls to this function with
+   * the same name are silently ignored.
+   *
+   * @param name Name (the string passed as the data type for records using this
+   *             schema)
+   * @param type Type of schema (e.g. "protobuf", "struct", etc)
+   * @param schema Schema data
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void AddSchema(std::string_view name, std::string_view type,
+                 std::span<const uint8_t> schema, int64_t timestamp = 0);
+
+  /**
+   * Registers a data schema.  Data schemas provide information for how a
+   * certain data type string can be decoded.  The type string of a data schema
+   * indicates the type of the schema itself (e.g. "protobuf" for protobuf
+   * schemas, "struct" for struct schemas, etc). In the data log, schemas are
+   * saved just like normal records, with the name being generated from the
+   * provided name: "/.schema/<name>".  Duplicate calls to this function with
+   * the same name are silently ignored.
+   *
+   * @param name Name (the string passed as the data type for records using this
+   *             schema)
+   * @param type Type of schema (e.g. "protobuf", "struct", etc)
+   * @param schema Schema data
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void AddSchema(std::string_view name, std::string_view type,
+                 std::string_view schema, int64_t timestamp = 0) {
+    AddSchema(
+        name, type,
+        std::span<const uint8_t>{
+            reinterpret_cast<const uint8_t*>(schema.data()), schema.size()},
+        timestamp);
+  }
+
+  /**
+   * Registers a protobuf schema. Duplicate calls to this function with the same
+   * name are silently ignored.
+   *
+   * @tparam T protobuf serializable type
+   * @param msg protobuf message
+   * @param timestamp Time stamp (0 to indicate now)
+   */
+  template <ProtobufSerializable T>
+  void AddProtobufSchema(ProtobufMessage<T>& msg, int64_t timestamp = 0) {
+    if (timestamp == 0) {
+      timestamp = Now();
+    }
+    msg.ForEachProtobufDescriptor(
+        [this](auto typeString) { return HasSchema(typeString); },
+        [this, timestamp](auto typeString, auto schema) {
+          AddSchema(typeString, "proto:FileDescriptorProto", schema, timestamp);
+        });
+  }
+
+  /**
+   * Registers a struct schema. Duplicate calls to this function with the same
+   * name are silently ignored.
+   *
+   * @tparam T struct serializable type
+   * @param timestamp Time stamp (0 to indicate now)
+   */
+  template <StructSerializable T>
+  void AddStructSchema(int64_t timestamp = 0) {
+    if (timestamp == 0) {
+      timestamp = Now();
+    }
+    ForEachStructSchema<T>([this, timestamp](auto typeString, auto schema) {
+      AddSchema(typeString, "structschema", schema, timestamp);
+    });
+  }
+
   /**
    * Start an entry.  Duplicate names are allowed (with the same type), and
    * result in the same index being returned (Start/Finish are reference
@@ -364,6 +461,8 @@ class DataLog final {
       std::function<void(std::span<const uint8_t> data)> write);
 
   // must be called with m_mutex held
+  int StartImpl(std::string_view name, std::string_view type,
+                std::string_view metadata, int64_t timestamp);
   uint8_t* StartRecord(uint32_t entry, uint64_t timestamp, uint32_t payloadSize,
                        size_t reserveSize);
   uint8_t* Reserve(size_t size);
@@ -821,6 +920,128 @@ class StringArrayLogEntry : public DataLogEntry {
   }
 };
 
+/**
+ * Log raw struct serializable objects.
+ */
+template <StructSerializable T>
+class StructLogEntry : public DataLogEntry {
+  using S = Struct<T>;
+
+ public:
+  StructLogEntry() = default;
+  StructLogEntry(DataLog& log, std::string_view name, int64_t timestamp = 0)
+      : StructLogEntry{log, name, {}, timestamp} {}
+  StructLogEntry(DataLog& log, std::string_view name, std::string_view metadata,
+                 int64_t timestamp = 0) {
+    m_log = &log;
+    log.AddStructSchema<T>(timestamp);
+    m_entry = log.Start(name, S::kTypeString, metadata, timestamp);
+  }
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param data Data to record
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void Append(const T& data, int64_t timestamp = 0) {
+    uint8_t buf[S::kSize];
+    S::Pack(buf, data);
+    m_log->AppendRaw(m_entry, buf, timestamp);
+  }
+};
+
+/**
+ * Log raw struct serializable array of objects.
+ */
+template <StructSerializable T>
+class StructArrayLogEntry : public DataLogEntry {
+  using S = Struct<T>;
+
+ public:
+  StructArrayLogEntry() = default;
+  StructArrayLogEntry(DataLog& log, std::string_view name,
+                      int64_t timestamp = 0)
+      : StructArrayLogEntry{log, name, {}, timestamp} {}
+  StructArrayLogEntry(DataLog& log, std::string_view name,
+                      std::string_view metadata, int64_t timestamp = 0) {
+    m_log = &log;
+    log.AddStructSchema<T>(timestamp);
+    m_entry =
+        log.Start(name, MakeStructArrayTypeString<T, std::dynamic_extent>(),
+                  metadata, timestamp);
+  }
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param data Data to record
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  template <typename U>
+#if __cpp_lib_ranges >= 201911L
+    requires std::ranges::range<U> &&
+             std::convertible_to<std::ranges::range_value_t<U>, T>
+#endif
+  void Append(U&& data, int64_t timestamp = 0) {
+    m_buf.Write(std::forward<U>(data), [&](auto bytes) {
+      m_log->AppendRaw(m_entry, bytes, timestamp);
+    });
+  }
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param data Data to record
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void Append(std::span<const T> data, int64_t timestamp = 0) {
+    m_buf.Write(
+        data, [&](auto bytes) { m_log->AppendRaw(m_entry, bytes, timestamp); });
+  }
+
+ private:
+  StructArrayBuffer<T> m_buf;
+};
+
+/**
+ * Log protobuf serializable objects.
+ */
+template <ProtobufSerializable T>
+class ProtobufLogEntry : public DataLogEntry {
+  using P = Protobuf<T>;
+
+ public:
+  ProtobufLogEntry() = default;
+  ProtobufLogEntry(DataLog& log, std::string_view name, int64_t timestamp = 0)
+      : ProtobufLogEntry{log, name, {}, timestamp} {}
+  ProtobufLogEntry(DataLog& log, std::string_view name,
+                   std::string_view metadata, int64_t timestamp = 0) {
+    m_log = &log;
+    log.AddProtobufSchema<T>(m_msg, timestamp);
+    m_entry = log.Start(name, m_msg.GetTypeString(), metadata, timestamp);
+  }
+
+  /**
+   * Appends a record to the log.
+   *
+   * @param data Data to record
+   * @param timestamp Time stamp (may be 0 to indicate now)
+   */
+  void Append(const T& data, int64_t timestamp = 0) {
+    SmallVector<uint8_t, 128> buf;
+    {
+      std::scoped_lock lock{m_mutex};
+      m_msg.Pack(buf, data);
+    }
+    m_log->AppendRaw(m_entry, buf, timestamp);
+  }
+
+ private:
+  wpi::mutex m_mutex;
+  ProtobufMessage<T> m_msg;
+};
+
 }  // namespace wpi::log
 
 extern "C" {

wpiutil/src/main/native/include/wpi/protobuf/Protobuf.h

@@ -0,0 +1,260 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#pragma once
+
+#include <stdint.h>
+
+#include <concepts>
+#include <optional>
+#include <span>
+#include <string>
+#include <string_view>
+#include <utility>
+#include <vector>
+
+#include "wpi/function_ref.h"
+
+namespace google::protobuf {
+class Arena;
+class Message;
+}  // namespace google::protobuf
+
+namespace wpi {
+
+template <typename T>
+class SmallVectorImpl;
+
+/**
+ * Protobuf serialization template. Unspecialized class has no members; only
+ * specializations of this class are useful, and only if they meet the
+ * ProtobufSerializable concept.
+ *
+ * @tparam T type to serialize/deserialize
+ */
+template <typename T>
+struct Protobuf {};
+
+/**
+ * Specifies that a type is capable of protobuf serialization and
+ * deserialization.
+ *
+ * This is designed for serializing complex flexible data structures using
+ * code generated from a .proto file. Serialization consists of writing
+ * values into a mutable protobuf Message and deserialization consists of
+ * reading values from an immutable protobuf Message.
+ *
+ * Implementations must define a template specialization for wpi::Protobuf with
+ * T being the type that is being serialized/deserialized, with the following
+ * static members (as enforced by this concept):
+ * - google::protobuf::Message* New(google::protobuf::Arena*): create a protobuf
+ *   message
+ * - T Unpack(const google::protobuf::Message&): function for deserialization
+ * - void Pack(google::protobuf::Message*, T&& value): function for
+ *   serialization
+ *
+ * To avoid pulling in the protobuf headers, these functions use
+ * google::protobuf::Message instead of a more specific type; implementations
+ * will need to static_cast to the correct type as created by New().
+ *
+ * Additionally: In a static block, call StructRegistry.registerClass() to
+ * register the class
+ */
+template <typename T>
+concept ProtobufSerializable = requires(
+    google::protobuf::Arena* arena, const google::protobuf::Message& inmsg,
+    google::protobuf::Message* outmsg, const T& value) {
+  typename Protobuf<typename std::remove_cvref_t<T>>;
+  {
+    Protobuf<typename std::remove_cvref_t<T>>::New(arena)
+  } -> std::same_as<google::protobuf::Message*>;
+  {
+    Protobuf<typename std::remove_cvref_t<T>>::Unpack(inmsg)
+  } -> std::same_as<typename std::remove_cvref_t<T>>;
+  Protobuf<typename std::remove_cvref_t<T>>::Pack(outmsg, value);
+};
+
+/**
+ * Specifies that a type is capable of in-place protobuf deserialization.
+ *
+ * In addition to meeting ProtobufSerializable, implementations must define a
+ * wpi::Protobuf<T> static member
+ * `void UnpackInto(T*, const google::protobuf::Message&)` to update the
+ * pointed-to T with the contents of the message.
+ */
+template <typename T>
+concept MutableProtobufSerializable =
+    ProtobufSerializable<T> &&
+    requires(T* out, const google::protobuf::Message& msg) {
+      Protobuf<typename std::remove_cvref_t<T>>::UnpackInto(out, msg);
+    };
+
+/**
+ * Unpack a serialized protobuf message.
+ *
+ * @tparam T object type
+ * @param msg protobuf message
+ * @return Deserialized object
+ */
+template <ProtobufSerializable T>
+inline T UnpackProtobuf(const google::protobuf::Message& msg) {
+  return Protobuf<T>::Unpack(msg);
+}
+
+/**
+ * Pack a serialized protobuf message.
+ *
+ * @param msg protobuf message (mutable, output)
+ * @param value object
+ */
+template <ProtobufSerializable T>
+inline void PackProtobuf(google::protobuf::Message* msg, const T& value) {
+  Protobuf<typename std::remove_cvref_t<T>>::Pack(msg, value);
+}
+
+/**
+ * Unpack a serialized struct into an existing object, overwriting its contents.
+ *
+ * @param out object (output)
+ * @param msg protobuf message
+ */
+template <ProtobufSerializable T>
+inline void UnpackProtobufInto(T* out, const google::protobuf::Message& msg) {
+  if constexpr (MutableProtobufSerializable<T>) {
+    Protobuf<T>::UnpackInto(out, msg);
+  } else {
+    *out = UnpackProtobuf<T>(msg);
+  }
+}
+
+// these detail functions avoid the need to include protobuf headers
+namespace detail {
+void DeleteProtobuf(google::protobuf::Message* msg);
+bool ParseProtobuf(google::protobuf::Message* msg,
+                   std::span<const uint8_t> data);
+bool SerializeProtobuf(wpi::SmallVectorImpl<uint8_t>& out,
+                       const google::protobuf::Message& msg);
+bool SerializeProtobuf(std::vector<uint8_t>& out,
+                       const google::protobuf::Message& msg);
+std::string GetTypeString(const google::protobuf::Message& msg);
+void ForEachProtobufDescriptor(
+    const google::protobuf::Message& msg,
+    function_ref<bool(std::string_view filename)> wants,
+    function_ref<void(std::string_view filename,
+                      std::span<const uint8_t> descriptor)>
+        fn);
+}  // namespace detail
+
+/**
+ * Owning wrapper (ala std::unique_ptr) for google::protobuf::Message* that does
+ * not require the protobuf headers be included. Note this object is not thread
+ * safe; users of this object are required to provide any necessary thread
+ * safety.
+ *
+ * @tparam T serialized object type
+ */
+template <ProtobufSerializable T>
+class ProtobufMessage {
+ public:
+  explicit ProtobufMessage(google::protobuf::Arena* arena = nullptr)
+      : m_msg{Protobuf<T>::New(arena)} {}
+  ~ProtobufMessage() { detail::DeleteProtobuf(m_msg); }
+  ProtobufMessage(const ProtobufMessage&) = delete;
+  ProtobufMessage& operator=(const ProtobufMessage&) = delete;
+  ProtobufMessage(ProtobufMessage&& rhs) : m_msg{rhs.m_msg} {
+    rhs.m_msg = nullptr;
+  }
+  ProtobufMessage& operator=(ProtobufMessage&& rhs) {
+    std::swap(m_msg, rhs.m_msg);
+    return *this;
+  }
+
+  /**
+   * Gets the stored message object.
+   *
+   * @return google::protobuf::Message*
+   */
+  google::protobuf::Message* GetMessage() { return m_msg; }
+  const google::protobuf::Message* GetMessage() const { return m_msg; }
+
+  /**
+   * Unpacks from a byte array.
+   *
+   * @param data byte array
+   * @return Optional; empty if parsing failed
+   */
+  std::optional<T> Unpack(std::span<const uint8_t> data) {
+    if (!detail::ParseProtobuf(m_msg, data)) {
+      return std::nullopt;
+    }
+    return Protobuf<T>::Unpack(*m_msg);
+  }
+
+  /**
+   * Unpacks from a byte array into an existing object.
+   *
+   * @param[out] out output object
+   * @param[in] data byte array
+   * @return true if successful
+   */
+  bool UnpackInto(T* out, std::span<const uint8_t> data) {
+    if (!detail::ParseProtobuf(m_msg, data)) {
+      return false;
+    }
+    UnpackProtobufInto(out, *m_msg);
+    return true;
+  }
+
+  /**
+   * Packs object into a SmallVector.
+   *
+   * @param[out] out output bytes
+   * @param[in] value value
+   * @return true if successful
+   */
+  bool Pack(wpi::SmallVectorImpl<uint8_t>& out, const T& value) {
+    Protobuf<T>::Pack(m_msg, value);
+    return detail::SerializeProtobuf(out, *m_msg);
+  }
+
+  /**
+   * Packs object into a std::vector.
+   *
+   * @param[out] out output bytes
+   * @param[in] value value
+   * @return true if successful
+   */
+  bool Pack(std::vector<uint8_t>& out, const T& value) {
+    Protobuf<T>::Pack(m_msg, value);
+    return detail::SerializeProtobuf(out, *m_msg);
+  }
+
+  /**
+   * Gets the type string for the message.
+   *
+   * @return type string
+   */
+  std::string GetTypeString() const { return detail::GetTypeString(*m_msg); }
+
+  /**
+   * Loops over all protobuf descriptors including nested/referenced
+   * descriptors.
+   *
+   * @param exists function that returns false if fn should be called for the
+   *               given type string
+   * @param fn function to call for each descriptor
+   */
+  void ForEachProtobufDescriptor(
+      function_ref<bool(std::string_view filename)> exists,
+      function_ref<void(std::string_view filename,
+                        std::span<const uint8_t> descriptor)>
+          fn) {
+    detail::ForEachProtobufDescriptor(*m_msg, exists, fn);
+  }
+
+ private:
+  google::protobuf::Message* m_msg = nullptr;
+};
+
+}  // namespace wpi

wpiutil/src/main/native/include/wpi/protobuf/ProtobufMessageDatabase.h

@@ -0,0 +1,76 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#pragma once
+
+#include <memory>
+#include <span>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include <google/protobuf/descriptor.pb.h>
+#include <google/protobuf/descriptor_database.h>
+#include <google/protobuf/dynamic_message.h>
+
+#include "wpi/StringMap.h"
+
+namespace wpi {
+
+/**
+ * Database of protobuf dynamic messages. Unlike the protobuf descriptor pools
+ * and databases, this gracefully handles adding descriptors out of order and
+ * descriptors being replaced.
+ */
+class ProtobufMessageDatabase {
+ public:
+  /**
+   * Adds a file descriptor to the database. If the file references other
+   * files that have not yet been added, no messages in that file will be
+   * available until those files are added. Replaces any existing file with
+   * the same name.
+   *
+   * @param filename filename
+   * @param data FileDescriptorProto serialized data
+   */
+  void Add(std::string_view filename, std::span<const uint8_t> data);
+
+  /**
+   * Finds a message in the database by name.
+   *
+   * @param name type name
+   * @return protobuf message, or nullptr if not found
+   */
+  google::protobuf::Message* Find(std::string_view name) const;
+
+  /**
+   * Gets message factory.
+   *
+   * @return message factory
+   */
+  google::protobuf::MessageFactory* GetMessageFactory() {
+    return m_factory.get();
+  }
+
+ private:
+  struct ProtoFile {
+    std::unique_ptr<google::protobuf::FileDescriptorProto> proto;
+    std::vector<std::string> uses;
+    bool complete = false;
+    bool inPool = false;
+  };
+
+  void Build(std::string_view filename, ProtoFile& file);
+  bool Rebuild(ProtoFile& file);
+
+  std::unique_ptr<google::protobuf::DescriptorPool> m_pool =
+      std::make_unique<google::protobuf::DescriptorPool>();
+  std::unique_ptr<google::protobuf::DynamicMessageFactory> m_factory =
+      std::make_unique<google::protobuf::DynamicMessageFactory>();
+  wpi::StringMap<ProtoFile> m_files;  // indexed by filename
+  // indexed by type string
+  mutable wpi::StringMap<std::unique_ptr<google::protobuf::Message>> m_msgs;
+};
+
+}  // namespace wpi

wpiutil/src/main/native/include/wpi/struct/DynamicStruct.h

@@ -0,0 +1,682 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#pragma once
+
+#include <stdint.h>
+
+#include <cassert>
+#include <memory>
+#include <span>
+#include <string>
+#include <string_view>
+#include <utility>
+#include <vector>
+
+#include "wpi/MathExtras.h"
+#include "wpi/StringMap.h"
+#include "wpi/bit.h"
+
+namespace wpi {
+
+template <typename T>
+class SmallVectorImpl;
+
+class DynamicStruct;
+class MutableDynamicStruct;
+class StructDescriptor;
+class StructDescriptorDatabase;
+
+/**
+ * Known data types for raw struct dynamic fields (see StructFieldDescriptor).
+ */
+enum class StructFieldType {
+  kBool,
+  kChar,
+  kInt8,
+  kInt16,
+  kInt32,
+  kInt64,
+  kUint8,
+  kUint16,
+  kUint32,
+  kUint64,
+  kFloat,
+  kDouble,
+  kStruct
+};
+
+/**
+ * Raw struct dynamic field descriptor.
+ */
+class StructFieldDescriptor {
+  struct private_init {};
+  friend class DynamicStruct;
+  friend class MutableDynamicStruct;
+  friend class StructDescriptor;
+  friend class StructDescriptorDatabase;
+
+ public:
+  /**
+   * Set of enumerated values.
+   */
+  using EnumValues = std::vector<std::pair<std::string, int64_t>>;
+
+  StructFieldDescriptor(const StructDescriptor* parent, std::string_view name,
+                        StructFieldType type, size_t size, size_t arraySize,
+                        unsigned int bitWidth, EnumValues enumValues,
+                        const StructDescriptor* structDesc,
+                        const private_init&);
+
+  /**
+   * Gets the dynamic struct this field is contained in.
+   *
+   * @return struct descriptor
+   */
+  const StructDescriptor* GetParent() const { return m_parent; }
+
+  /**
+   * Gets the field name.
+   *
+   * @return field name
+   */
+  const std::string& GetName() const { return m_name; }
+
+  /**
+   * Gets the field type.
+   *
+   * @return field type
+   */
+  StructFieldType GetType() const { return m_type; }
+
+  /**
+   * Returns whether the field type is a signed integer.
+   *
+   * @return true if signed integer, false otherwise
+   */
+  bool IsInt() const {
+    return m_type == StructFieldType::kInt8 ||
+           m_type == StructFieldType::kInt16 ||
+           m_type == StructFieldType::kInt32 ||
+           m_type == StructFieldType::kInt64;
+  }
+
+  /**
+   * Returns whether the field type is an unsigned integer.
+   *
+   * @return true if unsigned integer, false otherwise
+   */
+  bool IsUint() const {
+    return m_type == StructFieldType::kUint8 ||
+           m_type == StructFieldType::kUint16 ||
+           m_type == StructFieldType::kUint32 ||
+           m_type == StructFieldType::kUint64;
+  }
+
+  /**
+   * Gets the underlying storage size of the field, in bytes.
+   *
+   * @return number of bytes
+   */
+  size_t GetSize() const { return m_size; }
+
+  /**
+   * Gets the storage offset of the field, in bytes.
+   *
+   * @return number of bytes from the start of the struct
+   */
+  size_t GetOffset() const { return m_offset; }
+
+  /**
+   * Gets the bit width of the field, in bits.
+   *
+   * @return number of bits
+   */
+  unsigned int GetBitWidth() const {
+    return m_bitWidth == 0 ? m_size * 8 : m_bitWidth;
+  }
+
+  /**
+   * Gets the bit mask for the field. The mask is always the least significant
+   * bits (it is not shifted).
+   *
+   * @return bit mask
+   */
+  uint64_t GetBitMask() const { return m_bitMask; }
+
+  /**
+   * Gets the bit shift for the field (LSB=0).
+   *
+   * @return number of bits
+   */
+  unsigned int GetBitShift() const { return m_bitShift; }
+
+  /**
+   * Returns whether the field is an array.
+   *
+   * @return true if array
+   */
+  bool IsArray() const { return m_arraySize > 1; }
+
+  /**
+   * Gets the array size. Returns 1 if non-array.
+   *
+   * @return number of elements
+   */
+  size_t GetArraySize() const { return m_arraySize; }
+
+  /**
+   * Returns whether the field has enumerated values.
+   *
+   * @return true if there are enumerated values
+   */
+  bool HasEnum() const { return !m_enum.empty(); }
+
+  /**
+   * Gets the enumerated values.
+   *
+   * @return set of enumerated values
+   */
+  const EnumValues& GetEnumValues() { return m_enum; }
+
+  /**
+   * Gets the struct descriptor for a struct data type.
+   *
+   * @return struct descriptor; returns null for non-struct
+   */
+  const StructDescriptor* GetStruct() const { return m_struct; }
+
+  /**
+   * Gets the minimum unsigned integer value that can be stored in this field.
+   *
+   * @return minimum value
+   */
+  uint64_t GetUintMin() const { return 0; }
+
+  /**
+   * Gets the maximum unsigned integer value that can be stored in this field.
+   *
+   * @return maximum value
+   */
+  uint64_t GetUintMax() const { return m_bitMask; }
+
+  /**
+   * Gets the minimum signed integer value that can be stored in this field.
+   *
+   * @return minimum value
+   */
+  int64_t GetIntMin() const {
+    return static_cast<int64_t>(-(m_bitMask >> 1)) - 1;
+  }
+
+  /**
+   * Gets the maximum signed integer value that can be stored in this field.
+   *
+   * @return maximum value
+   */
+  int64_t GetIntMax() const { return m_bitMask >> 1; }
+
+  /**
+   * Returns whether the field is a bitfield.
+   *
+   * @return true if bitfield
+   */
+  bool IsBitField() const {
+    return m_bitShift != 0 || m_bitWidth != (m_size * 8);
+  }
+
+ private:
+  // note: constructor fills in everything except offset and shift
+  const StructDescriptor* m_parent;
+  std::string m_name;
+  size_t m_size;
+  size_t m_offset = 0;
+  size_t m_arraySize;  // 1 for non-arrays
+  EnumValues m_enum;
+  const StructDescriptor* m_struct;  // nullptr for non-structs
+  uint64_t m_bitMask;
+  StructFieldType m_type;
+  unsigned int m_bitWidth;
+  unsigned int m_bitShift = 0;
+};
+
+/**
+ * Raw struct dynamic struct descriptor.
+ */
+class StructDescriptor {
+  struct private_init {};
+  friend class StructDescriptorDatabase;
+
+ public:
+  StructDescriptor(std::string_view name, const private_init&) : m_name{name} {}
+
+  /**
+   * Gets the struct name.
+   *
+   * @return name
+   */
+  const std::string& GetName() const { return m_name; }
+
+  /**
+   * Gets the struct schema.
+   *
+   * @return schema
+   */
+  const std::string& GetSchema() const { return m_schema; }
+
+  /**
+   * Returns whether the struct is valid (e.g. the struct is fully defined and
+   * field offsets computed).
+   *
+   * @return true if valid
+   */
+  bool IsValid() const { return m_valid; }
+
+  /**
+   * Returns the struct size, in bytes. Not valid unless IsValid() is true.
+   *
+   * @return size in bytes
+   */
+  size_t GetSize() const {
+    assert(m_valid);
+    return m_size;
+  }
+
+  /**
+   * Gets a field descriptor by name. Note the field cannot be accessed until
+   * the struct is valid.
+   *
+   * @param name field name
+   * @return field descriptor, or nullptr if not found
+   */
+  const StructFieldDescriptor* FindFieldByName(std::string_view name) const;
+
+  /**
+   * Gets all field descriptors. Note fields cannot be accessed until the struct
+   * is valid.
+   *
+   * @return field descriptors
+   */
+  const std::vector<StructFieldDescriptor>& GetFields() const {
+    return m_fields;
+  }
+
+ private:
+  bool CheckCircular(
+      wpi::SmallVectorImpl<const StructDescriptor*>& stack) const;
+  std::string CalculateOffsets(
+      wpi::SmallVectorImpl<const StructDescriptor*>& stack);
+
+  std::string m_name;
+  std::string m_schema;
+  std::vector<StructDescriptor*> m_references;
+  std::vector<StructFieldDescriptor> m_fields;
+  StringMap<size_t> m_fieldsByName;
+  size_t m_size = 0;
+  bool m_valid = false;
+};
+
+/**
+ * Database of raw struct dynamic descriptors.
+ */
+class StructDescriptorDatabase {
+ public:
+  /**
+   * Adds a structure schema to the database. If the struct references other
+   * structs that have not yet been added, it will not be valid until those
+   * structs are also added.
+   *
+   * @param[in] name structure name
+   * @param[in] schema structure schema
+   * @param[out] err detailed error, if nullptr is returned
+   * @return Added struct, or nullptr on error
+   */
+  const StructDescriptor* Add(std::string_view name, std::string_view schema,
+                              std::string* err);
+
+  /**
+   * Finds a structure in the database by name.
+   *
+   * @param name structure name
+   * @return struct descriptor, or nullptr if not found
+   */
+  const StructDescriptor* Find(std::string_view name) const;
+
+ private:
+  StringMap<std::unique_ptr<StructDescriptor>> m_structs;
+};
+
+/**
+ * Dynamic (run-time) read-only access to a serialized raw struct.
+ */
+class DynamicStruct {
+ public:
+  /**
+   * Constructs a new dynamic struct. Note: the passed data is a span; no copy
+   * is made, so it's necessary for the lifetime of the referenced data to be
+   * longer than this object.
+   *
+   * @param desc struct descriptor
+   * @param data serialized data
+   */
+  DynamicStruct(const StructDescriptor* desc, std::span<const uint8_t> data)
+      : m_desc{desc}, m_data{data} {}
+
+  /**
+   * Gets the struct descriptor.
+   *
+   * @return struct descriptor
+   */
+  const StructDescriptor* GetDescriptor() const { return m_desc; }
+
+  /**
+   * Gets the serialized data.
+   *
+   * @return data
+   */
+  std::span<const uint8_t> GetData() const { return m_data; }
+
+  /**
+   * Gets a struct field descriptor by name.
+   *
+   * @param name field name
+   * @return field descriptor, or nullptr if no field with that name exists
+   */
+  const StructFieldDescriptor* FindField(std::string_view name) const {
+    return m_desc->FindFieldByName(name);
+  }
+
+  /**
+   * Gets the value of a boolean field.
+   *
+   * @param field field descriptor
+   * @param arrIndex array index (must be less than field array size)
+   * @return field value
+   */
+  bool GetBoolField(const StructFieldDescriptor* field,
+                    size_t arrIndex = 0) const {
+    assert(field->m_type == StructFieldType::kBool);
+    return GetFieldImpl(field, arrIndex);
+  }
+
+  /**
+   * Gets the value of a signed integer field.
+   *
+   * @param field field descriptor
+   * @param arrIndex array index (must be less than field array size)
+   * @return field value
+   */
+  int64_t GetIntField(const StructFieldDescriptor* field,
+                      size_t arrIndex = 0) const {
+    assert(field->IsInt());
+    return GetFieldImpl(field, arrIndex);
+  }
+
+  /**
+   * Gets the value of an unsigned integer field.
+   *
+   * @param field field descriptor
+   * @param arrIndex array index (must be less than field array size)
+   * @return field value
+   */
+  uint64_t GetUintField(const StructFieldDescriptor* field,
+                        size_t arrIndex = 0) const {
+    assert(field->IsUint());
+    return GetFieldImpl(field, arrIndex);
+  }
+
+  /**
+   * Gets the value of a float field.
+   *
+   * @param field field descriptor
+   * @param arrIndex array index (must be less than field array size)
+   * @return field value
+   */
+  float GetFloatField(const StructFieldDescriptor* field,
+                      size_t arrIndex = 0) const {
+    assert(field->m_type == StructFieldType::kFloat);
+    return bit_cast<float>(
+        static_cast<uint32_t>(GetFieldImpl(field, arrIndex)));
+  }
+
+  /**
+   * Gets the value of a double field.
+   *
+   * @param field field descriptor
+   * @param arrIndex array index (must be less than field array size)
+   * @return field value
+   */
+  double GetDoubleField(const StructFieldDescriptor* field,
+                        size_t arrIndex = 0) const {
+    assert(field->m_type == StructFieldType::kDouble);
+    return bit_cast<double>(GetFieldImpl(field, arrIndex));
+  }
+
+  /**
+   * Gets the value of a char or char array field.
+   *
+   * @param field field descriptor
+   * @return field value
+   */
+  std::string_view GetStringField(const StructFieldDescriptor* field) const {
+    assert(field->m_type == StructFieldType::kChar);
+    assert(field->m_parent == m_desc);
+    assert(m_desc->IsValid());
+    return {reinterpret_cast<const char*>(&m_data[field->m_offset]),
+            field->m_arraySize};
+  }
+
+  /**
+   * Gets the value of a struct field.
+   *
+   * @param field field descriptor
+   * @param arrIndex array index (must be less than field array size)
+   * @return field value
+   */
+  DynamicStruct GetStructField(const StructFieldDescriptor* field,
+                               size_t arrIndex = 0) const {
+    assert(field->m_type == StructFieldType::kStruct);
+    assert(field->m_parent == m_desc);
+    assert(m_desc->IsValid());
+    assert(arrIndex < field->m_arraySize);
+    return DynamicStruct{field->m_struct,
+                         m_data.subspan(field->m_offset +
+                                        arrIndex * field->m_struct->GetSize())};
+  }
+
+ protected:
+  const StructDescriptor* m_desc;
+
+ private:
+  uint64_t GetFieldImpl(const StructFieldDescriptor* field,
+                        size_t arrIndex) const;
+
+  std::span<const uint8_t> m_data;
+};
+
+/**
+ * Dynamic (run-time) mutable access to a serialized raw struct.
+ */
+class MutableDynamicStruct : public DynamicStruct {
+ public:
+  /**
+   * Constructs a new dynamic struct. Note: the passed data is a span; no copy
+   * is made, so it's necessary for the lifetime of the referenced data to be
+   * longer than this object.
+   *
+   * @param desc struct descriptor
+   * @param data serialized data
+   */
+  MutableDynamicStruct(const StructDescriptor* desc, std::span<uint8_t> data)
+      : DynamicStruct{desc, data}, m_data{data} {}
+
+  /**
+   * Gets the serialized data.
+   *
+   * @return data
+   */
+  std::span<uint8_t> GetData() { return m_data; }
+
+  using DynamicStruct::GetData;
+
+  /**
+   * Overwrites the entire serialized struct by copying data from a span.
+   *
+   * @param data replacement data for the struct
+   */
+  void SetData(std::span<const uint8_t> data);
+
+  /**
+   * Sets the value of a boolean field.
+   *
+   * @param field field descriptor
+   * @param value field value
+   * @param arrIndex array index (must be less than field array size)
+   */
+  void SetBoolField(const StructFieldDescriptor* field, bool value,
+                    size_t arrIndex = 0) {
+    assert(field->m_type == StructFieldType::kBool);
+    SetFieldImpl(field, value ? 1 : 0, arrIndex);
+  }
+
+  /**
+   * Sets the value of a signed integer field.
+   *
+   * @param field field descriptor
+   * @param value field value
+   * @param arrIndex array index (must be less than field array size)
+   */
+  void SetIntField(const StructFieldDescriptor* field, int64_t value,
+                   size_t arrIndex = 0) {
+    assert(field->IsInt());
+    SetFieldImpl(field, value, arrIndex);
+  }
+
+  /**
+   * Sets the value of an unsigned integer field.
+   *
+   * @param field field descriptor
+   * @param value field value
+   * @param arrIndex array index (must be less than field array size)
+   */
+  void SetUintField(const StructFieldDescriptor* field, uint64_t value,
+                    size_t arrIndex = 0) {
+    assert(field->IsUint());
+    SetFieldImpl(field, value, arrIndex);
+  }
+
+  /**
+   * Sets the value of a float field.
+   *
+   * @param field field descriptor
+   * @param value field value
+   * @param arrIndex array index (must be less than field array size)
+   */
+  void SetFloatField(const StructFieldDescriptor* field, float value,
+                     size_t arrIndex = 0) {
+    assert(field->m_type == StructFieldType::kFloat);
+    SetFieldImpl(field, bit_cast<uint32_t>(value), arrIndex);
+  }
+
+  /**
+   * Sets the value of a double field.
+   *
+   * @param field field descriptor
+   * @param value field value
+   * @param arrIndex array index (must be less than field array size)
+   */
+  void SetDoubleField(const StructFieldDescriptor* field, double value,
+                      size_t arrIndex = 0) {
+    assert(field->m_type == StructFieldType::kDouble);
+    SetFieldImpl(field, bit_cast<uint64_t>(value), arrIndex);
+  }
+
+  /**
+   * Sets the value of a char or char array field.
+   *
+   * @param field field descriptor
+   * @param value field value
+   */
+  void SetStringField(const StructFieldDescriptor* field,
+                      std::string_view value);
+
+  /**
+   * Sets the value of a struct field.
+   *
+   * @param field field descriptor
+   * @param value field value
+   * @param arrIndex array index (must be less than field array size)
+   */
+  void SetStructField(const StructFieldDescriptor* field,
+                      const DynamicStruct& value, size_t arrIndex = 0);
+
+  /**
+   * Gets the value of a struct field.
+   *
+   * @param field field descriptor
+   * @param arrIndex array index (must be less than field array size)
+   * @return field value
+   */
+  MutableDynamicStruct GetStructField(const StructFieldDescriptor* field,
+                                      size_t arrIndex = 0) {
+    assert(field->m_type == StructFieldType::kStruct);
+    assert(field->m_parent == m_desc);
+    assert(m_desc->IsValid());
+    assert(arrIndex < field->m_arraySize);
+    return MutableDynamicStruct{
+        field->m_struct, m_data.subspan(field->m_offset +
+                                        arrIndex * field->m_struct->GetSize())};
+  }
+
+  using DynamicStruct::GetStructField;
+
+ private:
+  void SetFieldImpl(const StructFieldDescriptor* field, uint64_t value,
+                    size_t arrIndex);
+
+  std::span<uint8_t> m_data;
+};
+
+namespace impl {
+struct DSOData {
+  explicit DSOData(size_t size) : m_dataStore(size) {}
+  explicit DSOData(std::span<const uint8_t> data)
+      : m_dataStore{data.begin(), data.end()} {}
+
+  std::vector<uint8_t> m_dataStore;
+};
+}  // namespace impl
+
+/**
+ * Dynamic (run-time) mutable access to a serialized raw struct, with internal
+ * data storage.
+ */
+class DynamicStructObject : private impl::DSOData, public MutableDynamicStruct {
+  /**
+   * Constructs a new dynamic struct object. The descriptor must be valid.
+   *
+   * @param desc struct descriptor
+   */
+  explicit DynamicStructObject(const StructDescriptor* desc)
+      : DSOData{desc->GetSize()}, MutableDynamicStruct{desc, m_dataStore} {}
+
+  /**
+   * Constructs a new dynamic struct object. Makes a copy of the serialized
+   * data so there are no lifetime constraints on the data parameter.
+   *
+   * @param desc struct descriptor
+   * @param data serialized data
+   */
+  DynamicStructObject(const StructDescriptor* desc,
+                      std::span<const uint8_t> data)
+      : DSOData{data}, MutableDynamicStruct{desc, m_dataStore} {
+    assert(data.size() >= desc->GetSize());
+  }
+
+  // can't be movable due to span references
+  DynamicStructObject(DynamicStructObject&&) = delete;
+  DynamicStructObject& operator=(DynamicStructObject&&) = delete;
+};
+
+}  // namespace wpi

wpiutil/src/main/native/include/wpi/struct/SchemaParser.h

@@ -0,0 +1,186 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#pragma once
+
+#include <stdint.h>
+
+#include <string>
+#include <string_view>
+#include <utility>
+#include <vector>
+
+namespace wpi::structparser {
+
+/**
+ * A lexed raw struct schema token.
+ */
+struct Token {
+  enum Kind {
+    kUnknown,
+    kInteger,
+    kIdentifier,
+    kLeftBracket,
+    kRightBracket,
+    kLeftBrace,
+    kRightBrace,
+    kColon,
+    kSemicolon,
+    kComma,
+    kEquals,
+    kEndOfInput,
+  };
+
+  Token() = default;
+  Token(Kind kind, std::string_view text) : kind{kind}, text{text} {}
+
+  bool Is(Kind k) const { return kind == k; }
+
+  Kind kind = kUnknown;
+  std::string_view text;
+};
+
+std::string_view ToString(Token::Kind kind);
+
+/**
+ * Raw struct schema lexer.
+ */
+class Lexer {
+ public:
+  /**
+   * Construct a raw struct schema lexer.
+   *
+   * @param in schema
+   */
+  explicit Lexer(std::string_view in) : m_in{in} {}
+
+  /**
+   * Gets the next token.
+   *
+   * @return Token
+   */
+  [[nodiscard]]
+  Token Scan();
+
+  /**
+   * Gets the starting position of the last lexed token.
+   *
+   * @return position (0 = first character)
+   */
+  size_t GetPosition() const { return m_tokenStart; }
+
+ private:
+  Token ScanInteger();
+  Token ScanIdentifier();
+
+  void Get() {
+    if (m_pos < m_in.size()) {
+      [[likely]] m_current = m_in[m_pos];
+    } else {
+      m_current = -1;
+    }
+    ++m_pos;
+  }
+
+  void Unget() {
+    if (m_pos > 0) {
+      [[likely]] m_pos--;
+      if (m_pos < m_in.size()) {
+        [[likely]] m_current = m_in[m_pos];
+      } else {
+        m_current = -1;
+      }
+    } else {
+      m_current = -1;
+    }
+  }
+
+  Token MakeToken(Token::Kind kind) {
+    return {kind, m_in.substr(m_tokenStart, m_pos - m_tokenStart)};
+  }
+
+  std::string_view m_in;
+  int m_current = -1;
+  size_t m_tokenStart = 0;
+  size_t m_pos = 0;
+};
+
+/**
+ * Raw struct set of enumerated values.
+ */
+using EnumValues = std::vector<std::pair<std::string, int64_t>>;
+
+/**
+ * Raw struct schema declaration.
+ */
+struct ParsedDeclaration {
+  std::string typeString;
+  std::string name;
+  EnumValues enumValues;
+  size_t arraySize = 1;
+  unsigned int bitWidth = 0;
+};
+
+/**
+ * Raw struct schema.
+ */
+struct ParsedSchema {
+  std::vector<ParsedDeclaration> declarations;
+};
+
+/**
+ * Raw struct schema parser.
+ */
+class Parser {
+ public:
+  /**
+   * Construct a raw struct schema parser.
+   *
+   * @param in schema
+   */
+  explicit Parser(std::string_view in) : m_lexer{in} {}
+
+  /**
+   * Parses the schema.
+   *
+   * @param[out] out parsed schema object
+   * @return true on success, false on failure (use GetError() to get error)
+   */
+  [[nodiscard]]
+  bool Parse(ParsedSchema* out);
+
+  /**
+   * Gets the parser error if one occurred.
+   *
+   * @return parser error; blank if no error occurred
+   */
+  const std::string& GetError() const { return m_error; }
+
+ private:
+  [[nodiscard]]
+  bool ParseDeclaration(ParsedDeclaration* out);
+  [[nodiscard]]
+  bool ParseEnum(EnumValues* out);
+
+  Token::Kind GetNextToken() {
+    m_token = m_lexer.Scan();
+    return m_token.kind;
+  }
+  [[nodiscard]]
+  bool Expect(Token::Kind kind) {
+    if (m_token.Is(kind)) {
+      [[likely]] return true;
+    }
+    FailExpect(kind);
+    return false;
+  }
+  void FailExpect(Token::Kind desired);
+  void Fail(std::string_view msg);
+
+  Lexer m_lexer;
+  Token m_token;
+  std::string m_error;
+};
+
+}  // namespace wpi::structparser

wpiutil/src/main/native/include/wpi/struct/Struct.h

@@ -0,0 +1,533 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#pragma once
+
+#include <stdint.h>
+
+#include <concepts>
+#include <span>
+#include <string>
+#include <string_view>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+#include "wpi/Endian.h"
+#include "wpi/MathExtras.h"
+#include "wpi/bit.h"
+#include "wpi/ct_string.h"
+#include "wpi/function_ref.h"
+#include "wpi/mutex.h"
+
+namespace wpi {
+
+/**
+ * Struct serialization template. Unspecialized class has no members; only
+ * specializations of this class are useful, and only if they meet the
+ * StructSerializable concept.
+ *
+ * @tparam T type to serialize/deserialize
+ */
+template <typename T>
+struct Struct {};
+
+/**
+ * Specifies that a type is capable of raw struct serialization and
+ * deserialization.
+ *
+ * This is designed for serializing small fixed-size data structures in the
+ * fastest and most compact means possible. Serialization consists of writing
+ * values into a mutable std::span and deserialization consists of reading
+ * values from an immutable std::span.
+ *
+ * Implementations must define a template specialization for wpi::Struct with
+ * T being the type that is being serialized/deserialized, with the following
+ * static members (as enforced by this concept):
+ * - std::string_view kTypeString: the type string
+ * - size_t kSize: the structure size in bytes
+ * - std::string_view kSchema: the struct schema
+ * - T Unpack(std::span<const uint8_t, kSize>): function for deserialization
+ * - void Pack(std::span<uint8_t, kSize>, T&& value): function for
+ *   serialization
+ *
+ * If the struct has nested structs, implementations should also meet the
+ * requirements of HasNestedStruct<T>.
+ */
+template <typename T>
+concept StructSerializable =
+    requires(std::span<const uint8_t> in, std::span<uint8_t> out, T&& value) {
+      typename Struct<typename std::remove_cvref_t<T>>;
+      {
+        Struct<typename std::remove_cvref_t<T>>::kTypeString
+      } -> std::convertible_to<std::string_view>;
+      {
+        Struct<typename std::remove_cvref_t<T>>::kSize
+      } -> std::convertible_to<size_t>;
+      {
+        Struct<typename std::remove_cvref_t<T>>::kSchema
+      } -> std::convertible_to<std::string_view>;
+      {
+        Struct<typename std::remove_cvref_t<T>>::Unpack(
+            in.subspan<0, Struct<typename std::remove_cvref_t<T>>::kSize>())
+      } -> std::same_as<typename std::remove_cvref_t<T>>;
+      Struct<typename std::remove_cvref_t<T>>::Pack(
+          out.subspan<0, Struct<typename std::remove_cvref_t<T>>::kSize>(),
+          std::forward<T>(value));
+    };
+
+/**
+ * Specifies that a type is capable of in-place raw struct deserialization.
+ *
+ * In addition to meeting StructSerializable, implementations must define a
+ * wpi::Struct<T> static member `void UnpackInto(T*, std::span<const uint8_t>)`
+ * to update the pointed-to T with the contents of the span.
+ */
+template <typename T>
+concept MutableStructSerializable =
+    StructSerializable<T> && requires(T* out, std::span<const uint8_t> in) {
+      Struct<typename std::remove_cvref_t<T>>::UnpackInto(
+          out, in.subspan<0, Struct<typename std::remove_cvref_t<T>>::kSize>());
+    };
+
+/**
+ * Specifies that a struct type has nested struct declarations.
+ *
+ * In addition to meeting StructSerializable, implementations must define a
+ * wpi::Struct<T> static member
+ * `void ForEachNested(std::invocable<std::string_view, std::string_view) auto
+ * fn)` (or equivalent) and call ForEachNestedStruct<Type> on each nested struct
+ * type.
+ */
+template <typename T>
+concept HasNestedStruct =
+    StructSerializable<T> &&
+    requires(function_ref<void(std::string_view, std::string_view)> fn) {
+      Struct<typename std::remove_cvref_t<T>>::ForEachNested(fn);
+    };
+
+/**
+ * Unpack a serialized struct.
+ *
+ * @tparam T object type
+ * @param data raw struct data
+ * @return Deserialized object
+ */
+template <StructSerializable T>
+inline T UnpackStruct(std::span<const uint8_t, Struct<T>::kSize> data) {
+  return Struct<T>::Unpack(data);
+}
+
+/**
+ * Unpack a serialized struct starting at a given offset within the data.
+ * This is primarily useful in unpack implementations to unpack nested structs.
+ *
+ * @tparam T object type
+ * @tparam Offset starting offset
+ * @param data raw struct data
+ * @return Deserialized object
+ */
+template <StructSerializable T, size_t Offset>
+inline T UnpackStruct(std::span<const uint8_t> data) {
+  return Struct<T>::Unpack(data.template subspan<Offset, Struct<T>::kSize>());
+}
+
+/**
+ * Pack a serialized struct.
+ *
+ * @param data struct storage (mutable, output)
+ * @param value object
+ */
+template <StructSerializable T>
+inline void PackStruct(
+    std::span<uint8_t, Struct<typename std::remove_cvref_t<T>>::kSize> data,
+    T&& value) {
+  Struct<typename std::remove_cvref_t<T>>::Pack(data, std::forward<T>(value));
+}
+
+/**
+ * Pack a serialized struct starting at a given offset within the data. This is
+ * primarily useful in pack implementations to pack nested structs.
+ *
+ * @tparam Offset starting offset
+ * @param data struct storage (mutable, output)
+ * @param value object
+ */
+template <size_t Offset, StructSerializable T>
+inline void PackStruct(std::span<uint8_t> data, T&& value) {
+  Struct<typename std::remove_cvref_t<T>>::Pack(
+      data.template subspan<Offset,
+                            Struct<typename std::remove_cvref_t<T>>::kSize>(),
+      std::forward<T>(value));
+}
+
+/**
+ * Unpack a serialized struct into an existing object, overwriting its contents.
+ *
+ * @param out object (output)
+ * @param data raw struct data
+ */
+template <StructSerializable T>
+inline void UnpackStructInto(T* out,
+                             std::span<const uint8_t, Struct<T>::kSize> data) {
+  if constexpr (MutableStructSerializable<T>) {
+    Struct<T>::UnpackInto(out, data);
+  } else {
+    *out = UnpackStruct<T>(data);
+  }
+}
+
+/**
+ * Unpack a serialized struct into an existing object, overwriting its contents,
+ * and starting at a given offset within the data.
+ * This is primarily useful in unpack implementations to unpack nested structs.
+ *
+ * @tparam Offset starting offset
+ * @param out object (output)
+ * @param data raw struct data
+ */
+template <size_t Offset, StructSerializable T>
+inline void UnpackStructInto(T* out, std::span<const uint8_t> data) {
+  if constexpr (MutableStructSerializable<T>) {
+    Struct<T>::UnpackInto(out,
+                          data.template subspan<Offset, Struct<T>::kSize>());
+  } else {
+    *out = UnpackStruct<T, Offset>(data);
+  }
+}
+
+/**
+ * Get the type string for a raw struct serializable type
+ *
+ * @tparam T serializable type
+ * @return type string
+ */
+template <StructSerializable T>
+constexpr auto GetStructTypeString() {
+  return Struct<T>::kTypeString;
+}
+
+template <StructSerializable T, size_t N>
+consteval auto MakeStructArrayTypeString() {
+  using namespace literals;
+  if constexpr (N == std::dynamic_extent) {
+    return Concat(
+        ct_string<char, std::char_traits<char>, Struct<T>::kTypeString.size()>{
+            Struct<T>::kTypeString},
+        "[]"_ct_string);
+  } else {
+    return Concat(
+        ct_string<char, std::char_traits<char>, Struct<T>::kTypeString.size()>{
+            Struct<T>::kTypeString},
+        "["_ct_string, NumToCtString<N>(), "]"_ct_string);
+  }
+}
+
+template <StructSerializable T, size_t N>
+consteval auto MakeStructArraySchema() {
+  using namespace literals;
+  if constexpr (N == std::dynamic_extent) {
+    return Concat(
+        ct_string<char, std::char_traits<char>, Struct<T>::kSchema.size()>{
+            Struct<T>::kSchema},
+        "[]"_ct_string);
+  } else {
+    return Concat(
+        ct_string<char, std::char_traits<char>, Struct<T>::kSchema.size()>{
+            Struct<T>::kSchema},
+        "["_ct_string, NumToCtString<N>(), "]"_ct_string);
+  }
+}
+
+template <StructSerializable T>
+constexpr std::string_view GetStructSchema() {
+  return Struct<T>::kSchema;
+}
+
+template <StructSerializable T>
+constexpr std::span<const uint8_t> GetStructSchemaBytes() {
+  return {reinterpret_cast<const uint8_t*>(Struct<T>::kSchema.data()),
+          Struct<T>::kSchema.size()};
+}
+
+template <StructSerializable T>
+void ForEachStructSchema(
+    std::invocable<std::string_view, std::string_view> auto fn) {
+  if constexpr (HasNestedStruct<T>) {
+    Struct<T>::ForEachNested(fn);
+  }
+  fn(Struct<T>::kTypeString, Struct<T>::kSchema);
+}
+
+template <StructSerializable T>
+class StructArrayBuffer {
+  using S = Struct<T>;
+
+ public:
+  StructArrayBuffer() = default;
+  StructArrayBuffer(const StructArrayBuffer&) = delete;
+  StructArrayBuffer& operator=(const StructArrayBuffer&) = delete;
+  StructArrayBuffer(StructArrayBuffer&& rhs) : m_buf{std::move(rhs.m_buf)} {}
+  StructArrayBuffer& operator=(StructArrayBuffer&& rhs) {
+    m_buf = std::move(rhs.m_buf);
+    return *this;
+  }
+
+  template <typename U, typename F>
+    requires
+#if __cpp_lib_ranges >= 201911L
+      std::ranges::range<U> &&
+      std::convertible_to<std::ranges::range_value_t<U>, T> &&
+#endif
+      std::invocable<F, std::span<const uint8_t>>
+    void Write(U&& data, F&& func) {
+    if ((std::size(data) * S::kSize) < 256) {
+      // use the stack
+      uint8_t buf[256];
+      auto out = buf;
+      for (auto&& val : data) {
+        S::Pack(std::span<uint8_t, S::kSize>{out, out + S::kSize},
+                std::forward<decltype(val)>(val));
+        out += S::kSize;
+      }
+      func(std::span<uint8_t>{buf, out});
+    } else {
+      std::scoped_lock lock{m_mutex};
+      m_buf.resize(std::size(data) * S::kSize);
+      auto out = m_buf.begin();
+      for (auto&& val : data) {
+        S::Pack(std::span<uint8_t, S::kSize>{out, out + S::kSize},
+                std::forward<decltype(val)>(val));
+        out += S::kSize;
+      }
+      func(m_buf);
+    }
+  }
+
+ private:
+  wpi::mutex m_mutex;
+  std::vector<uint8_t> m_buf;
+};
+
+/**
+ * Raw struct support for fixed-size arrays of other structs.
+ */
+template <StructSerializable T, size_t N>
+struct Struct<std::array<T, N>> {
+  static constexpr auto kTypeString = MakeStructArrayTypeString<T, N>();
+  static constexpr size_t kSize = N * Struct<T>::kSize;
+  static constexpr auto kSchema = MakeStructArraySchema<T, N>();
+  static std::array<T, N> Unpack(std::span<const uint8_t, kSize> data) {
+    std::array<T, N> result;
+    for (size_t i = 0; i < N; ++i) {
+      result[i] = UnpackStruct<T, 0>(data);
+      data = data.subspan(Struct<T>::kSize);
+    }
+    return result;
+  }
+  static void Pack(std::span<uint8_t, kSize> data,
+                   std::span<const T, N> values) {
+    std::span<uint8_t> unsizedData = data;
+    for (auto&& val : values) {
+      PackStruct<0>(unsizedData, val);
+      unsizedData = unsizedData.subspan(Struct<T>::kSize);
+    }
+  }
+  static void UnpackInto(std::array<T, N>* out,
+                         std::span<const uint8_t, kSize> data) {
+    UnpackInto(std::span{*out}, data);
+  }
+  // alternate span-based function
+  static void UnpackInto(std::span<T, N> out,
+                         std::span<const uint8_t, kSize> data) {
+    std::span<const uint8_t> unsizedData = data;
+    for (size_t i = 0; i < N; ++i) {
+      UnpackStructInto<0, T>(&out[i], unsizedData);
+      unsizedData = unsizedData.subspan(Struct<T>::kSize);
+    }
+  }
+};
+
+/**
+ * Raw struct support for boolean values.
+ * Primarily useful for higher level struct implementations.
+ */
+template <>
+struct Struct<bool> {
+  static constexpr std::string_view kTypeString = "struct:bool";
+  static constexpr size_t kSize = 1;
+  static constexpr std::string_view kSchema = "bool value";
+  static bool Unpack(std::span<const uint8_t, 1> data) { return data[0]; }
+  static void Pack(std::span<uint8_t, 1> data, bool value) {
+    data[0] = static_cast<char>(value ? 1 : 0);
+  }
+};
+
+/**
+ * Raw struct support for uint8_t values.
+ * Primarily useful for higher level struct implementations.
+ */
+template <>
+struct Struct<uint8_t> {
+  static constexpr std::string_view kTypeString = "struct:uint8";
+  static constexpr size_t kSize = 1;
+  static constexpr std::string_view kSchema = "uint8 value";
+  static uint8_t Unpack(std::span<const uint8_t, 1> data) { return data[0]; }
+  static void Pack(std::span<uint8_t, 1> data, uint8_t value) {
+    data[0] = value;
+  }
+};
+
+/**
+ * Raw struct support for int8_t values.
+ * Primarily useful for higher level struct implementations.
+ */
+template <>
+struct Struct<int8_t> {
+  static constexpr std::string_view kTypeString = "struct:int8";
+  static constexpr size_t kSize = 1;
+  static constexpr std::string_view kSchema = "int8 value";
+  static int8_t Unpack(std::span<const uint8_t, 1> data) { return data[0]; }
+  static void Pack(std::span<uint8_t, 1> data, int8_t value) {
+    data[0] = value;
+  }
+};
+
+/**
+ * Raw struct support for uint16_t values.
+ * Primarily useful for higher level struct implementations.
+ */
+template <>
+struct Struct<uint16_t> {
+  static constexpr std::string_view kTypeString = "struct:uint16";
+  static constexpr size_t kSize = 2;
+  static constexpr std::string_view kSchema = "uint16 value";
+  static uint16_t Unpack(std::span<const uint8_t, 2> data) {
+    return support::endian::read16le(data.data());
+  }
+  static void Pack(std::span<uint8_t, 2> data, uint16_t value) {
+    support::endian::write16le(data.data(), value);
+  }
+};
+
+/**
+ * Raw struct support for int16_t values.
+ * Primarily useful for higher level struct implementations.
+ */
+template <>
+struct Struct<int16_t> {
+  static constexpr std::string_view kTypeString = "struct:int16";
+  static constexpr size_t kSize = 2;
+  static constexpr std::string_view kSchema = "int16 value";
+  static int16_t Unpack(std::span<const uint8_t, 2> data) {
+    return support::endian::read16le(data.data());
+  }
+  static void Pack(std::span<uint8_t, 2> data, int16_t value) {
+    support::endian::write16le(data.data(), value);
+  }
+};
+
+/**
+ * Raw struct support for uint32_t values.
+ * Primarily useful for higher level struct implementations.
+ */
+template <>
+struct Struct<uint32_t> {
+  static constexpr std::string_view kTypeString = "struct:uint32";
+  static constexpr size_t kSize = 4;
+  static constexpr std::string_view kSchema = "uint32 value";
+  static uint32_t Unpack(std::span<const uint8_t, 4> data) {
+    return support::endian::read32le(data.data());
+  }
+  static void Pack(std::span<uint8_t, 4> data, uint32_t value) {
+    support::endian::write32le(data.data(), value);
+  }
+};
+
+/**
+ * Raw struct support for int32_t values.
+ * Primarily useful for higher level struct implementations.
+ */
+template <>
+struct Struct<int32_t> {
+  static constexpr std::string_view kTypeString = "struct:int32";
+  static constexpr size_t kSize = 4;
+  static constexpr std::string_view kSchema = "int32 value";
+  static int32_t Unpack(std::span<const uint8_t, 4> data) {
+    return support::endian::read32le(data.data());
+  }
+  static void Pack(std::span<uint8_t, 4> data, int32_t value) {
+    support::endian::write32le(data.data(), value);
+  }
+};
+
+/**
+ * Raw struct support for uint64_t values.
+ * Primarily useful for higher level struct implementations.
+ */
+template <>
+struct Struct<uint64_t> {
+  static constexpr std::string_view kTypeString = "struct:uint64";
+  static constexpr size_t kSize = 8;
+  static constexpr std::string_view kSchema = "uint64 value";
+  static uint64_t Unpack(std::span<const uint8_t, 8> data) {
+    return support::endian::read64le(data.data());
+  }
+  static void Pack(std::span<uint8_t, 8> data, uint64_t value) {
+    support::endian::write64le(data.data(), value);
+  }
+};
+
+/**
+ * Raw struct support for int64_t values.
+ * Primarily useful for higher level struct implementations.
+ */
+template <>
+struct Struct<int64_t> {
+  static constexpr std::string_view kTypeString = "struct:int64";
+  static constexpr size_t kSize = 8;
+  static constexpr std::string_view kSchema = "int64 value";
+  static int64_t Unpack(std::span<const uint8_t, 8> data) {
+    return support::endian::read64le(data.data());
+  }
+  static void Pack(std::span<uint8_t, 8> data, int64_t value) {
+    support::endian::write64le(data.data(), value);
+  }
+};
+
+/**
+ * Raw struct support for float values.
+ * Primarily useful for higher level struct implementations.
+ */
+template <>
+struct Struct<float> {
+  static constexpr std::string_view kTypeString = "struct:float";
+  static constexpr size_t kSize = 4;
+  static constexpr std::string_view kSchema = "float value";
+  static float Unpack(std::span<const uint8_t, 4> data) {
+    return bit_cast<float>(support::endian::read32le(data.data()));
+  }
+  static void Pack(std::span<uint8_t, 4> data, float value) {
+    support::endian::write32le(data.data(), bit_cast<uint32_t>(value));
+  }
+};
+
+/**
+ * Raw struct support for double values.
+ * Primarily useful for higher level struct implementations.
+ */
+template <>
+struct Struct<double> {
+  static constexpr std::string_view kTypeString = "struct:double";
+  static constexpr size_t kSize = 8;
+  static constexpr std::string_view kSchema = "double value";
+  static double Unpack(std::span<const uint8_t, 8> data) {
+    return bit_cast<double>(support::endian::read64le(data.data()));
+  }
+  static void Pack(std::span<uint8_t, 8> data, double value) {
+    support::endian::write64le(data.data(), bit_cast<uint64_t>(value));
+  }
+};
+
+}  // namespace wpi