[wpiutil, ntcore] Add structured data support (#5391)

This adds support for two serialization formats for complex data types:

- Protobuf for complex objects with variable length internals that need forward and backward wire compatibility (lower speed, more flexible)
- Raw struct (ByteBuffer-style) for fixed-length objects (higher speed, less flexible)

Deserialization can be done either by creating a new object (for immutable objects) or overwriting the contents of an existing object (for mutable objects).

Implementing classes should provide inner classes that implement the Protobuf or Struct interface (in Java) or specialize the wpi::Protobuf or wpi::Struct struct (in C++). It is possible for classes to implement both. If the class itself does not implement serialization, it's possible for third parties/users to provide an implementation instead.

Uses the Google protobuf implementation for C++ and the QuickBuffers alternative protobuf implementation for Java.

ntcore/src/main/native/cpp/LocalStorage.cpp

@@ -7,6 +7,7 @@
 #include <algorithm>
 
 #include <wpi/DataLog.h>
+#include <wpi/SmallString.h>
 #include <wpi/StringExtras.h>
 #include <wpi/json.h>
 
@@ -1481,6 +1482,41 @@ void LocalStorage::StopDataLog(NT_DataLogger logger) {
   }
 }
 
+bool LocalStorage::HasSchema(std::string_view name) {
+  std::scoped_lock lock{m_mutex};
+  wpi::SmallString<128> fullName{"/.schema/"};
+  fullName += name;
+  auto it = m_impl.m_schemas.find(fullName);
+  return it != m_impl.m_schemas.end();
+}
+
+void LocalStorage::AddSchema(std::string_view name, std::string_view type,
+                             std::span<const uint8_t> schema) {
+  std::scoped_lock lock{m_mutex};
+  wpi::SmallString<128> fullName{"/.schema/"};
+  fullName += name;
+  auto& pubHandle = m_impl.m_schemas[fullName];
+  if (pubHandle != 0) {
+    return;
+  }
+
+  auto topic = m_impl.GetOrCreateTopic(fullName);
+
+  if (topic->localPublishers.size() >= kMaxPublishers) {
+    WPI_ERROR(m_impl.m_logger,
+              "reached maximum number of publishers to '{}', not publishing",
+              topic->name);
+    return;
+  }
+
+  pubHandle = m_impl
+                  .AddLocalPublisher(topic, {{"retained", true}},
+                                     PubSubConfig{NT_RAW, type, {}})
+                  ->handle;
+
+  m_impl.SetDefaultEntryValue(pubHandle, Value::MakeRaw(schema));
+}
+
 void LocalStorage::Reset() {
   std::scoped_lock lock{m_mutex};
   m_impl.m_network = nullptr;

ntcore/src/main/native/cpp/LocalStorage.h

@@ -321,6 +321,13 @@ class LocalStorage final : public net::ILocalStorage {
                              std::string_view logPrefix);
   void StopDataLog(NT_DataLogger logger);
 
+  //
+  // Schema functions
+  //
+  bool HasSchema(std::string_view name);
+  void AddSchema(std::string_view name, std::string_view type,
+                 std::span<const uint8_t> schema);
+
   void Reset();
 
  private:
@@ -549,6 +556,9 @@ class LocalStorage final : public net::ILocalStorage {
     // string-based listeners
     VectorSet<ListenerData*> m_topicPrefixListeners;
 
+    // schema publishers
+    wpi::StringMap<NT_Publisher> m_schemas;
+
     // topic functions
     void NotifyTopic(TopicData* topic, unsigned int eventFlags);
 

ntcore/src/main/native/cpp/networktables/Topic.cpp

@@ -7,9 +7,14 @@
 #include <wpi/json.h>
 
 #include "networktables/GenericEntry.h"
+#include "networktables/NetworkTableInstance.h"
 
 using namespace nt;
 
+NetworkTableInstance Topic::GetInstance() const {
+  return NetworkTableInstance{GetInstanceFromHandle(m_handle)};
+}
+
 wpi::json Topic::GetProperty(std::string_view name) const {
   return ::nt::GetTopicProperty(m_handle, name);
 }

ntcore/src/main/native/cpp/ntcore_c.cpp

@@ -625,6 +625,15 @@ NT_Listener NT_AddPolledLogger(NT_ListenerPoller poller, unsigned int min_level,
   return nt::AddPolledLogger(poller, min_level, max_level);
 }
 
+NT_Bool NT_HasSchema(NT_Inst inst, const char* name) {
+  return nt::HasSchema(inst, name);
+}
+
+void NT_AddSchema(NT_Inst inst, const char* name, const char* type,
+                  const uint8_t* schema, size_t schemaSize) {
+  nt::AddSchema(inst, name, type, {schema, schemaSize});
+}
+
 void NT_DisposeValue(NT_Value* value) {
   switch (value->type) {
     case NT_UNASSIGNED:

ntcore/src/main/native/cpp/ntcore_cpp.cpp

@@ -782,4 +782,19 @@ NT_Listener AddPolledLogger(NT_ListenerPoller poller, unsigned int minLevel,
   }
 }
 
+bool HasSchema(NT_Inst inst, std::string_view name) {
+  if (auto ii = InstanceImpl::GetTyped(inst, Handle::kInstance)) {
+    return ii->localStorage.HasSchema(name);
+  } else {
+    return false;
+  }
+}
+
+void AddSchema(NT_Inst inst, std::string_view name, std::string_view type,
+               std::span<const uint8_t> schema) {
+  if (auto ii = InstanceImpl::GetTyped(inst, Handle::kInstance)) {
+    ii->localStorage.AddSchema(name, type, schema);
+  }
+}
+
 }  // namespace nt