mirror of
https://github.com/wpilibsuite/allwpilib
synced 2026-07-02 02:51:42 +00:00
[wpiutil, ntcore] Add structured data support (#5391)
This adds support for two serialization formats for complex data types: - Protobuf for complex objects with variable length internals that need forward and backward wire compatibility (lower speed, more flexible) - Raw struct (ByteBuffer-style) for fixed-length objects (higher speed, less flexible) Deserialization can be done either by creating a new object (for immutable objects) or overwriting the contents of an existing object (for mutable objects). Implementing classes should provide inner classes that implement the Protobuf or Struct interface (in Java) or specialize the wpi::Protobuf or wpi::Struct struct (in C++). It is possible for classes to implement both. If the class itself does not implement serialization, it's possible for third parties/users to provide an implementation instead. Uses the Google protobuf implementation for C++ and the QuickBuffers alternative protobuf implementation for Java.
This commit is contained in:
Peter Johnson
260
wpiutil/src/main/native/include/wpi/protobuf/Protobuf.h
Normal file
260
wpiutil/src/main/native/include/wpi/protobuf/Protobuf.h
Normal file
@@ -0,0 +1,260 @@
|
||||
// Copyright (c) FIRST and other WPILib contributors.
|
||||
// Open Source Software; you can modify and/or share it under the terms of
|
||||
// the WPILib BSD license file in the root directory of this project.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include <concepts>
|
||||
#include <optional>
|
||||
#include <span>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "wpi/function_ref.h"
|
||||
|
||||
namespace google::protobuf {
|
||||
class Arena;
|
||||
class Message;
|
||||
} // namespace google::protobuf
|
||||
|
||||
namespace wpi {
|
||||
|
||||
template <typename T>
|
||||
class SmallVectorImpl;
|
||||
|
||||
/**
|
||||
* Protobuf serialization template. Unspecialized class has no members; only
|
||||
* specializations of this class are useful, and only if they meet the
|
||||
* ProtobufSerializable concept.
|
||||
*
|
||||
* @tparam T type to serialize/deserialize
|
||||
*/
|
||||
template <typename T>
|
||||
struct Protobuf {};
|
||||
|
||||
/**
|
||||
* Specifies that a type is capable of protobuf serialization and
|
||||
* deserialization.
|
||||
*
|
||||
* This is designed for serializing complex flexible data structures using
|
||||
* code generated from a .proto file. Serialization consists of writing
|
||||
* values into a mutable protobuf Message and deserialization consists of
|
||||
* reading values from an immutable protobuf Message.
|
||||
*
|
||||
* Implementations must define a template specialization for wpi::Protobuf with
|
||||
* T being the type that is being serialized/deserialized, with the following
|
||||
* static members (as enforced by this concept):
|
||||
* - google::protobuf::Message* New(google::protobuf::Arena*): create a protobuf
|
||||
* message
|
||||
* - T Unpack(const google::protobuf::Message&): function for deserialization
|
||||
* - void Pack(google::protobuf::Message*, T&& value): function for
|
||||
* serialization
|
||||
*
|
||||
* To avoid pulling in the protobuf headers, these functions use
|
||||
* google::protobuf::Message instead of a more specific type; implementations
|
||||
* will need to static_cast to the correct type as created by New().
|
||||
*
|
||||
* Additionally: In a static block, call StructRegistry.registerClass() to
|
||||
* register the class
|
||||
*/
|
||||
template <typename T>
|
||||
concept ProtobufSerializable = requires(
|
||||
google::protobuf::Arena* arena, const google::protobuf::Message& inmsg,
|
||||
google::protobuf::Message* outmsg, const T& value) {
|
||||
typename Protobuf<typename std::remove_cvref_t<T>>;
|
||||
{
|
||||
Protobuf<typename std::remove_cvref_t<T>>::New(arena)
|
||||
} -> std::same_as<google::protobuf::Message*>;
|
||||
{
|
||||
Protobuf<typename std::remove_cvref_t<T>>::Unpack(inmsg)
|
||||
} -> std::same_as<typename std::remove_cvref_t<T>>;
|
||||
Protobuf<typename std::remove_cvref_t<T>>::Pack(outmsg, value);
|
||||
};
|
||||
|
||||
/**
|
||||
* Specifies that a type is capable of in-place protobuf deserialization.
|
||||
*
|
||||
* In addition to meeting ProtobufSerializable, implementations must define a
|
||||
* wpi::Protobuf<T> static member
|
||||
* `void UnpackInto(T*, const google::protobuf::Message&)` to update the
|
||||
* pointed-to T with the contents of the message.
|
||||
*/
|
||||
template <typename T>
|
||||
concept MutableProtobufSerializable =
|
||||
ProtobufSerializable<T> &&
|
||||
requires(T* out, const google::protobuf::Message& msg) {
|
||||
Protobuf<typename std::remove_cvref_t<T>>::UnpackInto(out, msg);
|
||||
};
|
||||
|
||||
/**
|
||||
* Unpack a serialized protobuf message.
|
||||
*
|
||||
* @tparam T object type
|
||||
* @param msg protobuf message
|
||||
* @return Deserialized object
|
||||
*/
|
||||
template <ProtobufSerializable T>
|
||||
inline T UnpackProtobuf(const google::protobuf::Message& msg) {
|
||||
return Protobuf<T>::Unpack(msg);
|
||||
}
|
||||
|
||||
/**
|
||||
* Pack a serialized protobuf message.
|
||||
*
|
||||
* @param msg protobuf message (mutable, output)
|
||||
* @param value object
|
||||
*/
|
||||
template <ProtobufSerializable T>
|
||||
inline void PackProtobuf(google::protobuf::Message* msg, const T& value) {
|
||||
Protobuf<typename std::remove_cvref_t<T>>::Pack(msg, value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Unpack a serialized struct into an existing object, overwriting its contents.
|
||||
*
|
||||
* @param out object (output)
|
||||
* @param msg protobuf message
|
||||
*/
|
||||
template <ProtobufSerializable T>
|
||||
inline void UnpackProtobufInto(T* out, const google::protobuf::Message& msg) {
|
||||
if constexpr (MutableProtobufSerializable<T>) {
|
||||
Protobuf<T>::UnpackInto(out, msg);
|
||||
} else {
|
||||
*out = UnpackProtobuf<T>(msg);
|
||||
}
|
||||
}
|
||||
|
||||
// these detail functions avoid the need to include protobuf headers
|
||||
namespace detail {
|
||||
void DeleteProtobuf(google::protobuf::Message* msg);
|
||||
bool ParseProtobuf(google::protobuf::Message* msg,
|
||||
std::span<const uint8_t> data);
|
||||
bool SerializeProtobuf(wpi::SmallVectorImpl<uint8_t>& out,
|
||||
const google::protobuf::Message& msg);
|
||||
bool SerializeProtobuf(std::vector<uint8_t>& out,
|
||||
const google::protobuf::Message& msg);
|
||||
std::string GetTypeString(const google::protobuf::Message& msg);
|
||||
void ForEachProtobufDescriptor(
|
||||
const google::protobuf::Message& msg,
|
||||
function_ref<bool(std::string_view filename)> wants,
|
||||
function_ref<void(std::string_view filename,
|
||||
std::span<const uint8_t> descriptor)>
|
||||
fn);
|
||||
} // namespace detail
|
||||
|
||||
/**
|
||||
* Owning wrapper (ala std::unique_ptr) for google::protobuf::Message* that does
|
||||
* not require the protobuf headers be included. Note this object is not thread
|
||||
* safe; users of this object are required to provide any necessary thread
|
||||
* safety.
|
||||
*
|
||||
* @tparam T serialized object type
|
||||
*/
|
||||
template <ProtobufSerializable T>
|
||||
class ProtobufMessage {
|
||||
public:
|
||||
explicit ProtobufMessage(google::protobuf::Arena* arena = nullptr)
|
||||
: m_msg{Protobuf<T>::New(arena)} {}
|
||||
~ProtobufMessage() { detail::DeleteProtobuf(m_msg); }
|
||||
ProtobufMessage(const ProtobufMessage&) = delete;
|
||||
ProtobufMessage& operator=(const ProtobufMessage&) = delete;
|
||||
ProtobufMessage(ProtobufMessage&& rhs) : m_msg{rhs.m_msg} {
|
||||
rhs.m_msg = nullptr;
|
||||
}
|
||||
ProtobufMessage& operator=(ProtobufMessage&& rhs) {
|
||||
std::swap(m_msg, rhs.m_msg);
|
||||
return *this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the stored message object.
|
||||
*
|
||||
* @return google::protobuf::Message*
|
||||
*/
|
||||
google::protobuf::Message* GetMessage() { return m_msg; }
|
||||
const google::protobuf::Message* GetMessage() const { return m_msg; }
|
||||
|
||||
/**
|
||||
* Unpacks from a byte array.
|
||||
*
|
||||
* @param data byte array
|
||||
* @return Optional; empty if parsing failed
|
||||
*/
|
||||
std::optional<T> Unpack(std::span<const uint8_t> data) {
|
||||
if (!detail::ParseProtobuf(m_msg, data)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
return Protobuf<T>::Unpack(*m_msg);
|
||||
}
|
||||
|
||||
/**
|
||||
* Unpacks from a byte array into an existing object.
|
||||
*
|
||||
* @param[out] out output object
|
||||
* @param[in] data byte array
|
||||
* @return true if successful
|
||||
*/
|
||||
bool UnpackInto(T* out, std::span<const uint8_t> data) {
|
||||
if (!detail::ParseProtobuf(m_msg, data)) {
|
||||
return false;
|
||||
}
|
||||
UnpackProtobufInto(out, *m_msg);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Packs object into a SmallVector.
|
||||
*
|
||||
* @param[out] out output bytes
|
||||
* @param[in] value value
|
||||
* @return true if successful
|
||||
*/
|
||||
bool Pack(wpi::SmallVectorImpl<uint8_t>& out, const T& value) {
|
||||
Protobuf<T>::Pack(m_msg, value);
|
||||
return detail::SerializeProtobuf(out, *m_msg);
|
||||
}
|
||||
|
||||
/**
|
||||
* Packs object into a std::vector.
|
||||
*
|
||||
* @param[out] out output bytes
|
||||
* @param[in] value value
|
||||
* @return true if successful
|
||||
*/
|
||||
bool Pack(std::vector<uint8_t>& out, const T& value) {
|
||||
Protobuf<T>::Pack(m_msg, value);
|
||||
return detail::SerializeProtobuf(out, *m_msg);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the type string for the message.
|
||||
*
|
||||
* @return type string
|
||||
*/
|
||||
std::string GetTypeString() const { return detail::GetTypeString(*m_msg); }
|
||||
|
||||
/**
|
||||
* Loops over all protobuf descriptors including nested/referenced
|
||||
* descriptors.
|
||||
*
|
||||
* @param exists function that returns false if fn should be called for the
|
||||
* given type string
|
||||
* @param fn function to call for each descriptor
|
||||
*/
|
||||
void ForEachProtobufDescriptor(
|
||||
function_ref<bool(std::string_view filename)> exists,
|
||||
function_ref<void(std::string_view filename,
|
||||
std::span<const uint8_t> descriptor)>
|
||||
fn) {
|
||||
detail::ForEachProtobufDescriptor(*m_msg, exists, fn);
|
||||
}
|
||||
|
||||
private:
|
||||
google::protobuf::Message* m_msg = nullptr;
|
||||
};
|
||||
|
||||
} // namespace wpi
|
||||
@@ -0,0 +1,76 @@
|
||||
// Copyright (c) FIRST and other WPILib contributors.
|
||||
// Open Source Software; you can modify and/or share it under the terms of
|
||||
// the WPILib BSD license file in the root directory of this project.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include <span>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
#include <google/protobuf/descriptor.pb.h>
|
||||
#include <google/protobuf/descriptor_database.h>
|
||||
#include <google/protobuf/dynamic_message.h>
|
||||
|
||||
#include "wpi/StringMap.h"
|
||||
|
||||
namespace wpi {
|
||||
|
||||
/**
|
||||
* Database of protobuf dynamic messages. Unlike the protobuf descriptor pools
|
||||
* and databases, this gracefully handles adding descriptors out of order and
|
||||
* descriptors being replaced.
|
||||
*/
|
||||
class ProtobufMessageDatabase {
|
||||
public:
|
||||
/**
|
||||
* Adds a file descriptor to the database. If the file references other
|
||||
* files that have not yet been added, no messages in that file will be
|
||||
* available until those files are added. Replaces any existing file with
|
||||
* the same name.
|
||||
*
|
||||
* @param filename filename
|
||||
* @param data FileDescriptorProto serialized data
|
||||
*/
|
||||
void Add(std::string_view filename, std::span<const uint8_t> data);
|
||||
|
||||
/**
|
||||
* Finds a message in the database by name.
|
||||
*
|
||||
* @param name type name
|
||||
* @return protobuf message, or nullptr if not found
|
||||
*/
|
||||
google::protobuf::Message* Find(std::string_view name) const;
|
||||
|
||||
/**
|
||||
* Gets message factory.
|
||||
*
|
||||
* @return message factory
|
||||
*/
|
||||
google::protobuf::MessageFactory* GetMessageFactory() {
|
||||
return m_factory.get();
|
||||
}
|
||||
|
||||
private:
|
||||
struct ProtoFile {
|
||||
std::unique_ptr<google::protobuf::FileDescriptorProto> proto;
|
||||
std::vector<std::string> uses;
|
||||
bool complete = false;
|
||||
bool inPool = false;
|
||||
};
|
||||
|
||||
void Build(std::string_view filename, ProtoFile& file);
|
||||
bool Rebuild(ProtoFile& file);
|
||||
|
||||
std::unique_ptr<google::protobuf::DescriptorPool> m_pool =
|
||||
std::make_unique<google::protobuf::DescriptorPool>();
|
||||
std::unique_ptr<google::protobuf::DynamicMessageFactory> m_factory =
|
||||
std::make_unique<google::protobuf::DynamicMessageFactory>();
|
||||
wpi::StringMap<ProtoFile> m_files; // indexed by filename
|
||||
// indexed by type string
|
||||
mutable wpi::StringMap<std::unique_ptr<google::protobuf::Message>> m_msgs;
|
||||
};
|
||||
|
||||
} // namespace wpi
|
||||
Reference in New Issue
Block a user