[wpiutil, ntcore] Add structured data support (#5391)

This adds support for two serialization formats for complex data types:

- Protobuf for complex objects with variable length internals that need forward and backward wire compatibility (lower speed, more flexible)
- Raw struct (ByteBuffer-style) for fixed-length objects (higher speed, less flexible)

Deserialization can be done either by creating a new object (for immutable objects) or overwriting the contents of an existing object (for mutable objects).

Implementing classes should provide inner classes that implement the Protobuf or Struct interface (in Java) or specialize the wpi::Protobuf or wpi::Struct struct (in C++). It is possible for classes to implement both. If the class itself does not implement serialization, it's possible for third parties/users to provide an implementation instead.

Uses the Google protobuf implementation for C++ and the QuickBuffers alternative protobuf implementation for Java.
This commit is contained in:
Peter Johnson
2023-10-19 21:41:47 -07:00
committed by GitHub
parent ecb7cfa9ef
commit cf54d9ccb7
133 changed files with 13506 additions and 90 deletions

View File

@@ -0,0 +1,682 @@
// Copyright (c) FIRST and other WPILib contributors.
// Open Source Software; you can modify and/or share it under the terms of
// the WPILib BSD license file in the root directory of this project.
#pragma once
#include <stdint.h>
#include <cassert>
#include <memory>
#include <span>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
#include "wpi/MathExtras.h"
#include "wpi/StringMap.h"
#include "wpi/bit.h"
namespace wpi {
template <typename T>
class SmallVectorImpl;
class DynamicStruct;
class MutableDynamicStruct;
class StructDescriptor;
class StructDescriptorDatabase;
/**
* Known data types for raw struct dynamic fields (see StructFieldDescriptor).
*/
enum class StructFieldType {
kBool,
kChar,
kInt8,
kInt16,
kInt32,
kInt64,
kUint8,
kUint16,
kUint32,
kUint64,
kFloat,
kDouble,
kStruct
};
/**
* Raw struct dynamic field descriptor.
*/
class StructFieldDescriptor {
struct private_init {};
friend class DynamicStruct;
friend class MutableDynamicStruct;
friend class StructDescriptor;
friend class StructDescriptorDatabase;
public:
/**
* Set of enumerated values.
*/
using EnumValues = std::vector<std::pair<std::string, int64_t>>;
StructFieldDescriptor(const StructDescriptor* parent, std::string_view name,
StructFieldType type, size_t size, size_t arraySize,
unsigned int bitWidth, EnumValues enumValues,
const StructDescriptor* structDesc,
const private_init&);
/**
* Gets the dynamic struct this field is contained in.
*
* @return struct descriptor
*/
const StructDescriptor* GetParent() const { return m_parent; }
/**
* Gets the field name.
*
* @return field name
*/
const std::string& GetName() const { return m_name; }
/**
* Gets the field type.
*
* @return field type
*/
StructFieldType GetType() const { return m_type; }
/**
* Returns whether the field type is a signed integer.
*
* @return true if signed integer, false otherwise
*/
bool IsInt() const {
return m_type == StructFieldType::kInt8 ||
m_type == StructFieldType::kInt16 ||
m_type == StructFieldType::kInt32 ||
m_type == StructFieldType::kInt64;
}
/**
* Returns whether the field type is an unsigned integer.
*
* @return true if unsigned integer, false otherwise
*/
bool IsUint() const {
return m_type == StructFieldType::kUint8 ||
m_type == StructFieldType::kUint16 ||
m_type == StructFieldType::kUint32 ||
m_type == StructFieldType::kUint64;
}
/**
* Gets the underlying storage size of the field, in bytes.
*
* @return number of bytes
*/
size_t GetSize() const { return m_size; }
/**
* Gets the storage offset of the field, in bytes.
*
* @return number of bytes from the start of the struct
*/
size_t GetOffset() const { return m_offset; }
/**
* Gets the bit width of the field, in bits.
*
* @return number of bits
*/
unsigned int GetBitWidth() const {
return m_bitWidth == 0 ? m_size * 8 : m_bitWidth;
}
/**
* Gets the bit mask for the field. The mask is always the least significant
* bits (it is not shifted).
*
* @return bit mask
*/
uint64_t GetBitMask() const { return m_bitMask; }
/**
* Gets the bit shift for the field (LSB=0).
*
* @return number of bits
*/
unsigned int GetBitShift() const { return m_bitShift; }
/**
* Returns whether the field is an array.
*
* @return true if array
*/
bool IsArray() const { return m_arraySize > 1; }
/**
* Gets the array size. Returns 1 if non-array.
*
* @return number of elements
*/
size_t GetArraySize() const { return m_arraySize; }
/**
* Returns whether the field has enumerated values.
*
* @return true if there are enumerated values
*/
bool HasEnum() const { return !m_enum.empty(); }
/**
* Gets the enumerated values.
*
* @return set of enumerated values
*/
const EnumValues& GetEnumValues() { return m_enum; }
/**
* Gets the struct descriptor for a struct data type.
*
* @return struct descriptor; returns null for non-struct
*/
const StructDescriptor* GetStruct() const { return m_struct; }
/**
* Gets the minimum unsigned integer value that can be stored in this field.
*
* @return minimum value
*/
uint64_t GetUintMin() const { return 0; }
/**
* Gets the maximum unsigned integer value that can be stored in this field.
*
* @return maximum value
*/
uint64_t GetUintMax() const { return m_bitMask; }
/**
* Gets the minimum signed integer value that can be stored in this field.
*
* @return minimum value
*/
int64_t GetIntMin() const {
return static_cast<int64_t>(-(m_bitMask >> 1)) - 1;
}
/**
* Gets the maximum signed integer value that can be stored in this field.
*
* @return maximum value
*/
int64_t GetIntMax() const { return m_bitMask >> 1; }
/**
* Returns whether the field is a bitfield.
*
* @return true if bitfield
*/
bool IsBitField() const {
return m_bitShift != 0 || m_bitWidth != (m_size * 8);
}
private:
// note: constructor fills in everything except offset and shift
const StructDescriptor* m_parent;
std::string m_name;
size_t m_size;
size_t m_offset = 0;
size_t m_arraySize; // 1 for non-arrays
EnumValues m_enum;
const StructDescriptor* m_struct; // nullptr for non-structs
uint64_t m_bitMask;
StructFieldType m_type;
unsigned int m_bitWidth;
unsigned int m_bitShift = 0;
};
/**
* Raw struct dynamic struct descriptor.
*/
class StructDescriptor {
struct private_init {};
friend class StructDescriptorDatabase;
public:
StructDescriptor(std::string_view name, const private_init&) : m_name{name} {}
/**
* Gets the struct name.
*
* @return name
*/
const std::string& GetName() const { return m_name; }
/**
* Gets the struct schema.
*
* @return schema
*/
const std::string& GetSchema() const { return m_schema; }
/**
* Returns whether the struct is valid (e.g. the struct is fully defined and
* field offsets computed).
*
* @return true if valid
*/
bool IsValid() const { return m_valid; }
/**
* Returns the struct size, in bytes. Not valid unless IsValid() is true.
*
* @return size in bytes
*/
size_t GetSize() const {
assert(m_valid);
return m_size;
}
/**
* Gets a field descriptor by name. Note the field cannot be accessed until
* the struct is valid.
*
* @param name field name
* @return field descriptor, or nullptr if not found
*/
const StructFieldDescriptor* FindFieldByName(std::string_view name) const;
/**
* Gets all field descriptors. Note fields cannot be accessed until the struct
* is valid.
*
* @return field descriptors
*/
const std::vector<StructFieldDescriptor>& GetFields() const {
return m_fields;
}
private:
bool CheckCircular(
wpi::SmallVectorImpl<const StructDescriptor*>& stack) const;
std::string CalculateOffsets(
wpi::SmallVectorImpl<const StructDescriptor*>& stack);
std::string m_name;
std::string m_schema;
std::vector<StructDescriptor*> m_references;
std::vector<StructFieldDescriptor> m_fields;
StringMap<size_t> m_fieldsByName;
size_t m_size = 0;
bool m_valid = false;
};
/**
* Database of raw struct dynamic descriptors.
*/
class StructDescriptorDatabase {
public:
/**
* Adds a structure schema to the database. If the struct references other
* structs that have not yet been added, it will not be valid until those
* structs are also added.
*
* @param[in] name structure name
* @param[in] schema structure schema
* @param[out] err detailed error, if nullptr is returned
* @return Added struct, or nullptr on error
*/
const StructDescriptor* Add(std::string_view name, std::string_view schema,
std::string* err);
/**
* Finds a structure in the database by name.
*
* @param name structure name
* @return struct descriptor, or nullptr if not found
*/
const StructDescriptor* Find(std::string_view name) const;
private:
StringMap<std::unique_ptr<StructDescriptor>> m_structs;
};
/**
* Dynamic (run-time) read-only access to a serialized raw struct.
*/
class DynamicStruct {
public:
/**
* Constructs a new dynamic struct. Note: the passed data is a span; no copy
* is made, so it's necessary for the lifetime of the referenced data to be
* longer than this object.
*
* @param desc struct descriptor
* @param data serialized data
*/
DynamicStruct(const StructDescriptor* desc, std::span<const uint8_t> data)
: m_desc{desc}, m_data{data} {}
/**
* Gets the struct descriptor.
*
* @return struct descriptor
*/
const StructDescriptor* GetDescriptor() const { return m_desc; }
/**
* Gets the serialized data.
*
* @return data
*/
std::span<const uint8_t> GetData() const { return m_data; }
/**
* Gets a struct field descriptor by name.
*
* @param name field name
* @return field descriptor, or nullptr if no field with that name exists
*/
const StructFieldDescriptor* FindField(std::string_view name) const {
return m_desc->FindFieldByName(name);
}
/**
* Gets the value of a boolean field.
*
* @param field field descriptor
* @param arrIndex array index (must be less than field array size)
* @return field value
*/
bool GetBoolField(const StructFieldDescriptor* field,
size_t arrIndex = 0) const {
assert(field->m_type == StructFieldType::kBool);
return GetFieldImpl(field, arrIndex);
}
/**
* Gets the value of a signed integer field.
*
* @param field field descriptor
* @param arrIndex array index (must be less than field array size)
* @return field value
*/
int64_t GetIntField(const StructFieldDescriptor* field,
size_t arrIndex = 0) const {
assert(field->IsInt());
return GetFieldImpl(field, arrIndex);
}
/**
* Gets the value of an unsigned integer field.
*
* @param field field descriptor
* @param arrIndex array index (must be less than field array size)
* @return field value
*/
uint64_t GetUintField(const StructFieldDescriptor* field,
size_t arrIndex = 0) const {
assert(field->IsUint());
return GetFieldImpl(field, arrIndex);
}
/**
* Gets the value of a float field.
*
* @param field field descriptor
* @param arrIndex array index (must be less than field array size)
* @return field value
*/
float GetFloatField(const StructFieldDescriptor* field,
size_t arrIndex = 0) const {
assert(field->m_type == StructFieldType::kFloat);
return bit_cast<float>(
static_cast<uint32_t>(GetFieldImpl(field, arrIndex)));
}
/**
* Gets the value of a double field.
*
* @param field field descriptor
* @param arrIndex array index (must be less than field array size)
* @return field value
*/
double GetDoubleField(const StructFieldDescriptor* field,
size_t arrIndex = 0) const {
assert(field->m_type == StructFieldType::kDouble);
return bit_cast<double>(GetFieldImpl(field, arrIndex));
}
/**
* Gets the value of a char or char array field.
*
* @param field field descriptor
* @return field value
*/
std::string_view GetStringField(const StructFieldDescriptor* field) const {
assert(field->m_type == StructFieldType::kChar);
assert(field->m_parent == m_desc);
assert(m_desc->IsValid());
return {reinterpret_cast<const char*>(&m_data[field->m_offset]),
field->m_arraySize};
}
/**
* Gets the value of a struct field.
*
* @param field field descriptor
* @param arrIndex array index (must be less than field array size)
* @return field value
*/
DynamicStruct GetStructField(const StructFieldDescriptor* field,
size_t arrIndex = 0) const {
assert(field->m_type == StructFieldType::kStruct);
assert(field->m_parent == m_desc);
assert(m_desc->IsValid());
assert(arrIndex < field->m_arraySize);
return DynamicStruct{field->m_struct,
m_data.subspan(field->m_offset +
arrIndex * field->m_struct->GetSize())};
}
protected:
const StructDescriptor* m_desc;
private:
uint64_t GetFieldImpl(const StructFieldDescriptor* field,
size_t arrIndex) const;
std::span<const uint8_t> m_data;
};
/**
* Dynamic (run-time) mutable access to a serialized raw struct.
*/
class MutableDynamicStruct : public DynamicStruct {
public:
/**
* Constructs a new dynamic struct. Note: the passed data is a span; no copy
* is made, so it's necessary for the lifetime of the referenced data to be
* longer than this object.
*
* @param desc struct descriptor
* @param data serialized data
*/
MutableDynamicStruct(const StructDescriptor* desc, std::span<uint8_t> data)
: DynamicStruct{desc, data}, m_data{data} {}
/**
* Gets the serialized data.
*
* @return data
*/
std::span<uint8_t> GetData() { return m_data; }
using DynamicStruct::GetData;
/**
* Overwrites the entire serialized struct by copying data from a span.
*
* @param data replacement data for the struct
*/
void SetData(std::span<const uint8_t> data);
/**
* Sets the value of a boolean field.
*
* @param field field descriptor
* @param value field value
* @param arrIndex array index (must be less than field array size)
*/
void SetBoolField(const StructFieldDescriptor* field, bool value,
size_t arrIndex = 0) {
assert(field->m_type == StructFieldType::kBool);
SetFieldImpl(field, value ? 1 : 0, arrIndex);
}
/**
* Sets the value of a signed integer field.
*
* @param field field descriptor
* @param value field value
* @param arrIndex array index (must be less than field array size)
*/
void SetIntField(const StructFieldDescriptor* field, int64_t value,
size_t arrIndex = 0) {
assert(field->IsInt());
SetFieldImpl(field, value, arrIndex);
}
/**
* Sets the value of an unsigned integer field.
*
* @param field field descriptor
* @param value field value
* @param arrIndex array index (must be less than field array size)
*/
void SetUintField(const StructFieldDescriptor* field, uint64_t value,
size_t arrIndex = 0) {
assert(field->IsUint());
SetFieldImpl(field, value, arrIndex);
}
/**
* Sets the value of a float field.
*
* @param field field descriptor
* @param value field value
* @param arrIndex array index (must be less than field array size)
*/
void SetFloatField(const StructFieldDescriptor* field, float value,
size_t arrIndex = 0) {
assert(field->m_type == StructFieldType::kFloat);
SetFieldImpl(field, bit_cast<uint32_t>(value), arrIndex);
}
/**
* Sets the value of a double field.
*
* @param field field descriptor
* @param value field value
* @param arrIndex array index (must be less than field array size)
*/
void SetDoubleField(const StructFieldDescriptor* field, double value,
size_t arrIndex = 0) {
assert(field->m_type == StructFieldType::kDouble);
SetFieldImpl(field, bit_cast<uint64_t>(value), arrIndex);
}
/**
* Sets the value of a char or char array field.
*
* @param field field descriptor
* @param value field value
*/
void SetStringField(const StructFieldDescriptor* field,
std::string_view value);
/**
* Sets the value of a struct field.
*
* @param field field descriptor
* @param value field value
* @param arrIndex array index (must be less than field array size)
*/
void SetStructField(const StructFieldDescriptor* field,
const DynamicStruct& value, size_t arrIndex = 0);
/**
* Gets the value of a struct field.
*
* @param field field descriptor
* @param arrIndex array index (must be less than field array size)
* @return field value
*/
MutableDynamicStruct GetStructField(const StructFieldDescriptor* field,
size_t arrIndex = 0) {
assert(field->m_type == StructFieldType::kStruct);
assert(field->m_parent == m_desc);
assert(m_desc->IsValid());
assert(arrIndex < field->m_arraySize);
return MutableDynamicStruct{
field->m_struct, m_data.subspan(field->m_offset +
arrIndex * field->m_struct->GetSize())};
}
using DynamicStruct::GetStructField;
private:
void SetFieldImpl(const StructFieldDescriptor* field, uint64_t value,
size_t arrIndex);
std::span<uint8_t> m_data;
};
namespace impl {
struct DSOData {
explicit DSOData(size_t size) : m_dataStore(size) {}
explicit DSOData(std::span<const uint8_t> data)
: m_dataStore{data.begin(), data.end()} {}
std::vector<uint8_t> m_dataStore;
};
} // namespace impl
/**
* Dynamic (run-time) mutable access to a serialized raw struct, with internal
* data storage.
*/
class DynamicStructObject : private impl::DSOData, public MutableDynamicStruct {
/**
* Constructs a new dynamic struct object. The descriptor must be valid.
*
* @param desc struct descriptor
*/
explicit DynamicStructObject(const StructDescriptor* desc)
: DSOData{desc->GetSize()}, MutableDynamicStruct{desc, m_dataStore} {}
/**
* Constructs a new dynamic struct object. Makes a copy of the serialized
* data so there are no lifetime constraints on the data parameter.
*
* @param desc struct descriptor
* @param data serialized data
*/
DynamicStructObject(const StructDescriptor* desc,
std::span<const uint8_t> data)
: DSOData{data}, MutableDynamicStruct{desc, m_dataStore} {
assert(data.size() >= desc->GetSize());
}
// can't be movable due to span references
DynamicStructObject(DynamicStructObject&&) = delete;
DynamicStructObject& operator=(DynamicStructObject&&) = delete;
};
} // namespace wpi

View File

@@ -0,0 +1,186 @@
// Copyright (c) FIRST and other WPILib contributors.
// Open Source Software; you can modify and/or share it under the terms of
// the WPILib BSD license file in the root directory of this project.
#pragma once
#include <stdint.h>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
namespace wpi::structparser {
/**
* A lexed raw struct schema token.
*/
struct Token {
enum Kind {
kUnknown,
kInteger,
kIdentifier,
kLeftBracket,
kRightBracket,
kLeftBrace,
kRightBrace,
kColon,
kSemicolon,
kComma,
kEquals,
kEndOfInput,
};
Token() = default;
Token(Kind kind, std::string_view text) : kind{kind}, text{text} {}
bool Is(Kind k) const { return kind == k; }
Kind kind = kUnknown;
std::string_view text;
};
std::string_view ToString(Token::Kind kind);
/**
* Raw struct schema lexer.
*/
class Lexer {
public:
/**
* Construct a raw struct schema lexer.
*
* @param in schema
*/
explicit Lexer(std::string_view in) : m_in{in} {}
/**
* Gets the next token.
*
* @return Token
*/
[[nodiscard]]
Token Scan();
/**
* Gets the starting position of the last lexed token.
*
* @return position (0 = first character)
*/
size_t GetPosition() const { return m_tokenStart; }
private:
Token ScanInteger();
Token ScanIdentifier();
void Get() {
if (m_pos < m_in.size()) {
[[likely]] m_current = m_in[m_pos];
} else {
m_current = -1;
}
++m_pos;
}
void Unget() {
if (m_pos > 0) {
[[likely]] m_pos--;
if (m_pos < m_in.size()) {
[[likely]] m_current = m_in[m_pos];
} else {
m_current = -1;
}
} else {
m_current = -1;
}
}
Token MakeToken(Token::Kind kind) {
return {kind, m_in.substr(m_tokenStart, m_pos - m_tokenStart)};
}
std::string_view m_in;
int m_current = -1;
size_t m_tokenStart = 0;
size_t m_pos = 0;
};
/**
* Raw struct set of enumerated values.
*/
using EnumValues = std::vector<std::pair<std::string, int64_t>>;
/**
* Raw struct schema declaration.
*/
struct ParsedDeclaration {
std::string typeString;
std::string name;
EnumValues enumValues;
size_t arraySize = 1;
unsigned int bitWidth = 0;
};
/**
* Raw struct schema.
*/
struct ParsedSchema {
std::vector<ParsedDeclaration> declarations;
};
/**
* Raw struct schema parser.
*/
class Parser {
public:
/**
* Construct a raw struct schema parser.
*
* @param in schema
*/
explicit Parser(std::string_view in) : m_lexer{in} {}
/**
* Parses the schema.
*
* @param[out] out parsed schema object
* @return true on success, false on failure (use GetError() to get error)
*/
[[nodiscard]]
bool Parse(ParsedSchema* out);
/**
* Gets the parser error if one occurred.
*
* @return parser error; blank if no error occurred
*/
const std::string& GetError() const { return m_error; }
private:
[[nodiscard]]
bool ParseDeclaration(ParsedDeclaration* out);
[[nodiscard]]
bool ParseEnum(EnumValues* out);
Token::Kind GetNextToken() {
m_token = m_lexer.Scan();
return m_token.kind;
}
[[nodiscard]]
bool Expect(Token::Kind kind) {
if (m_token.Is(kind)) {
[[likely]] return true;
}
FailExpect(kind);
return false;
}
void FailExpect(Token::Kind desired);
void Fail(std::string_view msg);
Lexer m_lexer;
Token m_token;
std::string m_error;
};
} // namespace wpi::structparser

View File

@@ -0,0 +1,533 @@
// Copyright (c) FIRST and other WPILib contributors.
// Open Source Software; you can modify and/or share it under the terms of
// the WPILib BSD license file in the root directory of this project.
#pragma once
#include <stdint.h>
#include <concepts>
#include <span>
#include <string>
#include <string_view>
#include <type_traits>
#include <utility>
#include <vector>
#include "wpi/Endian.h"
#include "wpi/MathExtras.h"
#include "wpi/bit.h"
#include "wpi/ct_string.h"
#include "wpi/function_ref.h"
#include "wpi/mutex.h"
namespace wpi {
/**
* Struct serialization template. Unspecialized class has no members; only
* specializations of this class are useful, and only if they meet the
* StructSerializable concept.
*
* @tparam T type to serialize/deserialize
*/
template <typename T>
struct Struct {};
/**
* Specifies that a type is capable of raw struct serialization and
* deserialization.
*
* This is designed for serializing small fixed-size data structures in the
* fastest and most compact means possible. Serialization consists of writing
* values into a mutable std::span and deserialization consists of reading
* values from an immutable std::span.
*
* Implementations must define a template specialization for wpi::Struct with
* T being the type that is being serialized/deserialized, with the following
* static members (as enforced by this concept):
* - std::string_view kTypeString: the type string
* - size_t kSize: the structure size in bytes
* - std::string_view kSchema: the struct schema
* - T Unpack(std::span<const uint8_t, kSize>): function for deserialization
* - void Pack(std::span<uint8_t, kSize>, T&& value): function for
* serialization
*
* If the struct has nested structs, implementations should also meet the
* requirements of HasNestedStruct<T>.
*/
template <typename T>
concept StructSerializable =
requires(std::span<const uint8_t> in, std::span<uint8_t> out, T&& value) {
typename Struct<typename std::remove_cvref_t<T>>;
{
Struct<typename std::remove_cvref_t<T>>::kTypeString
} -> std::convertible_to<std::string_view>;
{
Struct<typename std::remove_cvref_t<T>>::kSize
} -> std::convertible_to<size_t>;
{
Struct<typename std::remove_cvref_t<T>>::kSchema
} -> std::convertible_to<std::string_view>;
{
Struct<typename std::remove_cvref_t<T>>::Unpack(
in.subspan<0, Struct<typename std::remove_cvref_t<T>>::kSize>())
} -> std::same_as<typename std::remove_cvref_t<T>>;
Struct<typename std::remove_cvref_t<T>>::Pack(
out.subspan<0, Struct<typename std::remove_cvref_t<T>>::kSize>(),
std::forward<T>(value));
};
/**
* Specifies that a type is capable of in-place raw struct deserialization.
*
* In addition to meeting StructSerializable, implementations must define a
* wpi::Struct<T> static member `void UnpackInto(T*, std::span<const uint8_t>)`
* to update the pointed-to T with the contents of the span.
*/
template <typename T>
concept MutableStructSerializable =
StructSerializable<T> && requires(T* out, std::span<const uint8_t> in) {
Struct<typename std::remove_cvref_t<T>>::UnpackInto(
out, in.subspan<0, Struct<typename std::remove_cvref_t<T>>::kSize>());
};
/**
* Specifies that a struct type has nested struct declarations.
*
* In addition to meeting StructSerializable, implementations must define a
* wpi::Struct<T> static member
* `void ForEachNested(std::invocable<std::string_view, std::string_view) auto
* fn)` (or equivalent) and call ForEachNestedStruct<Type> on each nested struct
* type.
*/
template <typename T>
concept HasNestedStruct =
StructSerializable<T> &&
requires(function_ref<void(std::string_view, std::string_view)> fn) {
Struct<typename std::remove_cvref_t<T>>::ForEachNested(fn);
};
/**
* Unpack a serialized struct.
*
* @tparam T object type
* @param data raw struct data
* @return Deserialized object
*/
template <StructSerializable T>
inline T UnpackStruct(std::span<const uint8_t, Struct<T>::kSize> data) {
return Struct<T>::Unpack(data);
}
/**
* Unpack a serialized struct starting at a given offset within the data.
* This is primarily useful in unpack implementations to unpack nested structs.
*
* @tparam T object type
* @tparam Offset starting offset
* @param data raw struct data
* @return Deserialized object
*/
template <StructSerializable T, size_t Offset>
inline T UnpackStruct(std::span<const uint8_t> data) {
return Struct<T>::Unpack(data.template subspan<Offset, Struct<T>::kSize>());
}
/**
* Pack a serialized struct.
*
* @param data struct storage (mutable, output)
* @param value object
*/
template <StructSerializable T>
inline void PackStruct(
std::span<uint8_t, Struct<typename std::remove_cvref_t<T>>::kSize> data,
T&& value) {
Struct<typename std::remove_cvref_t<T>>::Pack(data, std::forward<T>(value));
}
/**
* Pack a serialized struct starting at a given offset within the data. This is
* primarily useful in pack implementations to pack nested structs.
*
* @tparam Offset starting offset
* @param data struct storage (mutable, output)
* @param value object
*/
template <size_t Offset, StructSerializable T>
inline void PackStruct(std::span<uint8_t> data, T&& value) {
Struct<typename std::remove_cvref_t<T>>::Pack(
data.template subspan<Offset,
Struct<typename std::remove_cvref_t<T>>::kSize>(),
std::forward<T>(value));
}
/**
* Unpack a serialized struct into an existing object, overwriting its contents.
*
* @param out object (output)
* @param data raw struct data
*/
template <StructSerializable T>
inline void UnpackStructInto(T* out,
std::span<const uint8_t, Struct<T>::kSize> data) {
if constexpr (MutableStructSerializable<T>) {
Struct<T>::UnpackInto(out, data);
} else {
*out = UnpackStruct<T>(data);
}
}
/**
* Unpack a serialized struct into an existing object, overwriting its contents,
* and starting at a given offset within the data.
* This is primarily useful in unpack implementations to unpack nested structs.
*
* @tparam Offset starting offset
* @param out object (output)
* @param data raw struct data
*/
template <size_t Offset, StructSerializable T>
inline void UnpackStructInto(T* out, std::span<const uint8_t> data) {
if constexpr (MutableStructSerializable<T>) {
Struct<T>::UnpackInto(out,
data.template subspan<Offset, Struct<T>::kSize>());
} else {
*out = UnpackStruct<T, Offset>(data);
}
}
/**
* Get the type string for a raw struct serializable type
*
* @tparam T serializable type
* @return type string
*/
template <StructSerializable T>
constexpr auto GetStructTypeString() {
return Struct<T>::kTypeString;
}
template <StructSerializable T, size_t N>
consteval auto MakeStructArrayTypeString() {
using namespace literals;
if constexpr (N == std::dynamic_extent) {
return Concat(
ct_string<char, std::char_traits<char>, Struct<T>::kTypeString.size()>{
Struct<T>::kTypeString},
"[]"_ct_string);
} else {
return Concat(
ct_string<char, std::char_traits<char>, Struct<T>::kTypeString.size()>{
Struct<T>::kTypeString},
"["_ct_string, NumToCtString<N>(), "]"_ct_string);
}
}
template <StructSerializable T, size_t N>
consteval auto MakeStructArraySchema() {
using namespace literals;
if constexpr (N == std::dynamic_extent) {
return Concat(
ct_string<char, std::char_traits<char>, Struct<T>::kSchema.size()>{
Struct<T>::kSchema},
"[]"_ct_string);
} else {
return Concat(
ct_string<char, std::char_traits<char>, Struct<T>::kSchema.size()>{
Struct<T>::kSchema},
"["_ct_string, NumToCtString<N>(), "]"_ct_string);
}
}
template <StructSerializable T>
constexpr std::string_view GetStructSchema() {
return Struct<T>::kSchema;
}
template <StructSerializable T>
constexpr std::span<const uint8_t> GetStructSchemaBytes() {
return {reinterpret_cast<const uint8_t*>(Struct<T>::kSchema.data()),
Struct<T>::kSchema.size()};
}
template <StructSerializable T>
void ForEachStructSchema(
std::invocable<std::string_view, std::string_view> auto fn) {
if constexpr (HasNestedStruct<T>) {
Struct<T>::ForEachNested(fn);
}
fn(Struct<T>::kTypeString, Struct<T>::kSchema);
}
template <StructSerializable T>
class StructArrayBuffer {
using S = Struct<T>;
public:
StructArrayBuffer() = default;
StructArrayBuffer(const StructArrayBuffer&) = delete;
StructArrayBuffer& operator=(const StructArrayBuffer&) = delete;
StructArrayBuffer(StructArrayBuffer&& rhs) : m_buf{std::move(rhs.m_buf)} {}
StructArrayBuffer& operator=(StructArrayBuffer&& rhs) {
m_buf = std::move(rhs.m_buf);
return *this;
}
template <typename U, typename F>
requires
#if __cpp_lib_ranges >= 201911L
std::ranges::range<U> &&
std::convertible_to<std::ranges::range_value_t<U>, T> &&
#endif
std::invocable<F, std::span<const uint8_t>>
void Write(U&& data, F&& func) {
if ((std::size(data) * S::kSize) < 256) {
// use the stack
uint8_t buf[256];
auto out = buf;
for (auto&& val : data) {
S::Pack(std::span<uint8_t, S::kSize>{out, out + S::kSize},
std::forward<decltype(val)>(val));
out += S::kSize;
}
func(std::span<uint8_t>{buf, out});
} else {
std::scoped_lock lock{m_mutex};
m_buf.resize(std::size(data) * S::kSize);
auto out = m_buf.begin();
for (auto&& val : data) {
S::Pack(std::span<uint8_t, S::kSize>{out, out + S::kSize},
std::forward<decltype(val)>(val));
out += S::kSize;
}
func(m_buf);
}
}
private:
wpi::mutex m_mutex;
std::vector<uint8_t> m_buf;
};
/**
* Raw struct support for fixed-size arrays of other structs.
*/
template <StructSerializable T, size_t N>
struct Struct<std::array<T, N>> {
static constexpr auto kTypeString = MakeStructArrayTypeString<T, N>();
static constexpr size_t kSize = N * Struct<T>::kSize;
static constexpr auto kSchema = MakeStructArraySchema<T, N>();
static std::array<T, N> Unpack(std::span<const uint8_t, kSize> data) {
std::array<T, N> result;
for (size_t i = 0; i < N; ++i) {
result[i] = UnpackStruct<T, 0>(data);
data = data.subspan(Struct<T>::kSize);
}
return result;
}
static void Pack(std::span<uint8_t, kSize> data,
std::span<const T, N> values) {
std::span<uint8_t> unsizedData = data;
for (auto&& val : values) {
PackStruct<0>(unsizedData, val);
unsizedData = unsizedData.subspan(Struct<T>::kSize);
}
}
static void UnpackInto(std::array<T, N>* out,
std::span<const uint8_t, kSize> data) {
UnpackInto(std::span{*out}, data);
}
// alternate span-based function
static void UnpackInto(std::span<T, N> out,
std::span<const uint8_t, kSize> data) {
std::span<const uint8_t> unsizedData = data;
for (size_t i = 0; i < N; ++i) {
UnpackStructInto<0, T>(&out[i], unsizedData);
unsizedData = unsizedData.subspan(Struct<T>::kSize);
}
}
};
/**
* Raw struct support for boolean values.
* Primarily useful for higher level struct implementations.
*/
template <>
struct Struct<bool> {
static constexpr std::string_view kTypeString = "struct:bool";
static constexpr size_t kSize = 1;
static constexpr std::string_view kSchema = "bool value";
static bool Unpack(std::span<const uint8_t, 1> data) { return data[0]; }
static void Pack(std::span<uint8_t, 1> data, bool value) {
data[0] = static_cast<char>(value ? 1 : 0);
}
};
/**
* Raw struct support for uint8_t values.
* Primarily useful for higher level struct implementations.
*/
template <>
struct Struct<uint8_t> {
static constexpr std::string_view kTypeString = "struct:uint8";
static constexpr size_t kSize = 1;
static constexpr std::string_view kSchema = "uint8 value";
static uint8_t Unpack(std::span<const uint8_t, 1> data) { return data[0]; }
static void Pack(std::span<uint8_t, 1> data, uint8_t value) {
data[0] = value;
}
};
/**
* Raw struct support for int8_t values.
* Primarily useful for higher level struct implementations.
*/
template <>
struct Struct<int8_t> {
static constexpr std::string_view kTypeString = "struct:int8";
static constexpr size_t kSize = 1;
static constexpr std::string_view kSchema = "int8 value";
static int8_t Unpack(std::span<const uint8_t, 1> data) { return data[0]; }
static void Pack(std::span<uint8_t, 1> data, int8_t value) {
data[0] = value;
}
};
/**
* Raw struct support for uint16_t values.
* Primarily useful for higher level struct implementations.
*/
template <>
struct Struct<uint16_t> {
static constexpr std::string_view kTypeString = "struct:uint16";
static constexpr size_t kSize = 2;
static constexpr std::string_view kSchema = "uint16 value";
static uint16_t Unpack(std::span<const uint8_t, 2> data) {
return support::endian::read16le(data.data());
}
static void Pack(std::span<uint8_t, 2> data, uint16_t value) {
support::endian::write16le(data.data(), value);
}
};
/**
* Raw struct support for int16_t values.
* Primarily useful for higher level struct implementations.
*/
template <>
struct Struct<int16_t> {
static constexpr std::string_view kTypeString = "struct:int16";
static constexpr size_t kSize = 2;
static constexpr std::string_view kSchema = "int16 value";
static int16_t Unpack(std::span<const uint8_t, 2> data) {
return support::endian::read16le(data.data());
}
static void Pack(std::span<uint8_t, 2> data, int16_t value) {
support::endian::write16le(data.data(), value);
}
};
/**
* Raw struct support for uint32_t values.
* Primarily useful for higher level struct implementations.
*/
template <>
struct Struct<uint32_t> {
static constexpr std::string_view kTypeString = "struct:uint32";
static constexpr size_t kSize = 4;
static constexpr std::string_view kSchema = "uint32 value";
static uint32_t Unpack(std::span<const uint8_t, 4> data) {
return support::endian::read32le(data.data());
}
static void Pack(std::span<uint8_t, 4> data, uint32_t value) {
support::endian::write32le(data.data(), value);
}
};
/**
* Raw struct support for int32_t values.
* Primarily useful for higher level struct implementations.
*/
template <>
struct Struct<int32_t> {
static constexpr std::string_view kTypeString = "struct:int32";
static constexpr size_t kSize = 4;
static constexpr std::string_view kSchema = "int32 value";
static int32_t Unpack(std::span<const uint8_t, 4> data) {
return support::endian::read32le(data.data());
}
static void Pack(std::span<uint8_t, 4> data, int32_t value) {
support::endian::write32le(data.data(), value);
}
};
/**
* Raw struct support for uint64_t values.
* Primarily useful for higher level struct implementations.
*/
template <>
struct Struct<uint64_t> {
static constexpr std::string_view kTypeString = "struct:uint64";
static constexpr size_t kSize = 8;
static constexpr std::string_view kSchema = "uint64 value";
static uint64_t Unpack(std::span<const uint8_t, 8> data) {
return support::endian::read64le(data.data());
}
static void Pack(std::span<uint8_t, 8> data, uint64_t value) {
support::endian::write64le(data.data(), value);
}
};
/**
* Raw struct support for int64_t values.
* Primarily useful for higher level struct implementations.
*/
template <>
struct Struct<int64_t> {
static constexpr std::string_view kTypeString = "struct:int64";
static constexpr size_t kSize = 8;
static constexpr std::string_view kSchema = "int64 value";
static int64_t Unpack(std::span<const uint8_t, 8> data) {
return support::endian::read64le(data.data());
}
static void Pack(std::span<uint8_t, 8> data, int64_t value) {
support::endian::write64le(data.data(), value);
}
};
/**
* Raw struct support for float values.
* Primarily useful for higher level struct implementations.
*/
template <>
struct Struct<float> {
static constexpr std::string_view kTypeString = "struct:float";
static constexpr size_t kSize = 4;
static constexpr std::string_view kSchema = "float value";
static float Unpack(std::span<const uint8_t, 4> data) {
return bit_cast<float>(support::endian::read32le(data.data()));
}
static void Pack(std::span<uint8_t, 4> data, float value) {
support::endian::write32le(data.data(), bit_cast<uint32_t>(value));
}
};
/**
* Raw struct support for double values.
* Primarily useful for higher level struct implementations.
*/
template <>
struct Struct<double> {
static constexpr std::string_view kTypeString = "struct:double";
static constexpr size_t kSize = 8;
static constexpr std::string_view kSchema = "double value";
static double Unpack(std::span<const uint8_t, 8> data) {
return bit_cast<double>(support::endian::read64le(data.data()));
}
static void Pack(std::span<uint8_t, 8> data, double value) {
support::endian::write64le(data.data(), bit_cast<uint64_t>(value));
}
};
} // namespace wpi