mirror of
https://github.com/wpilibsuite/allwpilib
synced 2026-06-20 00:51:42 +00:00
[wpiutil, ntcore] Add structured data support (#5391)
This adds support for two serialization formats for complex data types: - Protobuf for complex objects with variable length internals that need forward and backward wire compatibility (lower speed, more flexible) - Raw struct (ByteBuffer-style) for fixed-length objects (higher speed, less flexible) Deserialization can be done either by creating a new object (for immutable objects) or overwriting the contents of an existing object (for mutable objects). Implementing classes should provide inner classes that implement the Protobuf or Struct interface (in Java) or specialize the wpi::Protobuf or wpi::Struct struct (in C++). It is possible for classes to implement both. If the class itself does not implement serialization, it's possible for third parties/users to provide an implementation instead. Uses the Google protobuf implementation for C++ and the QuickBuffers alternative protobuf implementation for Java.
This commit is contained in:
Peter Johnson
@@ -31,6 +31,7 @@
|
||||
#include "wpi/Endian.h"
|
||||
#include "wpi/Logger.h"
|
||||
#include "wpi/MathExtras.h"
|
||||
#include "wpi/SmallString.h"
|
||||
#include "wpi/fs.h"
|
||||
#include "wpi/timestamp.h"
|
||||
|
||||
@@ -211,6 +212,33 @@ void DataLog::Resume() {
|
||||
m_paused = false;
|
||||
}
|
||||
|
||||
bool DataLog::HasSchema(std::string_view name) const {
|
||||
std::scoped_lock lock{m_mutex};
|
||||
wpi::SmallString<128> fullName{"/.schema/"};
|
||||
fullName += name;
|
||||
auto it = m_entries.find(fullName);
|
||||
return it != m_entries.end();
|
||||
}
|
||||
|
||||
void DataLog::AddSchema(std::string_view name, std::string_view type,
|
||||
std::span<const uint8_t> schema, int64_t timestamp) {
|
||||
std::scoped_lock lock{m_mutex};
|
||||
wpi::SmallString<128> fullName{"/.schema/"};
|
||||
fullName += name;
|
||||
auto& entryInfo = m_entries[fullName];
|
||||
if (entryInfo.id != 0) {
|
||||
return; // don't add duplicates
|
||||
}
|
||||
int entry = StartImpl(fullName, type, {}, timestamp);
|
||||
|
||||
// inline AppendRaw() without releasing lock
|
||||
if (entry <= 0) {
|
||||
[[unlikely]] return; // should never happen, but check anyway
|
||||
}
|
||||
StartRecord(entry, timestamp, schema.size(), 0);
|
||||
AppendImpl(schema);
|
||||
}
|
||||
|
||||
static void WriteToFile(fs::file_t f, std::span<const uint8_t> data,
|
||||
std::string_view filename, wpi::Logger& msglog) {
|
||||
do {
|
||||
@@ -484,6 +512,11 @@ void DataLog::WriterThreadMain(
|
||||
int DataLog::Start(std::string_view name, std::string_view type,
|
||||
std::string_view metadata, int64_t timestamp) {
|
||||
std::scoped_lock lock{m_mutex};
|
||||
return StartImpl(name, type, metadata, timestamp);
|
||||
}
|
||||
|
||||
int DataLog::StartImpl(std::string_view name, std::string_view type,
|
||||
std::string_view metadata, int64_t timestamp) {
|
||||
auto& entryInfo = m_entries[name];
|
||||
if (entryInfo.id == 0) {
|
||||
entryInfo.id = ++m_lastId;
|
||||
|
||||
@@ -111,6 +111,47 @@ Java_edu_wpi_first_util_datalog_DataLogJNI_resume
|
||||
reinterpret_cast<DataLog*>(impl)->Resume();
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: edu_wpi_first_util_datalog_DataLogJNI
|
||||
* Method: addSchema
|
||||
* Signature: (JLjava/lang/String;Ljava/lang/String;[BJ)V
|
||||
*/
|
||||
JNIEXPORT void JNICALL
|
||||
Java_edu_wpi_first_util_datalog_DataLogJNI_addSchema
|
||||
(JNIEnv* env, jclass, jlong impl, jstring name, jstring type,
|
||||
jbyteArray schema, jlong timestamp)
|
||||
{
|
||||
if (impl == 0) {
|
||||
wpi::ThrowNullPointerException(env, "impl is null");
|
||||
return;
|
||||
}
|
||||
reinterpret_cast<DataLog*>(impl)->AddSchema(
|
||||
JStringRef{env, name}, JStringRef{env, type},
|
||||
JSpan<const jbyte>{env, schema}.uarray(), timestamp);
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: edu_wpi_first_util_datalog_DataLogJNI
|
||||
* Method: addSchemaString
|
||||
* Signature: (JLjava/lang/String;Ljava/lang/String;Ljava/lang/String;J)V
|
||||
*/
|
||||
JNIEXPORT void JNICALL
|
||||
Java_edu_wpi_first_util_datalog_DataLogJNI_addSchemaString
|
||||
(JNIEnv* env, jclass, jlong impl, jstring name, jstring type, jstring schema,
|
||||
jlong timestamp)
|
||||
{
|
||||
if (impl == 0) {
|
||||
wpi::ThrowNullPointerException(env, "impl is null");
|
||||
return;
|
||||
}
|
||||
JStringRef schemaStr{env, schema};
|
||||
std::string_view schemaView = schemaStr.str();
|
||||
reinterpret_cast<DataLog*>(impl)->AddSchema(
|
||||
JStringRef{env, name}, JStringRef{env, type},
|
||||
{reinterpret_cast<const uint8_t*>(schemaView.data()), schemaView.size()},
|
||||
timestamp);
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: edu_wpi_first_util_datalog_DataLogJNI
|
||||
* Method: start
|
||||
|
||||
194
wpiutil/src/main/native/cpp/protobuf/Protobuf.cpp
Normal file
194
wpiutil/src/main/native/cpp/protobuf/Protobuf.cpp
Normal file
@@ -0,0 +1,194 @@
|
||||
// Copyright (c) FIRST and other WPILib contributors.
|
||||
// Open Source Software; you can modify and/or share it under the terms of
|
||||
// the WPILib BSD license file in the root directory of this project.
|
||||
|
||||
#include "wpi/protobuf/Protobuf.h"
|
||||
|
||||
#include <fmt/format.h>
|
||||
#include <google/protobuf/descriptor.h>
|
||||
#include <google/protobuf/descriptor.pb.h>
|
||||
#include <google/protobuf/io/zero_copy_stream.h>
|
||||
#include <google/protobuf/message.h>
|
||||
|
||||
#include "wpi/SmallVector.h"
|
||||
|
||||
using namespace wpi;
|
||||
|
||||
using google::protobuf::Arena;
|
||||
using google::protobuf::FileDescriptor;
|
||||
using google::protobuf::FileDescriptorProto;
|
||||
|
||||
namespace {
|
||||
class VectorOutputStream final
|
||||
: public google::protobuf::io::ZeroCopyOutputStream {
|
||||
public:
|
||||
// Create a StringOutputStream which appends bytes to the given string.
|
||||
// The string remains property of the caller, but it is mutated in arbitrary
|
||||
// ways and MUST NOT be accessed in any way until you're done with the
|
||||
// stream. Either be sure there's no further usage, or (safest) destroy the
|
||||
// stream before using the contents.
|
||||
//
|
||||
// Hint: If you call target->reserve(n) before creating the stream,
|
||||
// the first call to Next() will return at least n bytes of buffer
|
||||
// space.
|
||||
explicit VectorOutputStream(std::vector<uint8_t>& target) : target_{target} {}
|
||||
VectorOutputStream(const VectorOutputStream&) = delete;
|
||||
~VectorOutputStream() override = default;
|
||||
|
||||
VectorOutputStream& operator=(const VectorOutputStream&) = delete;
|
||||
|
||||
// implements ZeroCopyOutputStream ---------------------------------
|
||||
bool Next(void** data, int* size) override;
|
||||
void BackUp(int count) override { target_.resize(target_.size() - count); }
|
||||
int64_t ByteCount() const override { return target_.size(); }
|
||||
|
||||
private:
|
||||
static constexpr size_t kMinimumSize = 16;
|
||||
|
||||
std::vector<uint8_t>& target_;
|
||||
};
|
||||
|
||||
class SmallVectorOutputStream final
|
||||
: public google::protobuf::io::ZeroCopyOutputStream {
|
||||
public:
|
||||
// Create a StringOutputStream which appends bytes to the given string.
|
||||
// The string remains property of the caller, but it is mutated in arbitrary
|
||||
// ways and MUST NOT be accessed in any way until you're done with the
|
||||
// stream. Either be sure there's no further usage, or (safest) destroy the
|
||||
// stream before using the contents.
|
||||
//
|
||||
// Hint: If you call target->reserve(n) before creating the stream,
|
||||
// the first call to Next() will return at least n bytes of buffer
|
||||
// space.
|
||||
explicit SmallVectorOutputStream(wpi::SmallVectorImpl<uint8_t>& target)
|
||||
: target_{target} {
|
||||
target.resize(0);
|
||||
}
|
||||
SmallVectorOutputStream(const SmallVectorOutputStream&) = delete;
|
||||
~SmallVectorOutputStream() override = default;
|
||||
|
||||
SmallVectorOutputStream& operator=(const SmallVectorOutputStream&) = delete;
|
||||
|
||||
// implements ZeroCopyOutputStream ---------------------------------
|
||||
bool Next(void** data, int* size) override;
|
||||
void BackUp(int count) override { target_.resize(target_.size() - count); }
|
||||
int64_t ByteCount() const override { return target_.size(); }
|
||||
|
||||
private:
|
||||
static constexpr size_t kMinimumSize = 16;
|
||||
|
||||
wpi::SmallVectorImpl<uint8_t>& target_;
|
||||
};
|
||||
} // namespace
|
||||
|
||||
bool VectorOutputStream::Next(void** data, int* size) {
|
||||
size_t old_size = target_.size();
|
||||
|
||||
// Grow the string.
|
||||
size_t new_size;
|
||||
if (old_size < target_.capacity()) {
|
||||
// Resize to match its capacity, since we can get away
|
||||
// without a memory allocation this way.
|
||||
new_size = target_.capacity();
|
||||
} else {
|
||||
// Size has reached capacity, try to double it.
|
||||
new_size = old_size * 2;
|
||||
}
|
||||
// Avoid integer overflow in returned '*size'.
|
||||
new_size = (std::min)(new_size, old_size + (std::numeric_limits<int>::max)());
|
||||
// Increase the size, also make sure that it is at least kMinimumSize.
|
||||
target_.resize((std::max)(new_size, kMinimumSize));
|
||||
|
||||
*data = target_.data() + old_size;
|
||||
*size = target_.size() - old_size;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool SmallVectorOutputStream::Next(void** data, int* size) {
|
||||
size_t old_size = target_.size();
|
||||
|
||||
// Grow the string.
|
||||
size_t new_size;
|
||||
if (old_size < target_.capacity()) {
|
||||
// Resize to match its capacity, since we can get away
|
||||
// without a memory allocation this way.
|
||||
new_size = target_.capacity();
|
||||
} else {
|
||||
// Size has reached capacity, try to double it.
|
||||
new_size = old_size * 2;
|
||||
}
|
||||
// Avoid integer overflow in returned '*size'.
|
||||
new_size = (std::min)(new_size, old_size + (std::numeric_limits<int>::max)());
|
||||
// Increase the size, also make sure that it is at least kMinimumSize.
|
||||
target_.resize_for_overwrite((std::max)(new_size, kMinimumSize));
|
||||
|
||||
*data = target_.data() + old_size;
|
||||
*size = target_.size() - old_size;
|
||||
return true;
|
||||
}
|
||||
|
||||
void detail::DeleteProtobuf(google::protobuf::Message* msg) {
|
||||
if (msg && !msg->GetArena()) {
|
||||
delete msg;
|
||||
}
|
||||
}
|
||||
|
||||
bool detail::ParseProtobuf(google::protobuf::Message* msg,
|
||||
std::span<const uint8_t> data) {
|
||||
return msg->ParseFromArray(data.data(), data.size());
|
||||
}
|
||||
|
||||
bool detail::SerializeProtobuf(wpi::SmallVectorImpl<uint8_t>& out,
|
||||
const google::protobuf::Message& msg) {
|
||||
SmallVectorOutputStream stream{out};
|
||||
return msg.SerializeToZeroCopyStream(&stream);
|
||||
}
|
||||
|
||||
bool detail::SerializeProtobuf(std::vector<uint8_t>& out,
|
||||
const google::protobuf::Message& msg) {
|
||||
VectorOutputStream stream{out};
|
||||
return msg.SerializeToZeroCopyStream(&stream);
|
||||
}
|
||||
|
||||
std::string detail::GetTypeString(const google::protobuf::Message& msg) {
|
||||
return fmt::format("proto:{}", msg.GetDescriptor()->full_name());
|
||||
}
|
||||
|
||||
static void ForEachProtobufDescriptorImpl(
|
||||
const FileDescriptor* desc,
|
||||
function_ref<bool(std::string_view typeString)> exists,
|
||||
function_ref<void(std::string_view typeString,
|
||||
std::span<const uint8_t> schema)>
|
||||
fn,
|
||||
Arena* arena, FileDescriptorProto** descproto) {
|
||||
std::string name = fmt::format("proto:{}", desc->name());
|
||||
if (exists(name)) {
|
||||
return;
|
||||
}
|
||||
for (int i = 0, ndep = desc->dependency_count(); i < ndep; ++i) {
|
||||
ForEachProtobufDescriptorImpl(desc->dependency(i), exists, fn, arena,
|
||||
descproto);
|
||||
}
|
||||
if (!*descproto) {
|
||||
*descproto = Arena::CreateMessage<FileDescriptorProto>(arena);
|
||||
}
|
||||
(*descproto)->Clear();
|
||||
desc->CopyTo(*descproto);
|
||||
SmallVector<uint8_t, 128> buf;
|
||||
detail::SerializeProtobuf(buf, **descproto);
|
||||
fn(name, buf);
|
||||
}
|
||||
|
||||
void detail::ForEachProtobufDescriptor(
|
||||
const google::protobuf::Message& msg,
|
||||
function_ref<bool(std::string_view filename)> exists,
|
||||
function_ref<void(std::string_view filename,
|
||||
std::span<const uint8_t> descriptor)>
|
||||
fn) {
|
||||
FileDescriptorProto* descproto = nullptr;
|
||||
ForEachProtobufDescriptorImpl(msg.GetDescriptor()->file(), exists, fn,
|
||||
msg.GetArena(), &descproto);
|
||||
if (descproto && !msg.GetArena()) {
|
||||
delete descproto;
|
||||
}
|
||||
}
|
||||
119
wpiutil/src/main/native/cpp/protobuf/ProtobufMessageDatabase.cpp
Normal file
119
wpiutil/src/main/native/cpp/protobuf/ProtobufMessageDatabase.cpp
Normal file
@@ -0,0 +1,119 @@
|
||||
// Copyright (c) FIRST and other WPILib contributors.
|
||||
// Open Source Software; you can modify and/or share it under the terms of
|
||||
// the WPILib BSD license file in the root directory of this project.
|
||||
|
||||
#include "wpi/protobuf/ProtobufMessageDatabase.h"
|
||||
|
||||
#include <google/protobuf/descriptor.h>
|
||||
|
||||
using namespace wpi;
|
||||
|
||||
using google::protobuf::Arena;
|
||||
using google::protobuf::FileDescriptorProto;
|
||||
using google::protobuf::Message;
|
||||
|
||||
void ProtobufMessageDatabase::Add(std::string_view filename,
|
||||
std::span<const uint8_t> data) {
|
||||
auto& file = m_files[filename];
|
||||
if (file.complete) {
|
||||
file.complete = false;
|
||||
|
||||
// rebuild the pool EXCEPT for this descriptor
|
||||
m_pool = std::make_unique<google::protobuf::DescriptorPool>();
|
||||
|
||||
for (auto&& p : m_files) {
|
||||
p.second.inPool = false;
|
||||
}
|
||||
for (auto&& p : m_files) {
|
||||
if (p.second.complete && !p.second.inPool) {
|
||||
Rebuild(p.second);
|
||||
}
|
||||
}
|
||||
|
||||
// clear messages and reset factory; Find() will recreate as needed
|
||||
m_msgs.clear();
|
||||
m_factory = std::make_unique<google::protobuf::DynamicMessageFactory>();
|
||||
}
|
||||
|
||||
if (!file.proto) {
|
||||
file.proto = std::unique_ptr<FileDescriptorProto>{
|
||||
Arena::CreateMessage<FileDescriptorProto>(nullptr)};
|
||||
} else {
|
||||
// replacing an existing one; remove any previously existing refs
|
||||
for (auto&& dep : file.proto->dependency()) {
|
||||
auto& depFile = m_files[dep];
|
||||
std::erase(depFile.uses, filename);
|
||||
}
|
||||
file.proto->Clear();
|
||||
}
|
||||
|
||||
// parse data
|
||||
if (!file.proto->ParseFromArray(data.data(), data.size())) {
|
||||
return;
|
||||
}
|
||||
|
||||
Build(filename, file);
|
||||
}
|
||||
|
||||
Message* ProtobufMessageDatabase::Find(std::string_view name) const {
|
||||
// cached
|
||||
auto& msg = m_msgs[name];
|
||||
if (msg) {
|
||||
return msg.get();
|
||||
}
|
||||
|
||||
// need to create it
|
||||
auto desc = m_pool->FindMessageTypeByName(std::string{name});
|
||||
if (!desc) {
|
||||
return nullptr;
|
||||
}
|
||||
msg = std::unique_ptr<Message>{m_factory->GetPrototype(desc)->New(nullptr)};
|
||||
return msg.get();
|
||||
}
|
||||
|
||||
void ProtobufMessageDatabase::Build(std::string_view filename,
|
||||
ProtoFile& file) {
|
||||
if (file.complete) {
|
||||
return;
|
||||
}
|
||||
// are all of the dependencies complete?
|
||||
bool complete = true;
|
||||
for (auto&& dep : file.proto->dependency()) {
|
||||
auto& depFile = m_files[dep];
|
||||
if (!depFile.complete) {
|
||||
complete = false;
|
||||
}
|
||||
depFile.uses.emplace_back(filename);
|
||||
}
|
||||
if (!complete) {
|
||||
return;
|
||||
}
|
||||
|
||||
// add to pool
|
||||
if (!m_pool->BuildFile(*file.proto)) {
|
||||
return;
|
||||
}
|
||||
file.inPool = true;
|
||||
file.complete = true;
|
||||
|
||||
// recursively validate all uses
|
||||
for (auto&& use : file.uses) {
|
||||
Build(use, m_files[use]);
|
||||
}
|
||||
}
|
||||
|
||||
bool ProtobufMessageDatabase::Rebuild(ProtoFile& file) {
|
||||
for (auto&& dep : file.proto->dependency()) {
|
||||
auto& depFile = m_files[dep];
|
||||
if (!depFile.inPool) {
|
||||
if (!Rebuild(depFile)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!m_pool->BuildFile(*file.proto)) {
|
||||
return false;
|
||||
}
|
||||
file.inPool = true;
|
||||
return true;
|
||||
}
|
||||
444
wpiutil/src/main/native/cpp/struct/DynamicStruct.cpp
Normal file
444
wpiutil/src/main/native/cpp/struct/DynamicStruct.cpp
Normal file
@@ -0,0 +1,444 @@
|
||||
// Copyright (c) FIRST and other WPILib contributors.
|
||||
// Open Source Software; you can modify and/or share it under the terms of
|
||||
// the WPILib BSD license file in the root directory of this project.
|
||||
|
||||
#include "wpi/struct/DynamicStruct.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
#include <fmt/format.h>
|
||||
|
||||
#include "wpi/Endian.h"
|
||||
#include "wpi/SmallString.h"
|
||||
#include "wpi/SmallVector.h"
|
||||
#include "wpi/raw_ostream.h"
|
||||
#include "wpi/struct/SchemaParser.h"
|
||||
|
||||
using namespace wpi;
|
||||
|
||||
static size_t TypeToSize(StructFieldType type) {
|
||||
switch (type) {
|
||||
case StructFieldType::kBool:
|
||||
case StructFieldType::kChar:
|
||||
case StructFieldType::kInt8:
|
||||
case StructFieldType::kUint8:
|
||||
return 1;
|
||||
case StructFieldType::kInt16:
|
||||
case StructFieldType::kUint16:
|
||||
return 2;
|
||||
case StructFieldType::kInt32:
|
||||
case StructFieldType::kUint32:
|
||||
case StructFieldType::kFloat:
|
||||
return 4;
|
||||
case StructFieldType::kInt64:
|
||||
case StructFieldType::kUint64:
|
||||
case StructFieldType::kDouble:
|
||||
return 8;
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
static StructFieldType TypeStringToType(std::string_view str) {
|
||||
if (str == "bool") {
|
||||
return StructFieldType::kBool;
|
||||
} else if (str == "char") {
|
||||
return StructFieldType::kChar;
|
||||
} else if (str == "int8") {
|
||||
return StructFieldType::kInt8;
|
||||
} else if (str == "int16") {
|
||||
return StructFieldType::kInt16;
|
||||
} else if (str == "int32") {
|
||||
return StructFieldType::kInt32;
|
||||
} else if (str == "int64") {
|
||||
return StructFieldType::kInt64;
|
||||
} else if (str == "uint8") {
|
||||
return StructFieldType::kUint8;
|
||||
} else if (str == "uint16") {
|
||||
return StructFieldType::kUint16;
|
||||
} else if (str == "uint32") {
|
||||
return StructFieldType::kUint32;
|
||||
} else if (str == "uint64") {
|
||||
return StructFieldType::kUint64;
|
||||
} else if (str == "float" || str == "float32") {
|
||||
return StructFieldType::kFloat;
|
||||
} else if (str == "double" || str == "float64") {
|
||||
return StructFieldType::kDouble;
|
||||
} else {
|
||||
return StructFieldType::kStruct;
|
||||
}
|
||||
}
|
||||
|
||||
static inline unsigned int ToBitWidth(size_t size, unsigned int bitWidth) {
|
||||
if (bitWidth == 0) {
|
||||
return size * 8;
|
||||
} else {
|
||||
return bitWidth;
|
||||
}
|
||||
}
|
||||
|
||||
static inline uint64_t ToBitMask(size_t size, unsigned int bitWidth) {
|
||||
if (size == 0) {
|
||||
return 0;
|
||||
} else {
|
||||
return UINT64_MAX >> (64 - ToBitWidth(size, bitWidth));
|
||||
}
|
||||
}
|
||||
|
||||
StructFieldDescriptor::StructFieldDescriptor(
|
||||
const StructDescriptor* parent, std::string_view name, StructFieldType type,
|
||||
size_t size, size_t arraySize, unsigned int bitWidth, EnumValues enumValues,
|
||||
const StructDescriptor* structDesc, const private_init&)
|
||||
: m_parent{parent},
|
||||
m_name{name},
|
||||
m_size{size},
|
||||
m_arraySize{arraySize},
|
||||
m_enum{std::move(enumValues)},
|
||||
m_struct{structDesc},
|
||||
m_bitMask{ToBitMask(size, bitWidth)},
|
||||
m_type{type},
|
||||
m_bitWidth{ToBitWidth(size, bitWidth)} {}
|
||||
|
||||
const StructFieldDescriptor* StructDescriptor::FindFieldByName(
|
||||
std::string_view name) const {
|
||||
auto it = m_fieldsByName.find(name);
|
||||
if (it == m_fieldsByName.end()) {
|
||||
return nullptr;
|
||||
}
|
||||
return &m_fields[it->second];
|
||||
}
|
||||
|
||||
bool StructDescriptor::CheckCircular(
|
||||
wpi::SmallVectorImpl<const StructDescriptor*>& stack) const {
|
||||
stack.emplace_back(this);
|
||||
for (auto&& ref : m_references) {
|
||||
if (std::find(stack.begin(), stack.end(), ref) != stack.end()) {
|
||||
[[unlikely]] return false;
|
||||
}
|
||||
if (!ref->CheckCircular(stack)) {
|
||||
[[unlikely]] return false;
|
||||
}
|
||||
}
|
||||
stack.pop_back();
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string StructDescriptor::CalculateOffsets(
|
||||
wpi::SmallVectorImpl<const StructDescriptor*>& stack) {
|
||||
size_t offset = 0;
|
||||
unsigned int shift = 0;
|
||||
size_t prevBitfieldSize = 0;
|
||||
for (auto&& field : m_fields) {
|
||||
if (!field.IsBitField()) {
|
||||
[[likely]] shift = 0; // reset shift on non-bitfield element
|
||||
offset += prevBitfieldSize; // finish bitfield if active
|
||||
prevBitfieldSize = 0; // previous is now not bitfield
|
||||
field.m_offset = offset;
|
||||
if (field.m_struct) {
|
||||
if (!field.m_struct->IsValid()) {
|
||||
m_valid = false;
|
||||
[[unlikely]] return {};
|
||||
}
|
||||
field.m_size = field.m_struct->m_size;
|
||||
}
|
||||
offset += field.m_size * field.m_arraySize;
|
||||
} else {
|
||||
if (field.m_type == StructFieldType::kBool && prevBitfieldSize != 0 &&
|
||||
(shift + 1) <= (prevBitfieldSize * 8)) {
|
||||
// bool takes on size of preceding bitfield type (if it fits)
|
||||
field.m_size = prevBitfieldSize;
|
||||
} else if (field.m_size != prevBitfieldSize ||
|
||||
(shift + field.m_bitWidth) > (field.m_size * 8)) {
|
||||
shift = 0;
|
||||
offset += prevBitfieldSize;
|
||||
}
|
||||
prevBitfieldSize = field.m_size;
|
||||
field.m_offset = offset;
|
||||
field.m_bitShift = shift;
|
||||
shift += field.m_bitWidth;
|
||||
}
|
||||
}
|
||||
|
||||
// update struct size
|
||||
m_size = offset + prevBitfieldSize;
|
||||
m_valid = true;
|
||||
|
||||
// now that we're valid, referring types may be too
|
||||
stack.emplace_back(this);
|
||||
for (auto&& ref : m_references) {
|
||||
if (std::find(stack.begin(), stack.end(), ref) != stack.end()) {
|
||||
[[unlikely]] return fmt::format(
|
||||
"internal error (inconsistent data): circular struct reference "
|
||||
"between {} and {}",
|
||||
m_name, ref->m_name);
|
||||
}
|
||||
auto err = ref->CalculateOffsets(stack);
|
||||
if (!err.empty()) {
|
||||
[[unlikely]] return err;
|
||||
}
|
||||
}
|
||||
stack.pop_back();
|
||||
return {};
|
||||
}
|
||||
|
||||
const StructDescriptor* StructDescriptorDatabase::Add(std::string_view name,
|
||||
std::string_view schema,
|
||||
std::string* err) {
|
||||
structparser::Parser parser{schema};
|
||||
structparser::ParsedSchema parsed;
|
||||
if (!parser.Parse(&parsed)) {
|
||||
*err = fmt::format("parse error: {}", parser.GetError());
|
||||
[[unlikely]] return nullptr;
|
||||
}
|
||||
|
||||
// turn parsed schema into descriptors
|
||||
auto& theStruct = m_structs[name];
|
||||
if (!theStruct) {
|
||||
theStruct = std::make_unique<StructDescriptor>(
|
||||
name, StructDescriptor::private_init{});
|
||||
}
|
||||
theStruct->m_schema = schema;
|
||||
theStruct->m_fields.clear();
|
||||
theStruct->m_fields.reserve(parsed.declarations.size());
|
||||
bool isValid = true;
|
||||
for (auto&& decl : parsed.declarations) {
|
||||
auto type = TypeStringToType(decl.typeString);
|
||||
size_t size = TypeToSize(type);
|
||||
|
||||
// bitfield checks
|
||||
if (decl.bitWidth != 0) {
|
||||
// only integer or boolean types are allowed
|
||||
if (type == StructFieldType::kChar || type == StructFieldType::kFloat ||
|
||||
type == StructFieldType::kDouble ||
|
||||
type == StructFieldType::kStruct) {
|
||||
*err = fmt::format("field {}: type {} cannot be bitfield", decl.name,
|
||||
decl.typeString);
|
||||
[[unlikely]] return nullptr;
|
||||
}
|
||||
|
||||
// bit width cannot be larger than field size
|
||||
if (decl.bitWidth > (size * 8)) {
|
||||
*err = fmt::format("field {}: bit width {} exceeds type size",
|
||||
decl.name, decl.bitWidth);
|
||||
[[unlikely]] return nullptr;
|
||||
}
|
||||
|
||||
// bit width must be 1 for booleans
|
||||
if (type == StructFieldType::kBool && decl.bitWidth != 1) {
|
||||
*err = fmt::format("field {}: bit width must be 1 for bool type",
|
||||
decl.name);
|
||||
[[unlikely]] return nullptr;
|
||||
}
|
||||
|
||||
// cannot combine array and bitfield (shouldn't parse, but double-check)
|
||||
if (decl.arraySize > 1) {
|
||||
*err = fmt::format("field {}: cannot combine array and bitfield",
|
||||
decl.name);
|
||||
[[unlikely]] return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
// struct handling
|
||||
const StructDescriptor* structDesc = nullptr;
|
||||
if (type == StructFieldType::kStruct) {
|
||||
// recursive definitions are not allowed
|
||||
if (decl.typeString == name) {
|
||||
*err = fmt::format("field {}: recursive struct reference", decl.name);
|
||||
[[unlikely]] return nullptr;
|
||||
}
|
||||
|
||||
// cross-reference struct, creating a placeholder if necessary
|
||||
auto& aStruct = m_structs[decl.typeString];
|
||||
if (!aStruct) {
|
||||
aStruct = std::make_unique<StructDescriptor>(
|
||||
decl.typeString, StructDescriptor::private_init{});
|
||||
}
|
||||
|
||||
// if the struct isn't valid, we can't be valid either
|
||||
if (aStruct->IsValid()) {
|
||||
size = aStruct->GetSize();
|
||||
} else {
|
||||
isValid = false;
|
||||
}
|
||||
|
||||
// add to cross-references for when the struct does become valid
|
||||
aStruct->m_references.emplace_back(theStruct.get());
|
||||
structDesc = aStruct.get();
|
||||
}
|
||||
|
||||
// create field
|
||||
if (!theStruct->m_fieldsByName
|
||||
.insert({decl.name, theStruct->m_fields.size()})
|
||||
.second) {
|
||||
*err = fmt::format("duplicate field {}", decl.name);
|
||||
[[unlikely]] return nullptr;
|
||||
}
|
||||
|
||||
theStruct->m_fields.emplace_back(theStruct.get(), decl.name, type, size,
|
||||
decl.arraySize, decl.bitWidth,
|
||||
std::move(decl.enumValues), structDesc,
|
||||
StructFieldDescriptor::private_init{});
|
||||
}
|
||||
|
||||
theStruct->m_valid = isValid;
|
||||
if (isValid) {
|
||||
// we have all the info needed, so calculate field offset & shift
|
||||
wpi::SmallVector<const StructDescriptor*, 16> stack;
|
||||
auto err2 = theStruct->CalculateOffsets(stack);
|
||||
if (!err2.empty()) {
|
||||
*err = std::move(err2);
|
||||
[[unlikely]] return nullptr;
|
||||
}
|
||||
} else {
|
||||
// check for circular reference
|
||||
wpi::SmallVector<const StructDescriptor*, 16> stack;
|
||||
if (!theStruct->CheckCircular(stack)) {
|
||||
wpi::SmallString<128> buf;
|
||||
wpi::raw_svector_ostream os{buf};
|
||||
for (auto&& elem : stack) {
|
||||
if (!buf.empty()) {
|
||||
os << " <- ";
|
||||
}
|
||||
os << elem->GetName();
|
||||
}
|
||||
*err = fmt::format("circular struct reference: {}", os.str());
|
||||
[[unlikely]] return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
return theStruct.get();
|
||||
}
|
||||
|
||||
const StructDescriptor* StructDescriptorDatabase::Find(
|
||||
std::string_view name) const {
|
||||
auto it = m_structs.find(name);
|
||||
if (it == m_structs.end()) {
|
||||
return nullptr;
|
||||
}
|
||||
return it->second.get();
|
||||
}
|
||||
|
||||
uint64_t DynamicStruct::GetFieldImpl(const StructFieldDescriptor* field,
|
||||
size_t arrIndex) const {
|
||||
assert(field->m_parent == m_desc);
|
||||
assert(m_desc->IsValid());
|
||||
assert(arrIndex < field->m_arraySize);
|
||||
uint64_t val;
|
||||
switch (field->m_size) {
|
||||
case 1:
|
||||
val = m_data[field->m_offset + arrIndex];
|
||||
break;
|
||||
case 2:
|
||||
val = support::endian::read16le(&m_data[field->m_offset + arrIndex * 2]);
|
||||
break;
|
||||
case 4:
|
||||
val = support::endian::read32le(&m_data[field->m_offset + arrIndex * 4]);
|
||||
break;
|
||||
case 8:
|
||||
val = support::endian::read64le(&m_data[field->m_offset + arrIndex * 8]);
|
||||
break;
|
||||
default:
|
||||
assert(false && "invalid field size");
|
||||
return 0;
|
||||
}
|
||||
return (val >> field->m_bitShift) & field->m_bitMask;
|
||||
}
|
||||
|
||||
void MutableDynamicStruct::SetData(std::span<const uint8_t> data) {
|
||||
assert(data.size() >= m_desc->GetSize());
|
||||
std::copy(data.begin(), data.begin() + m_desc->GetSize(), m_data.begin());
|
||||
}
|
||||
|
||||
void MutableDynamicStruct::SetStringField(const StructFieldDescriptor* field,
|
||||
std::string_view value) {
|
||||
assert(field->m_type == StructFieldType::kChar);
|
||||
assert(field->m_parent == m_desc);
|
||||
assert(m_desc->IsValid());
|
||||
size_t len = (std::min)(field->m_arraySize, value.size());
|
||||
std::copy(value.begin(), value.begin() + len,
|
||||
reinterpret_cast<char*>(&m_data[field->m_offset]));
|
||||
std::fill(&m_data[field->m_offset + len],
|
||||
&m_data[field->m_offset + field->m_arraySize], 0);
|
||||
}
|
||||
|
||||
void MutableDynamicStruct::SetStructField(const StructFieldDescriptor* field,
|
||||
const DynamicStruct& value,
|
||||
size_t arrIndex) {
|
||||
assert(field->m_type == StructFieldType::kStruct);
|
||||
assert(field->m_parent == m_desc);
|
||||
assert(m_desc->IsValid());
|
||||
assert(value.GetDescriptor() == field->m_struct);
|
||||
assert(value.GetDescriptor()->IsValid());
|
||||
assert(arrIndex < field->m_arraySize);
|
||||
auto source = value.GetData();
|
||||
size_t len = field->m_struct->GetSize();
|
||||
std::copy(source.begin(), source.begin() + len,
|
||||
m_data.begin() + field->m_offset + arrIndex * len);
|
||||
}
|
||||
|
||||
void MutableDynamicStruct::SetFieldImpl(const StructFieldDescriptor* field,
|
||||
uint64_t value, size_t arrIndex) {
|
||||
assert(field->m_parent == m_desc);
|
||||
assert(m_desc->IsValid());
|
||||
assert(arrIndex < field->m_arraySize);
|
||||
|
||||
// common case is no bit shift and no masking
|
||||
if (!field->IsBitField()) {
|
||||
switch (field->m_size) {
|
||||
case 1:
|
||||
m_data[field->m_offset + arrIndex] = value;
|
||||
break;
|
||||
case 2:
|
||||
support::endian::write16le(&m_data[field->m_offset + arrIndex * 2],
|
||||
value);
|
||||
break;
|
||||
case 4:
|
||||
support::endian::write32le(&m_data[field->m_offset + arrIndex * 4],
|
||||
value);
|
||||
break;
|
||||
case 8:
|
||||
support::endian::write64le(&m_data[field->m_offset + arrIndex * 8],
|
||||
value);
|
||||
break;
|
||||
default:
|
||||
assert(false && "invalid field size");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// handle bit shifting and masking into current value
|
||||
switch (field->m_size) {
|
||||
case 1: {
|
||||
uint8_t* data = &m_data[field->m_offset + arrIndex];
|
||||
*data &= ~(field->m_bitMask << field->m_bitShift);
|
||||
*data |= (value & field->m_bitMask) << field->m_bitShift;
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
uint8_t* data = &m_data[field->m_offset + arrIndex * 2];
|
||||
uint16_t val = support::endian::read16le(data);
|
||||
val &= ~(field->m_bitMask << field->m_bitShift);
|
||||
val |= (value & field->m_bitMask) << field->m_bitShift;
|
||||
support::endian::write16le(data, val);
|
||||
break;
|
||||
}
|
||||
case 4: {
|
||||
uint8_t* data = &m_data[field->m_offset + arrIndex * 4];
|
||||
uint32_t val = support::endian::read32le(data);
|
||||
val &= ~(field->m_bitMask << field->m_bitShift);
|
||||
val |= (value & field->m_bitMask) << field->m_bitShift;
|
||||
support::endian::write32le(data, val);
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
uint8_t* data = &m_data[field->m_offset + arrIndex * 8];
|
||||
uint64_t val = support::endian::read64le(data);
|
||||
val &= ~(field->m_bitMask << field->m_bitShift);
|
||||
val |= (value & field->m_bitMask) << field->m_bitShift;
|
||||
support::endian::write64le(data, val);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
assert(false && "invalid field size");
|
||||
}
|
||||
}
|
||||
238
wpiutil/src/main/native/cpp/struct/SchemaParser.cpp
Normal file
238
wpiutil/src/main/native/cpp/struct/SchemaParser.cpp
Normal file
@@ -0,0 +1,238 @@
|
||||
// Copyright (c) FIRST and other WPILib contributors.
|
||||
// Open Source Software; you can modify and/or share it under the terms of
|
||||
// the WPILib BSD license file in the root directory of this project.
|
||||
|
||||
#include "wpi/struct/SchemaParser.h"
|
||||
|
||||
#include <fmt/format.h>
|
||||
|
||||
#include "wpi/StringExtras.h"
|
||||
|
||||
using namespace wpi::structparser;
|
||||
|
||||
std::string_view wpi::structparser::ToString(Token::Kind kind) {
|
||||
switch (kind) {
|
||||
case Token::kInteger:
|
||||
return "integer";
|
||||
case Token::kIdentifier:
|
||||
return "identifier";
|
||||
case Token::kLeftBracket:
|
||||
return "'['";
|
||||
case Token::kRightBracket:
|
||||
return "']'";
|
||||
case Token::kLeftBrace:
|
||||
return "'{'";
|
||||
case Token::kRightBrace:
|
||||
return "'}'";
|
||||
case Token::kColon:
|
||||
return "':'";
|
||||
case Token::kSemicolon:
|
||||
return "';'";
|
||||
case Token::kComma:
|
||||
return "','";
|
||||
case Token::kEquals:
|
||||
return "'='";
|
||||
case Token::kEndOfInput:
|
||||
return "<EOF>";
|
||||
default:
|
||||
return "unknown";
|
||||
}
|
||||
}
|
||||
|
||||
Token Lexer::Scan() {
|
||||
// skip whitespace
|
||||
do {
|
||||
Get();
|
||||
} while (m_current == ' ' || m_current == '\t' || m_current == '\n' ||
|
||||
m_current == '\r');
|
||||
m_tokenStart = m_pos - 1;
|
||||
|
||||
switch (m_current) {
|
||||
case '[':
|
||||
return MakeToken(Token::kLeftBracket);
|
||||
case ']':
|
||||
return MakeToken(Token::kRightBracket);
|
||||
case '{':
|
||||
return MakeToken(Token::kLeftBrace);
|
||||
case '}':
|
||||
return MakeToken(Token::kRightBrace);
|
||||
case ':':
|
||||
return MakeToken(Token::kColon);
|
||||
case ';':
|
||||
return MakeToken(Token::kSemicolon);
|
||||
case ',':
|
||||
return MakeToken(Token::kComma);
|
||||
case '=':
|
||||
return MakeToken(Token::kEquals);
|
||||
case '-':
|
||||
case '0':
|
||||
case '1':
|
||||
case '2':
|
||||
case '3':
|
||||
case '4':
|
||||
case '5':
|
||||
case '6':
|
||||
case '7':
|
||||
case '8':
|
||||
case '9':
|
||||
return ScanInteger();
|
||||
case -1:
|
||||
return {Token::kEndOfInput, {}};
|
||||
default:
|
||||
if (isAlpha(m_current) || m_current == '_') {
|
||||
[[likely]] return ScanIdentifier();
|
||||
}
|
||||
return MakeToken(Token::kUnknown);
|
||||
}
|
||||
}
|
||||
|
||||
Token Lexer::ScanInteger() {
|
||||
do {
|
||||
Get();
|
||||
} while (isDigit(m_current));
|
||||
Unget();
|
||||
return MakeToken(Token::kInteger);
|
||||
}
|
||||
|
||||
Token Lexer::ScanIdentifier() {
|
||||
do {
|
||||
Get();
|
||||
} while (isAlnum(m_current) || m_current == '_');
|
||||
Unget();
|
||||
return MakeToken(Token::kIdentifier);
|
||||
}
|
||||
|
||||
void Parser::FailExpect(Token::Kind desired) {
|
||||
Fail(fmt::format("expected {}, got '{}'", ToString(desired), m_token.text));
|
||||
}
|
||||
|
||||
void Parser::Fail(std::string_view msg) {
|
||||
m_error = fmt::format("{}: {}", m_lexer.GetPosition(), msg);
|
||||
}
|
||||
|
||||
bool Parser::Parse(ParsedSchema* out) {
|
||||
do {
|
||||
GetNextToken();
|
||||
if (m_token.Is(Token::kSemicolon)) {
|
||||
continue;
|
||||
}
|
||||
if (m_token.Is(Token::kEndOfInput)) {
|
||||
break;
|
||||
}
|
||||
if (!ParseDeclaration(&out->declarations.emplace_back())) {
|
||||
[[unlikely]] return false;
|
||||
}
|
||||
} while (m_token.kind != Token::kEndOfInput);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Parser::ParseDeclaration(ParsedDeclaration* out) {
|
||||
// optional enum specification
|
||||
if (m_token.Is(Token::kIdentifier) && m_token.text == "enum") {
|
||||
GetNextToken();
|
||||
if (!Expect(Token::kLeftBrace)) {
|
||||
[[unlikely]] return false;
|
||||
}
|
||||
if (!ParseEnum(&out->enumValues)) {
|
||||
[[unlikely]] return false;
|
||||
}
|
||||
GetNextToken();
|
||||
} else if (m_token.Is(Token::kLeftBrace)) {
|
||||
if (!ParseEnum(&out->enumValues)) {
|
||||
[[unlikely]] return false;
|
||||
}
|
||||
GetNextToken();
|
||||
}
|
||||
|
||||
// type name
|
||||
if (!Expect(Token::kIdentifier)) {
|
||||
[[unlikely]] return false;
|
||||
}
|
||||
out->typeString = m_token.text;
|
||||
GetNextToken();
|
||||
|
||||
// identifier name
|
||||
if (!Expect(Token::kIdentifier)) {
|
||||
[[unlikely]] return false;
|
||||
}
|
||||
out->name = m_token.text;
|
||||
GetNextToken();
|
||||
|
||||
// array or bit field
|
||||
if (m_token.Is(Token::kLeftBracket)) {
|
||||
GetNextToken();
|
||||
if (!Expect(Token::kInteger)) {
|
||||
[[unlikely]] return false;
|
||||
}
|
||||
auto val = parse_integer<uint64_t>(m_token.text, 10);
|
||||
if (val && *val > 0) {
|
||||
out->arraySize = *val;
|
||||
} else {
|
||||
Fail(fmt::format("array size '{}' is not a positive integer",
|
||||
m_token.text));
|
||||
[[unlikely]] return false;
|
||||
}
|
||||
GetNextToken();
|
||||
if (!Expect(Token::kRightBracket)) {
|
||||
[[unlikely]] return false;
|
||||
}
|
||||
GetNextToken();
|
||||
} else if (m_token.Is(Token::kColon)) {
|
||||
GetNextToken();
|
||||
if (!Expect(Token::kInteger)) {
|
||||
[[unlikely]] return false;
|
||||
}
|
||||
auto val = parse_integer<unsigned int>(m_token.text, 10);
|
||||
if (val && *val > 0) {
|
||||
out->bitWidth = *val;
|
||||
} else {
|
||||
Fail(fmt::format("bitfield width '{}' is not a positive integer",
|
||||
m_token.text));
|
||||
[[unlikely]] return false;
|
||||
}
|
||||
GetNextToken();
|
||||
}
|
||||
|
||||
// declaration must end with EOF or semicolon
|
||||
if (m_token.Is(Token::kEndOfInput)) {
|
||||
return true;
|
||||
}
|
||||
return Expect(Token::kSemicolon);
|
||||
}
|
||||
|
||||
bool Parser::ParseEnum(EnumValues* out) {
|
||||
// we start with current = '{'
|
||||
GetNextToken();
|
||||
while (!m_token.Is(Token::kRightBrace)) {
|
||||
if (!Expect(Token::kIdentifier)) {
|
||||
[[unlikely]] return false;
|
||||
}
|
||||
std::string name;
|
||||
name = m_token.text;
|
||||
GetNextToken();
|
||||
if (!Expect(Token::kEquals)) {
|
||||
[[unlikely]] return false;
|
||||
}
|
||||
GetNextToken();
|
||||
if (!Expect(Token::kInteger)) {
|
||||
[[unlikely]] return false;
|
||||
}
|
||||
int64_t value;
|
||||
if (auto val = parse_integer<int64_t>(m_token.text, 10)) {
|
||||
value = *val;
|
||||
} else {
|
||||
Fail(fmt::format("could not parse enum value '{}'", m_token.text));
|
||||
[[unlikely]] return false;
|
||||
}
|
||||
out->emplace_back(std::move(name), value);
|
||||
GetNextToken();
|
||||
if (m_token.Is(Token::kRightBrace)) {
|
||||
break;
|
||||
}
|
||||
if (!Expect(Token::kComma)) {
|
||||
[[unlikely]] return false;
|
||||
}
|
||||
GetNextToken();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
Reference in New Issue
Block a user