Team2890/allwpilib
4
0
Fork 0
mirror of https://github.com/wpilibsuite/allwpilib synced 2026-06-23 01:21:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

[wpiutil,glass,dlt] Replace libprotobuf with upb for dynamic decode (#7988)

Browse Source 
libprotobuf is a very annoying dependency to deal with, and with the switch to nanopb for generated C++ code, libprotobuf is only used for dynamic decode in the GUI apps. libprotobuf has been swapped out with upb, a much smaller C-based library that supports reflection and can therefore do dynamic decode. This means we can remove the libprotobuf dependency and stop dealing with build issues because of it.
This commit is contained in:
Gold856
2025-08-17 01:56:32 -04:00
committed by GitHub
parent 7f35104012
commit d4311d5a29
408 changed files with 31336 additions and 141509 deletions
Download Patch File Download Diff File Expand all files Collapse all files

368
glass/src/libnt/native/cpp/NetworkTables.cpp
View File

@@ -25,6 +25,12 @@
#include <ntcore_c.h>
#include <ntcore_cpp.h>
#include <ntcore_cpp_types.h>
#include <upb/message/message.h>
#include <upb/mini_table/message.h>
#include <upb/reflection/def.h>
#include <upb/reflection/message.h>
#include <upb/reflection/stage0/google/protobuf/descriptor.upb.h>
#include <upb/wire/decode.h>
#include <wpi/MessagePack.h>
#include <wpi/SmallString.h>
#include <wpi/SpanExtras.h>
@@ -33,11 +39,6 @@
#include <wpi/print.h>
#include <wpi/raw_ostream.h>
#ifndef NO_PROTOBUF
#include <google/protobuf/descriptor.h>
#include <google/protobuf/message.h>
#endif
#include "glass/Context.h"
#include "glass/DataSource.h"
#include "glass/Storage.h"
@@ -353,172 +354,218 @@ static void UpdateStructValueSource(NetworkTablesModel& model,
}
}
#ifndef NO_PROTOBUF
static void UpdateProtobufValueSource(NetworkTablesModel& model,
NetworkTablesModel::ValueSource* out,
const google::protobuf::Message& msg,
const upb_Message* msg,
const upb_MessageDef* msgDef,
std::string_view name, int64_t time) {
auto desc = msg.GetDescriptor();
out->typeStr = fmt::format("proto:{}", desc->full_name());
out->typeStr = fmt::format("proto:{}", upb_MessageDef_FullName(msgDef));
int fieldCount = upb_MessageDef_FieldCount(msgDef);
if (!out->valueChildrenMap ||
desc->field_count() != static_cast<int>(out->valueChildren.size())) {
fieldCount != static_cast<int>(out->valueChildren.size())) {
out->valueChildren.clear();
out->valueChildrenMap = true;
out->valueChildren.reserve(desc->field_count());
for (int i = 0, end = desc->field_count(); i < end; ++i) {
out->valueChildren.reserve(fieldCount);
for (int i = 0, end = fieldCount; i < end; ++i) {
out->valueChildren.emplace_back();
auto& child = out->valueChildren.back();
child.name = desc->field(i)->name();
child.name = upb_FieldDef_Name(upb_MessageDef_Field(msgDef, i));
child.path = fmt::format("{}/{}", name, child.name);
}
}
auto refl = msg.GetReflection();
auto outIt = out->valueChildren.begin();
for (int fieldNum = 0, end = desc->field_count(); fieldNum < end;
++fieldNum) {
auto field = desc->field(fieldNum);
for (int fieldNum = 0, end = fieldCount; fieldNum < end; ++fieldNum) {
const upb_FieldDef* field = upb_MessageDef_Field(msgDef, fieldNum);
auto& child = *outIt++;
switch (field->cpp_type()) {
case google::protobuf::FieldDescriptor::CPPTYPE_BOOL:
if (field->is_repeated()) {
size_t size = refl->FieldSize(msg, field);
switch (upb_FieldDef_CType(field)) {
case kUpb_CType_Bool: {
if (upb_FieldDef_IsRepeated(field)) {
const upb_Array* arr =
upb_Message_GetFieldByDef(msg, field).array_val;
size_t size = upb_Array_Size(arr);
std::vector<int> v;
v.reserve(size);
for (size_t i = 0; i < size; ++i) {
v.emplace_back(refl->GetRepeatedBool(msg, field, i));
v.emplace_back(upb_Array_Get(arr, i).bool_val);
}
child.value = nt::Value::MakeBooleanArray(std::move(v), time);
} else {
child.value = nt::Value::MakeBoolean(refl->GetBool(msg, field), time);
}
child.UpdateFromValue(model, child.path, "");
break;
case google::protobuf::FieldDescriptor::CPPTYPE_STRING:
if (field->is_repeated()) {
size_t size = refl->FieldSize(msg, field);
std::vector<std::string> v;
v.reserve(size);
for (size_t i = 0; i < size; ++i) {
v.emplace_back(refl->GetRepeatedString(msg, field, i));
}
child.value = nt::Value::MakeStringArray(std::move(v), time);
} else {
child.value =
nt::Value::MakeString(refl->GetString(msg, field), time);
child.value = nt::Value::MakeBoolean(
upb_Message_GetFieldByDef(msg, field).bool_val, time);
child.UpdateFromValue(model, child.path, "");
}
break;
case google::protobuf::FieldDescriptor::CPPTYPE_INT32:
if (field->is_repeated()) {
size_t size = refl->FieldSize(msg, field);
std::vector<int64_t> v;
v.reserve(size);
for (size_t i = 0; i < size; ++i) {
v.emplace_back(refl->GetRepeatedInt32(msg, field, i));
}
child.value = nt::Value::MakeIntegerArray(std::move(v), time);
} else {
child.value =
nt::Value::MakeInteger(refl->GetInt32(msg, field), time);
}
child.UpdateFromValue(model, child.path, "");
break;
case google::protobuf::FieldDescriptor::CPPTYPE_INT64:
if (field->is_repeated()) {
size_t size = refl->FieldSize(msg, field);
std::vector<int64_t> v;
v.reserve(size);
for (size_t i = 0; i < size; ++i) {
v.emplace_back(refl->GetRepeatedInt64(msg, field, i));
}
child.value = nt::Value::MakeIntegerArray(std::move(v), time);
} else {
child.value =
nt::Value::MakeInteger(refl->GetInt64(msg, field), time);
}
child.UpdateFromValue(model, child.path, "");
break;
case google::protobuf::FieldDescriptor::CPPTYPE_UINT32:
if (field->is_repeated()) {
size_t size = refl->FieldSize(msg, field);
std::vector<int64_t> v;
v.reserve(size);
for (size_t i = 0; i < size; ++i) {
v.emplace_back(refl->GetRepeatedUInt32(msg, field, i));
}
child.value = nt::Value::MakeIntegerArray(std::move(v), time);
} else {
child.value =
nt::Value::MakeInteger(refl->GetUInt32(msg, field), time);
}
child.UpdateFromValue(model, child.path, "");
break;
case google::protobuf::FieldDescriptor::CPPTYPE_UINT64:
if (field->is_repeated()) {
size_t size = refl->FieldSize(msg, field);
std::vector<int64_t> v;
v.reserve(size);
for (size_t i = 0; i < size; ++i) {
v.emplace_back(refl->GetRepeatedUInt64(msg, field, i));
}
child.value = nt::Value::MakeIntegerArray(std::move(v), time);
} else {
child.value =
nt::Value::MakeInteger(refl->GetUInt64(msg, field), time);
}
child.UpdateFromValue(model, child.path, "");
break;
case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT:
if (field->is_repeated()) {
size_t size = refl->FieldSize(msg, field);
std::vector<float> v;
v.reserve(size);
for (size_t i = 0; i < size; ++i) {
v.emplace_back(refl->GetRepeatedFloat(msg, field, i));
}
child.value = nt::Value::MakeFloatArray(std::move(v), time);
} else {
child.value = nt::Value::MakeFloat(refl->GetFloat(msg, field), time);
}
child.UpdateFromValue(model, child.path, "");
break;
case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE:
if (field->is_repeated()) {
size_t size = refl->FieldSize(msg, field);
std::vector<double> v;
v.reserve(size);
for (size_t i = 0; i < size; ++i) {
v.emplace_back(refl->GetRepeatedDouble(msg, field, i));
}
child.value = nt::Value::MakeDoubleArray(std::move(v), time);
} else {
child.value =
nt::Value::MakeDouble(refl->GetDouble(msg, field), time);
}
child.UpdateFromValue(model, child.path, "");
break;
case google::protobuf::FieldDescriptor::CPPTYPE_ENUM:
if (field->is_repeated()) {
size_t size = refl->FieldSize(msg, field);
}
case kUpb_CType_String: {
if (upb_FieldDef_IsRepeated(field)) {
const upb_Array* arr =
upb_Message_GetFieldByDef(msg, field).array_val;
size_t size = upb_Array_Size(arr);
std::vector<std::string> v;
v.reserve(size);
for (size_t i = 0; i < size; ++i) {
v.emplace_back(refl->GetRepeatedEnum(msg, field, i)->name());
upb_StringView sv = upb_Array_Get(arr, i).str_val;
v.emplace_back(sv.data, sv.size);
}
child.value = nt::Value::MakeStringArray(std::move(v), time);
} else {
child.value =
nt::Value::MakeString(refl->GetEnum(msg, field)->name(), time);
upb_StringView sv = upb_Message_GetFieldByDef(msg, field).str_val;
child.value = nt::Value::MakeString({sv.data, sv.size}, time);
child.UpdateFromValue(model, child.path, "");
}
break;
}
case kUpb_CType_Int32: {
if (upb_FieldDef_IsRepeated(field)) {
const upb_Array* arr =
upb_Message_GetFieldByDef(msg, field).array_val;
size_t size = upb_Array_Size(arr);
std::vector<int64_t> v;
v.reserve(size);
for (size_t i = 0; i < size; ++i) {
v.emplace_back(upb_Array_Get(arr, i).int32_val);
}
child.value = nt::Value::MakeIntegerArray(std::move(v), time);
} else {
child.value = nt::Value::MakeInteger(
upb_Message_GetFieldByDef(msg, field).int32_val, time);
}
child.UpdateFromValue(model, child.path, "");
break;
case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE:
if (field->is_repeated()) {
}
case kUpb_CType_Int64: {
if (upb_FieldDef_IsRepeated(field)) {
const upb_Array* arr =
upb_Message_GetFieldByDef(msg, field).array_val;
size_t size = upb_Array_Size(arr);
std::vector<int64_t> v;
v.reserve(size);
for (size_t i = 0; i < size; ++i) {
v.emplace_back(upb_Array_Get(arr, i).int64_val);
}
child.value = nt::Value::MakeIntegerArray(std::move(v), time);
} else {
child.value = nt::Value::MakeInteger(
upb_Message_GetFieldByDef(msg, field).int64_val, time);
}
child.UpdateFromValue(model, child.path, "");
break;
}
case kUpb_CType_UInt32: {
if (upb_FieldDef_IsRepeated(field)) {
const upb_Array* arr =
upb_Message_GetFieldByDef(msg, field).array_val;
size_t size = upb_Array_Size(arr);
std::vector<int64_t> v;
v.reserve(size);
for (size_t i = 0; i < size; ++i) {
v.emplace_back(upb_Array_Get(arr, i).uint32_val);
}
child.value = nt::Value::MakeIntegerArray(std::move(v), time);
} else {
child.value = nt::Value::MakeInteger(
upb_Message_GetFieldByDef(msg, field).uint32_val, time);
}
child.UpdateFromValue(model, child.path, "");
break;
}
case kUpb_CType_UInt64: {
if (upb_FieldDef_IsRepeated(field)) {
const upb_Array* arr =
upb_Message_GetFieldByDef(msg, field).array_val;
size_t size = upb_Array_Size(arr);
std::vector<int64_t> v;
v.reserve(size);
for (size_t i = 0; i < size; ++i) {
v.emplace_back(upb_Array_Get(arr, i).uint64_val);
}
child.value = nt::Value::MakeIntegerArray(std::move(v), time);
} else {
child.value = nt::Value::MakeInteger(
upb_Message_GetFieldByDef(msg, field).uint64_val, time);
}
child.UpdateFromValue(model, child.path, "");
break;
}
case kUpb_CType_Float: {
if (upb_FieldDef_IsRepeated(field)) {
const upb_Array* arr =
upb_Message_GetFieldByDef(msg, field).array_val;
size_t size = upb_Array_Size(arr);
std::vector<float> v;
v.reserve(size);
for (size_t i = 0; i < size; ++i) {
v.emplace_back(upb_Array_Get(arr, i).float_val);
}
child.value = nt::Value::MakeFloatArray(std::move(v), time);
} else {
child.value = nt::Value::MakeFloat(
upb_Message_GetFieldByDef(msg, field).float_val, time);
}
child.UpdateFromValue(model, child.path, "");
break;
}
case kUpb_CType_Double: {
if (upb_FieldDef_IsRepeated(field)) {
const upb_Array* arr =
upb_Message_GetFieldByDef(msg, field).array_val;
size_t size = upb_Array_Size(arr);
std::vector<double> v;
v.reserve(size);
for (size_t i = 0; i < size; ++i) {
v.emplace_back(upb_Array_Get(arr, i).double_val);
}
child.value = nt::Value::MakeDoubleArray(std::move(v), time);
} else {
child.value = nt::Value::MakeDouble(
upb_Message_GetFieldByDef(msg, field).double_val, time);
}
child.UpdateFromValue(model, child.path, "");
break;
}
case kUpb_CType_Enum: {
const upb_EnumDef* enumDef = upb_FieldDef_EnumSubDef(field);
if (upb_FieldDef_IsRepeated(field)) {
const upb_Array* arr =
upb_Message_GetFieldByDef(msg, field).array_val;
size_t size = upb_Array_Size(arr);
std::vector<std::string> v;
v.reserve(size);
for (size_t i = 0; i < size; ++i) {
int32_t value = upb_Message_GetFieldByDef(msg, field).int32_val;
const upb_EnumValueDef* enumValueDef =
upb_EnumDef_FindValueByNumber(enumDef, value);
if (enumValueDef) {
const char* name = upb_EnumValueDef_Name(enumValueDef);
v.emplace_back(name);
} else {
v.emplace_back(fmt::format("{}", value));
}
}
child.value = nt::Value::MakeStringArray(std::move(v), time);
} else {
int32_t value = upb_Message_GetFieldByDef(msg, field).int32_val;
const upb_EnumValueDef* enumValueDef =
upb_EnumDef_FindValueByNumber(enumDef, value);
if (enumValueDef) {
const char* name = upb_EnumValueDef_Name(enumValueDef);
child.value = nt::Value::MakeString(name, time);
} else {
child.value = nt::Value::MakeString(fmt::format("{}", value), time);
}
}
child.UpdateFromValue(model, child.path, "");
break;
}
case kUpb_CType_Message: {
if (upb_FieldDef_IsRepeated(field)) {
if (child.valueChildrenMap) {
child.valueChildren.clear();
child.valueChildrenMap = false;
}
size_t size = refl->FieldSize(msg, field);
const upb_Array* arr =
upb_Message_GetFieldByDef(msg, field).array_val;
size_t size = upb_Array_Size(arr);
child.valueChildren.resize(size);
unsigned int i = 0;
for (auto&& child2 : child.valueChildren) {
@@ -526,23 +573,24 @@ static void UpdateProtobufValueSource(NetworkTablesModel& model,
child2.name = fmt::format("[{}]", i);
child2.path = fmt::format("{}{}", name, child.name);
}
UpdateProtobufValueSource(model, &child2,
refl->GetRepeatedMessage(msg, field, i),
const upb_Message* submsg = upb_Array_Get(arr, i).msg_val;
const upb_MessageDef* submsgDef = upb_FieldDef_MessageSubDef(field);
UpdateProtobufValueSource(model, &child2, submsg, submsgDef,
child2.path, time); // recurse
++i;
}
} else {
UpdateProtobufValueSource(
model, &child,
refl->GetMessage(msg, field,
model.GetProtobufDatabase().GetMessageFactory()),
child.path, time); // recurse
model, &child, upb_Message_GetFieldByDef(msg, field).msg_val,
upb_FieldDef_MessageSubDef(field), child.path, time); // recurse
}
break;
}
case kUpb_CType_Bytes:
break;
}
}
}
#endif
static void UpdateJsonValueSource(NetworkTablesModel& model,
NetworkTablesModel::ValueSource* out,
@@ -817,13 +865,19 @@ void NetworkTablesModel::ValueSource::UpdateFromValue(
valueChildren.clear();
}
} else if (auto filename = wpi::remove_prefix(typeStr, "proto:")) {
#ifndef NO_PROTOBUF
auto msg = model.m_protoDb.Find(*filename);
if (msg) {
msg->Clear();
const upb_MessageDef* messageDef = upb_DefPool_FindMessageByName(
model.GetProtobufDatabase(), filename->data());
if (messageDef) {
auto raw = value.GetRaw();
if (msg->ParseFromArray(raw.data(), raw.size())) {
UpdateProtobufValueSource(model, this, *msg, name,
const upb_MiniTable* miniTable = upb_MessageDef_MiniTable(messageDef);
upb_Message* message =
upb_Message_New(miniTable, model.GetProtobufArena());
upb_DecodeStatus status = upb_Decode(
reinterpret_cast<const char*>(raw.data()), raw.size(), message,
miniTable, nullptr, 0, model.GetProtobufArena());
if (status == kUpb_DecodeStatus_Ok) {
UpdateProtobufValueSource(model, this, message, messageDef, name,
value.last_change());
} else {
valueChildren.clear();
@@ -831,9 +885,6 @@ void NetworkTablesModel::ValueSource::UpdateFromValue(
} else {
valueChildren.clear();
}
#else
valueChildren.clear();
#endif
} else {
valueChildren.clear();
}
@@ -959,9 +1010,17 @@ void NetworkTablesModel::Update() {
wpi::remove_prefix(entry->info.name, "/.schema/proto:");
entry->value.IsRaw() && filename &&
entry->info.type_str == "proto:FileDescriptorProto") {
#ifndef NO_PROTOBUF
// protobuf descriptor handling
if (!m_protoDb.Add(*filename, entry->value.GetRaw())) {
upb_Status status;
status.ok = true;
auto descriptor = entry->value.GetRaw();
upb_DefPool_AddFile(
m_protoPool,
google_protobuf_FileDescriptorProto_parse(
reinterpret_cast<const char*>(descriptor.data()),
descriptor.size(), m_arena),
&status);
if (!status.ok) {
wpi::print("could not decode protobuf '{}' filename '{}'\n",
entry->info.name, *filename);
} else {
@@ -976,7 +1035,6 @@ void NetworkTablesModel::Update() {
}
}
}
#endif
}
}
}