[wpiutil, ntcore] Add structured data support (#5391)

This adds support for two serialization formats for complex data types:

- Protobuf for complex objects with variable length internals that need forward and backward wire compatibility (lower speed, more flexible)
- Raw struct (ByteBuffer-style) for fixed-length objects (higher speed, less flexible)

Deserialization can be done either by creating a new object (for immutable objects) or overwriting the contents of an existing object (for mutable objects).

Implementing classes should provide inner classes that implement the Protobuf or Struct interface (in Java) or specialize the wpi::Protobuf or wpi::Struct struct (in C++). It is possible for classes to implement both. If the class itself does not implement serialization, it's possible for third parties/users to provide an implementation instead.

Uses the Google protobuf implementation for C++ and the QuickBuffers alternative protobuf implementation for Java.
This commit is contained in:
Peter Johnson
2023-10-19 21:41:47 -07:00
committed by GitHub
parent ecb7cfa9ef
commit cf54d9ccb7
133 changed files with 13506 additions and 90 deletions

View File

@@ -0,0 +1,390 @@
// Copyright (c) FIRST and other WPILib contributors.
// Open Source Software; you can modify and/or share it under the terms of
// the WPILib BSD license file in the root directory of this project.
package edu.wpi.first.util.struct;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.stream.Stream;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
class DynamicStructTest {
@SuppressWarnings("MemberName")
private StructDescriptorDatabase db;
@BeforeEach
public void init() {
db = new StructDescriptorDatabase();
}
@Test
void testEmpty() {
var desc = assertDoesNotThrow(() -> db.add("test", ""));
assertEquals(desc.getName(), "test");
assertEquals(desc.getSchema(), "");
assertTrue(desc.getFields().isEmpty());
assertTrue(desc.isValid());
assertEquals(desc.getSize(), 0);
}
@Test
void testNestedStruct() {
var desc = assertDoesNotThrow(() -> db.add("test", "int32 a"));
assertTrue(desc.isValid());
var desc2 = assertDoesNotThrow(() -> db.add("test2", "test a"));
assertTrue(desc2.isValid());
assertEquals(desc2.getSize(), 4);
}
@Test
void testDelayedValid() {
var desc = assertDoesNotThrow(() -> db.add("test", "foo a"));
assertFalse(desc.isValid());
var desc2 = assertDoesNotThrow(() -> db.add("test2", "foo a[2]"));
assertFalse(desc2.isValid());
var desc3 = assertDoesNotThrow(() -> db.add("foo", "int32 a"));
assertTrue(desc3.isValid());
assertTrue(desc.isValid());
assertEquals(desc.getSize(), 4);
assertTrue(desc2.isValid());
assertEquals(desc2.getSize(), 8);
}
@Test
void testInvalidBitfield() {
assertThrows(
BadSchemaException.class,
() -> db.add("test", "float a:1"),
"field a: type float cannot be bitfield");
assertThrows(
BadSchemaException.class,
() -> db.add("test", "double a:1"),
"field a: type double cannot be bitfield");
assertThrows(
BadSchemaException.class,
() -> db.add("test", "foo a:1"),
"field a: type foo cannot be bitfield");
}
@Test
void testCircularStructReference() {
assertThrows(
BadSchemaException.class,
() -> db.add("test", "test a"),
"field a: recursive struct reference");
}
@Test
void testNestedCircularStructRef() {
assertDoesNotThrow(() -> db.add("test", "foo a"));
assertDoesNotThrow(() -> db.add("foo", "bar a"));
assertThrows(
BadSchemaException.class,
() -> db.add("bar", "test a"),
"circular struct reference: bar <- foo <- test");
// ok
var desc = assertDoesNotThrow(() -> db.add("baz", "bar a"));
assertFalse(desc.isValid());
}
@Test
void testNestedCircularStructRef2() {
assertDoesNotThrow(() -> db.add("test", "foo a"));
assertDoesNotThrow(() -> db.add("bar", "test a"));
assertThrows(
BadSchemaException.class,
() -> db.add("foo", "bar a"),
"circular struct reference: foo <- test <- bar");
}
@Test
void testBitfieldBasic() {
var desc = assertDoesNotThrow(() -> db.add("test", "int32 a:2; uint32 b:30"));
assertEquals(desc.getSize(), 4);
var fields = desc.getFields();
assertEquals(fields.size(), 2);
var field = fields.get(0);
assertEquals(field.getBitWidth(), 2);
assertEquals(field.getBitShift(), 0);
assertEquals(field.getBitMask(), 0x3);
assertEquals(field.getOffset(), 0);
assertEquals(field.getSize(), 4);
field = fields.get(1);
assertEquals(field.getBitWidth(), 30);
assertEquals(field.getBitShift(), 2);
assertEquals(field.getBitMask(), 0x3fffffff);
assertEquals(field.getOffset(), 0);
assertEquals(field.getSize(), 4);
}
@Test
void testBitfieldDiffType() {
var desc = assertDoesNotThrow(() -> db.add("test", "int32 a:2; int16 b:2"));
assertEquals(desc.getSize(), 6);
var fields = desc.getFields();
assertEquals(fields.size(), 2);
var field = fields.get(0);
assertEquals(field.getBitWidth(), 2);
assertEquals(field.getBitShift(), 0);
assertEquals(field.getBitMask(), 0x3);
assertEquals(field.getOffset(), 0);
assertEquals(field.getSize(), 4);
field = fields.get(1);
assertEquals(field.getBitWidth(), 2);
assertEquals(field.getBitShift(), 0);
assertEquals(field.getBitMask(), 0x3);
assertEquals(field.getOffset(), 4);
assertEquals(field.getSize(), 2);
}
@Test
void testBitfieldOverflow() {
var desc = assertDoesNotThrow(() -> db.add("test", "int8 a:4; int8 b:5"));
assertEquals(desc.getSize(), 2);
var fields = desc.getFields();
assertEquals(fields.size(), 2);
var field = fields.get(0);
assertEquals(field.getBitWidth(), 4);
assertEquals(field.getBitShift(), 0);
assertEquals(field.getBitMask(), 0xf);
assertEquals(field.getOffset(), 0);
assertEquals(field.getSize(), 1);
field = fields.get(1);
assertEquals(field.getBitWidth(), 5);
assertEquals(field.getBitMask(), 0x1f);
assertEquals(field.getBitShift(), 0);
assertEquals(field.getOffset(), 1);
assertEquals(field.getSize(), 1);
}
@Test
void testBitfieldBoolBegin8() {
var desc = assertDoesNotThrow(() -> db.add("test", "bool a:1; int8 b:5"));
assertEquals(desc.getSize(), 1);
var fields = desc.getFields();
assertEquals(fields.size(), 2);
var field = fields.get(0);
assertEquals(field.getBitWidth(), 1);
assertEquals(field.getBitShift(), 0);
assertEquals(field.getBitMask(), 0x1);
assertEquals(field.getOffset(), 0);
assertEquals(field.getSize(), 1);
field = fields.get(1);
assertEquals(field.getBitWidth(), 5);
assertEquals(field.getBitMask(), 0x1f);
assertEquals(field.getBitShift(), 1);
assertEquals(field.getOffset(), 0);
assertEquals(field.getSize(), 1);
}
@Test
void testBitfieldBoolBegin16() {
var desc = assertDoesNotThrow(() -> db.add("test", "bool a:1; int16 b:5"));
assertEquals(desc.getSize(), 3);
var fields = desc.getFields();
assertEquals(fields.size(), 2);
var field = fields.get(0);
assertEquals(field.getBitWidth(), 1);
assertEquals(field.getBitShift(), 0);
assertEquals(field.getBitMask(), 0x1);
assertEquals(field.getOffset(), 0);
assertEquals(field.getSize(), 1);
field = fields.get(1);
assertEquals(field.getBitWidth(), 5);
assertEquals(field.getBitMask(), 0x1f);
assertEquals(field.getBitShift(), 0);
assertEquals(field.getOffset(), 1);
assertEquals(field.getSize(), 2);
}
@Test
void testBitfieldBoolMid() {
var desc =
assertDoesNotThrow(() -> db.add("test", "int16 a:2; bool b:1; bool c:1; uint16 d:5"));
assertEquals(desc.getSize(), 2);
var fields = desc.getFields();
assertEquals(fields.size(), 4);
var field = fields.get(0);
assertEquals(field.getBitWidth(), 2);
assertEquals(field.getBitShift(), 0);
assertEquals(field.getBitMask(), 0x3);
assertEquals(field.getOffset(), 0);
assertEquals(field.getSize(), 2);
field = fields.get(1);
assertEquals(field.getBitWidth(), 1);
assertEquals(field.getBitMask(), 0x1);
assertEquals(field.getBitShift(), 2);
assertEquals(field.getOffset(), 0);
assertEquals(field.getSize(), 2);
field = fields.get(2);
assertEquals(field.getBitWidth(), 1);
assertEquals(field.getBitMask(), 0x1);
assertEquals(field.getBitShift(), 3);
assertEquals(field.getOffset(), 0);
assertEquals(field.getSize(), 2);
field = fields.get(3);
assertEquals(field.getBitWidth(), 5);
assertEquals(field.getBitMask(), 0x1f);
assertEquals(field.getBitShift(), 4);
assertEquals(field.getOffset(), 0);
assertEquals(field.getSize(), 2);
}
@Test
void testBitfieldBoolEnd() {
var desc = assertDoesNotThrow(() -> db.add("test", "int16 a:15; bool b:1"));
assertEquals(desc.getSize(), 2);
var fields = desc.getFields();
assertEquals(fields.size(), 2);
var field = fields.get(0);
assertEquals(field.getBitWidth(), 15);
assertEquals(field.getBitShift(), 0);
assertEquals(field.getBitMask(), 0x7fff);
assertEquals(field.getOffset(), 0);
assertEquals(field.getSize(), 2);
field = fields.get(1);
assertEquals(field.getBitWidth(), 1);
assertEquals(field.getBitMask(), 0x1);
assertEquals(field.getBitShift(), 15);
assertEquals(field.getOffset(), 0);
assertEquals(field.getSize(), 2);
}
@Test
void testBitfieldBoolEnd2() {
var desc = assertDoesNotThrow(() -> db.add("test", "int16 a:16; bool b:1"));
assertEquals(desc.getSize(), 3);
var fields = desc.getFields();
assertEquals(fields.size(), 2);
var field = fields.get(0);
assertEquals(field.getBitWidth(), 16);
assertEquals(field.getBitShift(), 0);
assertEquals(field.getBitMask(), 0xffff);
assertEquals(field.getOffset(), 0);
assertEquals(field.getSize(), 2);
field = fields.get(1);
assertEquals(field.getBitWidth(), 1);
assertEquals(field.getBitMask(), 0x1);
assertEquals(field.getBitShift(), 0);
assertEquals(field.getOffset(), 2);
assertEquals(field.getSize(), 1);
}
@Test
void testBitfieldBoolWrongSize() {
assertThrows(
BadSchemaException.class,
() -> db.add("test", "bool a:2"),
"field a: bit width must be 1 for bool type");
}
@Test
void testBitfieldTooBig() {
assertThrows(
BadSchemaException.class,
() -> db.add("test", "int16 a:17"),
"field a: bit width 17 exceeds type size");
}
@Test
void testDuplicateFieldName() {
assertThrows(
BadSchemaException.class,
() -> db.add("test", "int16 a; int8 a"),
"field a: duplicate field name");
}
private static Stream<Arguments> provideSimpleTestParams() {
return Stream.of(
Arguments.of("bool a", 1, StructFieldType.kBool, false, false, 8, 0xff),
Arguments.of("char a", 1, StructFieldType.kChar, false, false, 8, 0xff),
Arguments.of("int8 a", 1, StructFieldType.kInt8, true, false, 8, 0xff),
Arguments.of("int16 a", 2, StructFieldType.kInt16, true, false, 16, 0xffff),
Arguments.of("int32 a", 4, StructFieldType.kInt32, true, false, 32, 0xffffffffL),
Arguments.of("int64 a", 8, StructFieldType.kInt64, true, false, 64, -1),
Arguments.of("uint8 a", 1, StructFieldType.kUint8, false, true, 8, 0xff),
Arguments.of("uint16 a", 2, StructFieldType.kUint16, false, true, 16, 0xffff),
Arguments.of("uint32 a", 4, StructFieldType.kUint32, false, true, 32, 0xffffffffL),
Arguments.of("uint64 a", 8, StructFieldType.kUint64, false, true, 64, -1),
Arguments.of("float a", 4, StructFieldType.kFloat, false, false, 32, 0xffffffffL),
Arguments.of("float32 a", 4, StructFieldType.kFloat, false, false, 32, 0xffffffffL),
Arguments.of("double a", 8, StructFieldType.kDouble, false, false, 64, -1),
Arguments.of("float64 a", 8, StructFieldType.kDouble, false, false, 64, -1),
Arguments.of("foo a", 0, StructFieldType.kStruct, false, false, 0, 0));
}
@ParameterizedTest
@MethodSource("provideSimpleTestParams")
void testStandardCheck(
String schema,
int size,
StructFieldType type,
boolean isInt,
boolean isUint,
int bitWidth,
long bitMask) {
var desc = assertDoesNotThrow(() -> db.add("test", schema));
assertEquals(desc.getName(), "test");
assertEquals(desc.getSchema(), schema);
var fields = desc.getFields();
assertEquals(fields.size(), 1);
var field = fields.get(0);
assertEquals(field.getParent(), desc);
assertEquals(field.getName(), "a");
assertEquals(field.isInt(), isInt);
assertEquals(field.isUint(), isUint);
assertFalse(field.isArray());
if (type != StructFieldType.kStruct) {
assertTrue(desc.isValid());
assertEquals(desc.getSize(), size);
assertEquals(field.getSize(), size);
assertEquals(field.getBitWidth(), bitWidth);
assertEquals(field.getBitMask(), bitMask);
} else {
assertFalse(desc.isValid());
assertNotNull(field.getStruct());
}
}
@ParameterizedTest
@MethodSource("provideSimpleTestParams")
void testStandardArray(
String schema,
int size,
StructFieldType type,
boolean isInt,
boolean isUint,
int bitWidth,
long bitMask) {
var desc = assertDoesNotThrow(() -> db.add("test", schema + "[2]"));
assertEquals(desc.getName(), "test");
assertEquals(desc.getSchema(), schema + "[2]");
var fields = desc.getFields();
assertEquals(fields.size(), 1);
var field = fields.get(0);
assertEquals(field.getParent(), desc);
assertEquals(field.getName(), "a");
assertEquals(field.isInt(), isInt);
assertEquals(field.isUint(), isUint);
assertTrue(field.isArray());
assertEquals(field.getArraySize(), 2);
if (type != StructFieldType.kStruct) {
assertTrue(desc.isValid());
assertEquals(desc.getSize(), size * 2);
} else {
assertFalse(desc.isValid());
assertNotNull(field.getStruct());
}
}
}

View File

@@ -0,0 +1,212 @@
// Copyright (c) FIRST and other WPILib contributors.
// Open Source Software; you can modify and/or share it under the terms of
// the WPILib BSD license file in the root directory of this project.
package edu.wpi.first.util.struct.parser;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.Test;
class ParserTest {
@Test
void testEmpty() {
Parser p = new Parser("");
ParsedSchema schema = assertDoesNotThrow(() -> p.parse());
assertTrue(schema.declarations.isEmpty());
}
@Test
void testEmptySemicolon() {
Parser p = new Parser(";");
ParsedSchema schema = assertDoesNotThrow(() -> p.parse());
assertTrue(schema.declarations.isEmpty());
}
@Test
void testSimple() {
Parser p = new Parser("int32 a");
ParsedSchema schema = assertDoesNotThrow(() -> p.parse());
assertEquals(schema.declarations.size(), 1);
var decl = schema.declarations.get(0);
assertEquals(decl.typeString, "int32");
assertEquals(decl.name, "a");
assertEquals(decl.arraySize, 1);
}
@Test
void testSimpleTrailingSemi() {
Parser p = new Parser("int32 a;");
ParsedSchema schema = assertDoesNotThrow(() -> p.parse());
assertEquals(schema.declarations.size(), 1);
}
@Test
void testArray() {
Parser p = new Parser("int32 a[2]");
ParsedSchema schema = assertDoesNotThrow(() -> p.parse());
assertEquals(schema.declarations.size(), 1);
var decl = schema.declarations.get(0);
assertEquals(decl.typeString, "int32");
assertEquals(decl.name, "a");
assertEquals(decl.arraySize, 2);
}
@Test
void testArrayTrailingSemi() {
Parser p = new Parser("int32 a[2];");
ParsedSchema schema = assertDoesNotThrow(() -> p.parse());
assertEquals(schema.declarations.size(), 1);
}
@Test
void testBitfield() {
Parser p = new Parser("int32 a:2");
ParsedSchema schema = assertDoesNotThrow(() -> p.parse());
assertEquals(schema.declarations.size(), 1);
var decl = schema.declarations.get(0);
assertEquals(decl.typeString, "int32");
assertEquals(decl.name, "a");
assertEquals(decl.bitWidth, 2);
}
@Test
void testBitfieldTrailingSemi() {
Parser p = new Parser("int32 a:2;");
ParsedSchema schema = assertDoesNotThrow(() -> p.parse());
assertEquals(schema.declarations.size(), 1);
}
@Test
void testEnumKeyword() {
Parser p = new Parser("enum {x=1} int32 a;");
ParsedSchema schema = assertDoesNotThrow(() -> p.parse());
assertEquals(schema.declarations.size(), 1);
var decl = schema.declarations.get(0);
assertEquals(decl.typeString, "int32");
assertEquals(decl.name, "a");
assertEquals(decl.enumValues.size(), 1);
assertEquals(decl.enumValues.get("x"), 1);
}
@Test
void testEnumNoKeyword() {
Parser p = new Parser("{x=1} int32 a;");
ParsedSchema schema = assertDoesNotThrow(() -> p.parse());
assertEquals(schema.declarations.size(), 1);
var decl = schema.declarations.get(0);
assertEquals(decl.typeString, "int32");
assertEquals(decl.name, "a");
assertEquals(decl.enumValues.size(), 1);
assertEquals(decl.enumValues.get("x"), 1);
}
@Test
void testEnumNoValues() {
Parser p = new Parser("{} int32 a;");
ParsedSchema schema = assertDoesNotThrow(() -> p.parse());
assertEquals(schema.declarations.size(), 1);
var decl = schema.declarations.get(0);
assertEquals(decl.typeString, "int32");
assertEquals(decl.name, "a");
assertTrue(decl.enumValues.isEmpty());
}
@Test
void testEnumMultipleValues() {
Parser p = new Parser("{x=1,y=-2} int32 a;");
ParsedSchema schema = assertDoesNotThrow(() -> p.parse());
assertEquals(schema.declarations.size(), 1);
var decl = schema.declarations.get(0);
assertEquals(decl.typeString, "int32");
assertEquals(decl.name, "a");
assertEquals(decl.enumValues.size(), 2);
assertEquals(decl.enumValues.get("x"), 1);
assertEquals(decl.enumValues.get("y"), -2);
}
@Test
void testEnumTrailingComma() {
Parser p = new Parser("{x=1,y=2,} int32 a;");
ParsedSchema schema = assertDoesNotThrow(() -> p.parse());
assertEquals(schema.declarations.size(), 1);
var decl = schema.declarations.get(0);
assertEquals(decl.typeString, "int32");
assertEquals(decl.name, "a");
assertEquals(decl.enumValues.size(), 2);
assertEquals(decl.enumValues.get("x"), 1);
assertEquals(decl.enumValues.get("y"), 2);
}
@Test
void testMultipleNoTrailingSemi() {
Parser p = new Parser("int32 a; int16 b");
ParsedSchema schema = assertDoesNotThrow(() -> p.parse());
assertEquals(schema.declarations.size(), 2);
assertEquals(schema.declarations.get(0).typeString, "int32");
assertEquals(schema.declarations.get(0).name, "a");
assertEquals(schema.declarations.get(1).typeString, "int16");
assertEquals(schema.declarations.get(1).name, "b");
}
@Test
void testErrBitfieldArray() {
Parser p = new Parser("int32 a[1]:2");
assertThrows(ParseException.class, () -> p.parse(), "10: expected ';', got ':'");
}
@Test
void testErrNoArrayValue() {
Parser p = new Parser("int32 a[]");
assertThrows(ParseException.class, () -> p.parse(), "8: expected integer, got ']'");
}
@Test
void testErrNoBitfieldValue() {
Parser p = new Parser("int32 a:");
assertThrows(ParseException.class, () -> p.parse(), "8: expected integer, got ''");
}
@Test
void testErrNoNameArray() {
Parser p = new Parser("int32 [2]");
assertThrows(ParseException.class, () -> p.parse(), "6: expected identifier, got '['");
}
@Test
void testErrNoNameBitField() {
Parser p = new Parser("int32 :2");
assertThrows(ParseException.class, () -> p.parse(), "6: expected identifier, got ':'");
}
@Test
void testNegativeBitField() {
Parser p = new Parser("int32 a:-1");
assertThrows(
ParseException.class, () -> p.parse(), "8: bitfield width '-1' is not a positive integer");
}
@Test
void testNegativeArraySize() {
Parser p = new Parser("int32 a[-1]");
assertThrows(
ParseException.class, () -> p.parse(), "8: array size '-1' is not a positive integer");
}
@Test
void testZeroBitField() {
Parser p = new Parser("int32 a:0");
assertThrows(
ParseException.class, () -> p.parse(), "8: bitfield width '0' is not a positive integer");
}
@Test
void testZeroArraySize() {
Parser p = new Parser("int32 a[0]");
assertThrows(
ParseException.class, () -> p.parse(), "8: array size '0' is not a positive integer");
}
}