[datalog] Move all DataLog functionality to new datalog library (#7641)

Currently the major DataLog backend API (reading and writing) is split between wpiutil and glass. In the interest of allowing code that wants to use these APIs to not need to link to glass and declutter wpiutil, all of those APIs are moved to a new library named "datalog".

Signed-off-by: Jade Turner <spacey-sooty@proton.me>
Co-authored-by: Jade Turner <spacey-sooty@proton.me>
Co-authored-by: Gold856 <117957790+Gold856@users.noreply.github.com>
This commit is contained in:
DeltaDizzy
2025-02-19 23:08:17 -06:00
committed by GitHub
parent ac1705ae2b
commit da47f06d70
99 changed files with 778 additions and 330 deletions

View File

@@ -0,0 +1,354 @@
#! /usr/bin/env python3
# Copyright (c) FIRST and other WPILib contributors.
# Open Source Software; you can modify and/or share it under the terms of
# the WPILib BSD license file in the root directory of this project.
import array
import struct
from typing import List, SupportsBytes
import msgpack
__all__ = ["StartRecordData", "MetadataRecordData", "DataLogRecord", "DataLogReader"]
floatStruct = struct.Struct("<f")
doubleStruct = struct.Struct("<d")
kControlStart = 0
kControlFinish = 1
kControlSetMetadata = 2
class StartRecordData:
"""Data contained in a start control record as created by DataLog.start() when
writing the log. This can be read by calling DataLogRecord.getStartData().
entry: Entry ID; this will be used for this entry in future records.
name: Entry name.
type: Type of the stored data for this entry, as a string, e.g. "double".
metadata: Initial metadata.
"""
def __init__(self, entry: int, name: str, type: str, metadata: str):
self.entry = entry
self.name = name
self.type = type
self.metadata = metadata
class MetadataRecordData:
"""Data contained in a set metadata control record as created by
DataLog.setMetadata(). This can be read by calling
DataLogRecord.getSetMetadataData().
entry: Entry ID.
metadata: New metadata for the entry.
"""
def __init__(self, entry: int, metadata: str):
self.entry = entry
self.metadata = metadata
class DataLogRecord:
"""A record in the data log. May represent either a control record
(entry == 0) or a data record."""
def __init__(self, entry: int, timestamp: int, data: SupportsBytes):
self.entry = entry
self.timestamp = timestamp
self.data = data
def isControl(self) -> bool:
return self.entry == 0
def _getControlType(self) -> int:
return self.data[0]
def isStart(self) -> bool:
return (
self.entry == 0
and len(self.data) >= 17
and self._getControlType() == kControlStart
)
def isFinish(self) -> bool:
return (
self.entry == 0
and len(self.data) == 5
and self._getControlType() == kControlFinish
)
def isSetMetadata(self) -> bool:
return (
self.entry == 0
and len(self.data) >= 9
and self._getControlType() == kControlSetMetadata
)
def getStartData(self) -> StartRecordData:
if not self.isStart():
raise TypeError("not a start record")
entry = int.from_bytes(self.data[1:5], byteorder="little", signed=False)
name, pos = self._readInnerString(5)
type, pos = self._readInnerString(pos)
metadata = self._readInnerString(pos)[0]
return StartRecordData(entry, name, type, metadata)
def getFinishEntry(self) -> int:
if not self.isFinish():
raise TypeError("not a finish record")
return int.from_bytes(self.data[1:5], byteorder="little", signed=False)
def getSetMetadataData(self) -> MetadataRecordData:
if not self.isSetMetadata():
raise TypeError("not a finish record")
entry = int.from_bytes(self.data[1:5], byteorder="little", signed=False)
metadata = self._readInnerString(5)[0]
return MetadataRecordData(entry, metadata)
def getBoolean(self) -> bool:
if len(self.data) != 1:
raise TypeError("not a boolean")
return self.data[0] != 0
def getInteger(self) -> int:
if len(self.data) != 8:
raise TypeError("not an integer")
return int.from_bytes(self.data, byteorder="little", signed=True)
def getFloat(self) -> float:
if len(self.data) != 4:
raise TypeError("not a float")
return floatStruct.unpack(self.data)[0]
def getDouble(self) -> float:
if len(self.data) != 8:
raise TypeError("not a double")
return doubleStruct.unpack(self.data)[0]
def getString(self) -> str:
return str(self.data, encoding="utf-8")
def getMsgPack(self):
return msgpack.unpackb(self.data)
def getBooleanArray(self) -> List[bool]:
return [x != 0 for x in self.data]
def getIntegerArray(self) -> array.array:
if (len(self.data) % 8) != 0:
raise TypeError("not an integer array")
arr = array.array("l")
arr.frombytes(self.data)
return arr
def getFloatArray(self) -> array.array:
if (len(self.data) % 4) != 0:
raise TypeError("not a float array")
arr = array.array("f")
arr.frombytes(self.data)
return arr
def getDoubleArray(self) -> array.array:
if (len(self.data) % 8) != 0:
raise TypeError("not a double array")
arr = array.array("d")
arr.frombytes(self.data)
return arr
def getStringArray(self) -> List[str]:
size = int.from_bytes(self.data[:4], byteorder="little", signed=False)
if size > ((len(self.data) - 4) / 4):
raise TypeError("not a string array")
arr = []
pos = 4
for _ in range(size):
val, pos = self._readInnerString(pos)
arr.append(val)
return arr
def _readInnerString(self, pos: int) -> tuple[str, int]:
size = int.from_bytes(
self.data[pos : pos + 4], byteorder="little", signed=False
)
end = pos + 4 + size
if end > len(self.data):
raise TypeError("invalid string size")
return str(self.data[pos + 4 : end], encoding="utf-8"), end
class DataLogIterator:
"""DataLogReader iterator."""
def __init__(self, buf: SupportsBytes, pos: int):
self.buf = buf
self.pos = pos
def __iter__(self):
return self
def _readVarInt(self, pos: int, len: int) -> int:
val = 0
for i in range(len):
val |= self.buf[pos + i] << (i * 8)
return val
def __next__(self) -> DataLogRecord:
if len(self.buf) < (self.pos + 4):
raise StopIteration
entryLen = (self.buf[self.pos] & 0x3) + 1
sizeLen = ((self.buf[self.pos] >> 2) & 0x3) + 1
timestampLen = ((self.buf[self.pos] >> 4) & 0x7) + 1
headerLen = 1 + entryLen + sizeLen + timestampLen
if len(self.buf) < (self.pos + headerLen):
raise StopIteration
entry = self._readVarInt(self.pos + 1, entryLen)
size = self._readVarInt(self.pos + 1 + entryLen, sizeLen)
timestamp = self._readVarInt(self.pos + 1 + entryLen + sizeLen, timestampLen)
if len(self.buf) < (self.pos + headerLen + size):
raise StopIteration
record = DataLogRecord(
entry,
timestamp,
self.buf[self.pos + headerLen : self.pos + headerLen + size],
)
self.pos += headerLen + size
return record
class DataLogReader:
"""Data log reader (reads logs written by the DataLog class)."""
def __init__(self, buf: SupportsBytes):
self.buf = buf
def __bool__(self):
return self.isValid()
def isValid(self) -> bool:
"""Returns true if the data log is valid (e.g. has a valid header)."""
return (
len(self.buf) >= 12
and self.buf[:6] == b"WPILOG"
and self.getVersion() >= 0x0100
)
def getVersion(self) -> int:
"""Gets the data log version. Returns 0 if data log is invalid.
@return Version number; most significant byte is major, least significant is
minor (so version 1.0 will be 0x0100)"""
if len(self.buf) < 12:
return 0
return int.from_bytes(self.buf[6:8], byteorder="little", signed=False)
def getExtraHeader(self) -> str:
"""Gets the extra header data.
@return Extra header data
"""
if len(self.buf) < 12:
return ""
size = int.from_bytes(self.buf[8:12], byteorder="little", signed=False)
return str(self.buf[12 : 12 + size], encoding="utf-8")
def __iter__(self) -> DataLogIterator:
extraHeaderSize = int.from_bytes(
self.buf[8:12], byteorder="little", signed=False
)
return DataLogIterator(self.buf, 12 + extraHeaderSize)
if __name__ == "__main__":
import mmap
import sys
from datetime import datetime
if len(sys.argv) != 2:
print("Usage: datalog.py <file>", file=sys.stderr)
sys.exit(1)
with open(sys.argv[1], "r") as f:
mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
reader = DataLogReader(mm)
if not reader:
print("not a log file", file=sys.stderr)
sys.exit(1)
entries = {}
for record in reader:
timestamp = record.timestamp / 1000000
if record.isStart():
try:
data = record.getStartData()
print(
f"Start({data.entry}, name='{data.name}', type='{data.type}', metadata='{data.metadata}') [{timestamp}]"
)
if data.entry in entries:
print("...DUPLICATE entry ID, overriding")
entries[data.entry] = data
except TypeError:
print("Start(INVALID)")
elif record.isFinish():
try:
entry = record.getFinishEntry()
print(f"Finish({entry}) [{timestamp}]")
if entry not in entries:
print("...ID not found")
else:
del entries[entry]
except TypeError:
print("Finish(INVALID)")
elif record.isSetMetadata():
try:
data = record.getSetMetadataData()
print(f"SetMetadata({data.entry}, '{data.metadata}') [{timestamp}]")
if data.entry not in entries:
print("...ID not found")
except TypeError:
print("SetMetadata(INVALID)")
elif record.isControl():
print("Unrecognized control record")
else:
print(f"Data({record.entry}, size={len(record.data)}) ", end="")
entry = entries.get(record.entry)
if entry is None:
print("<ID not found>")
continue
print(f"<name='{entry.name}', type='{entry.type}'> [{timestamp}]")
try:
# handle systemTime specially
if entry.name == "systemTime" and entry.type == "int64":
dt = datetime.fromtimestamp(record.getInteger() / 1000000)
print(" {:%Y-%m-%d %H:%M:%S.%f}".format(dt))
continue
if entry.type == "double":
print(f" {record.getDouble()}")
elif entry.type == "int64":
print(f" {record.getInteger()}")
elif entry.type in ("string", "json"):
print(f" '{record.getString()}'")
elif entry.type == "msgpack":
print(f" '{record.getMsgPack()}'")
elif entry.type == "boolean":
print(f" {record.getBoolean()}")
elif entry.type == "boolean[]":
arr = record.getBooleanArray()
print(f" {arr}")
elif entry.type == "double[]":
arr = record.getDoubleArray()
print(f" {arr}")
elif entry.type == "float[]":
arr = record.getFloatArray()
print(f" {arr}")
elif entry.type == "int64[]":
arr = record.getIntegerArray()
print(f" {arr}")
elif entry.type == "string[]":
arr = record.getStringArray()
print(f" {arr}")
except TypeError:
print(" invalid")

View File

@@ -0,0 +1,167 @@
// Copyright (c) FIRST and other WPILib contributors.
// Open Source Software; you can modify and/or share it under the terms of
// the WPILib BSD license file in the root directory of this project.
#include <ctime>
#include <utility>
#include <vector>
#include <fmt/chrono.h>
#include <fmt/format.h>
#include <fmt/ranges.h>
#include <wpi/DenseMap.h>
#include <wpi/MemoryBuffer.h>
#include <wpi/print.h>
#include "wpi/datalog/DataLogReader.h"
int main(int argc, const char** argv) {
if (argc != 2) {
wpi::print(stderr, "Usage: printlog <file>\n");
return EXIT_FAILURE;
}
auto fileBuffer = wpi::MemoryBuffer::GetFile(argv[1]);
if (!fileBuffer) {
wpi::print(stderr, "could not open file: {}\n",
fileBuffer.error().message());
return EXIT_FAILURE;
}
wpi::log::DataLogReader reader{std::move(*fileBuffer)};
if (!reader) {
wpi::print(stderr, "not a log file\n");
return EXIT_FAILURE;
}
wpi::DenseMap<int, wpi::log::StartRecordData> entries;
for (auto&& record : reader) {
if (record.IsStart()) {
wpi::log::StartRecordData data;
if (record.GetStartData(&data)) {
wpi::print("Start({}, name='{}', type='{}', metadata='{}') [{}]\n",
data.entry, data.name, data.type, data.metadata,
record.GetTimestamp() / 1000000.0);
if (entries.find(data.entry) != entries.end()) {
wpi::print("...DUPLICATE entry ID, overriding\n");
}
entries[data.entry] = data;
} else {
wpi::print("Start(INVALID)\n");
}
} else if (record.IsFinish()) {
int entry;
if (record.GetFinishEntry(&entry)) {
wpi::print("Finish({}) [{}]\n", entry,
record.GetTimestamp() / 1000000.0);
auto it = entries.find(entry);
if (it == entries.end()) {
wpi::print("...ID not found\n");
} else {
entries.erase(it);
}
} else {
wpi::print("Finish(INVALID)\n");
}
} else if (record.IsSetMetadata()) {
wpi::log::MetadataRecordData data;
if (record.GetSetMetadataData(&data)) {
wpi::print("SetMetadata({}, '{}') [{}]\n", data.entry, data.metadata,
record.GetTimestamp() / 1000000.0);
auto it = entries.find(data.entry);
if (it == entries.end()) {
wpi::print("...ID not found\n");
} else {
it->second.metadata = data.metadata;
}
} else {
wpi::print("SetMetadata(INVALID)\n");
}
} else if (record.IsControl()) {
wpi::print("Unrecognized control record\n");
} else {
wpi::print("Data({}, size={}) ", record.GetEntry(), record.GetSize());
auto entry = entries.find(record.GetEntry());
if (entry == entries.end()) {
wpi::print("<ID not found>\n");
continue;
}
wpi::print("<name='{}', type='{}'> [{}]\n", entry->second.name,
entry->second.type, record.GetTimestamp() / 1000000.0);
// handle systemTime specially
if (entry->second.name == "systemTime" && entry->second.type == "int64") {
int64_t val;
if (record.GetInteger(&val)) {
std::time_t timeval = val / 1000000;
wpi::print(" {:%Y-%m-%d %H:%M:%S}.{:06}\n",
*std::localtime(&timeval), val % 1000000);
} else {
wpi::print(" invalid\n");
}
continue;
}
if (entry->second.type == "double") {
double val;
if (record.GetDouble(&val)) {
wpi::print(" {}\n", val);
} else {
wpi::print(" invalid\n");
}
} else if (entry->second.type == "int64") {
int64_t val;
if (record.GetInteger(&val)) {
wpi::print(" {}\n", val);
} else {
wpi::print(" invalid\n");
}
} else if (entry->second.type == "string" ||
entry->second.type == "json") {
std::string_view val;
record.GetString(&val);
wpi::print(" '{}'\n", val);
} else if (entry->second.type == "boolean") {
bool val;
if (record.GetBoolean(&val)) {
wpi::print(" {}\n", val);
} else {
wpi::print(" invalid\n");
}
} else if (entry->second.type == "boolean[]") {
std::vector<int> val;
if (record.GetBooleanArray(&val)) {
wpi::print(" {}\n", fmt::join(val, ", "));
} else {
wpi::print(" invalid\n");
}
} else if (entry->second.type == "double[]") {
std::vector<double> val;
if (record.GetDoubleArray(&val)) {
wpi::print(" {}\n", fmt::join(val, ", "));
} else {
wpi::print(" invalid\n");
}
} else if (entry->second.type == "float[]") {
std::vector<float> val;
if (record.GetFloatArray(&val)) {
wpi::print(" {}\n", fmt::join(val, ", "));
} else {
wpi::print(" invalid\n");
}
} else if (entry->second.type == "int64[]") {
std::vector<int64_t> val;
if (record.GetIntegerArray(&val)) {
wpi::print(" {}\n", fmt::join(val, ", "));
} else {
wpi::print(" invalid\n");
}
} else if (entry->second.type == "string[]") {
std::vector<std::string_view> val;
if (record.GetStringArray(&val)) {
wpi::print(" {}\n", fmt::join(val, ", "));
} else {
wpi::print(" invalid\n");
}
}
}
}
}

View File

@@ -0,0 +1,83 @@
// Copyright (c) FIRST and other WPILib contributors.
// Open Source Software; you can modify and/or share it under the terms of
// the WPILib BSD license file in the root directory of this project.
#include <chrono>
#include <numeric>
#include <string>
#include <utility>
#include <vector>
#include <wpi/print.h>
#include "wpi/datalog/DataLogBackgroundWriter.h"
int main(int argc, char** argv) {
using std::chrono::duration_cast;
using std::chrono::high_resolution_clock;
using std::chrono::microseconds;
int kNumRuns = 10;
if (argc == 2) {
kNumRuns = std::stoi(argv[1]);
}
wpi::log::DataLogBackgroundWriter log;
log.SetFilename("test.wpilog");
auto testVec =
std::vector<std::pair<std::string, void (*)(wpi::log::DataLog&)>>();
testVec.push_back({"50 double append", [](auto& log) {
wpi::log::DoubleLogEntry entry{log, "fifty", 1};
for (int i = 0; i < 50; ++i) {
entry.Append(1.3 * i, 20000 * i);
}
}});
#if 0
testVec.push_back({"500k double append", [](auto& log) {
wpi::log::DoubleLogEntry entry{log, "500k", 1};
for (uint64_t i = 0; i < 500000; ++i) {
entry.Append(1.3 * i, 20000 * i);
}
}});
#endif
testVec.push_back({"50 string append", [](auto& log) {
wpi::log::StringLogEntry entry{log, "string", 1};
for (int i = 0; i < 50; ++i) {
entry.Append("hello", 20000 * i);
}
}});
testVec.push_back({"Double array append", [](auto& log) {
wpi::log::DoubleArrayLogEntry entry{log, "double_array",
1};
entry.Append({1, 2, 3}, 20000);
entry.Append({4, 5}, 30000);
}});
testVec.push_back({"String array append", [](auto& log) {
wpi::log::StringArrayLogEntry entry{log, "string_array",
1};
entry.Append({"Hello", "World"}, 20000);
entry.Append({"This", "Is", "Fun"}, 30000);
}});
for (const auto& [name, fn] : testVec) {
auto resVec = std::vector<microseconds::rep>();
wpi::print("{}: ", name);
for (int i = 0; i < kNumRuns; ++i) {
auto start = high_resolution_clock::now();
fn(log);
auto stop = high_resolution_clock::now();
resVec.push_back(duration_cast<microseconds>(stop - start).count());
}
wpi::print("{}us\n",
std::accumulate(resVec.begin(), resVec.end(), 0) / kNumRuns);
}
return EXIT_SUCCESS;
}