Team2890/allwpilib
4
0
Fork 0
mirror of https://github.com/wpilibsuite/allwpilib synced 2026-06-19 00:41:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

[wpiutil] Split DataLog background writer into different class (#6590)

Browse Source 
DataLog is now a base class, with DataLogBackgroundWriter being the
background thread version and DataLogWriter being a non-threaded version.

Also split the C header into a separate file to make it more wpiformat friendly.
This commit is contained in:
Peter Johnson
2024-05-12 14:09:43 -07:00
committed by GitHub
parent 305a0657e2
commit 178fe99f12
20 changed files with 1588 additions and 1089 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
ntcoreffi/src/main/native/cpp/DataLogManager.cpp
View File

@@ -7,11 +7,13 @@
#include <algorithm>
#include <ctime>
#include <random>
#include <string>
#include <string_view>
#include <fmt/chrono.h>
#include <fmt/format.h>
#include <networktables/NetworkTableInstance.h>
#include <wpi/DataLog.h>
#include <wpi/DataLogBackgroundWriter.h>
#include <wpi/SafeThread.h>
#include <wpi/StringExtras.h>
#include <wpi/fs.h>
@@ -193,7 +195,7 @@ struct Thread final : public wpi::SafeThread {
std::string m_logDir;
bool m_filenameOverride;
wpi::log::DataLog m_log;
wpi::log::DataLogBackgroundWriter m_log;
bool m_ntLoggerEnabled = false;
NT_DataLogger m_ntEntryLogger = 0;
NT_ConnectionDataLogger m_ntConnLogger = 0;

11
ntcoreffi/src/main/native/symbols.txt
View File

@@ -236,16 +236,17 @@ WPI_DataLog_AppendIntegerArray
WPI_DataLog_AppendRaw
WPI_DataLog_AppendString
WPI_DataLog_AppendStringArray
WPI_DataLog_Create
WPI_DataLog_Create_Func
WPI_DataLog_CreateBackgroundWriter
WPI_DataLog_CreateBackgroundWriter_Func
WPI_DataLog_CreateWriter
WPI_DataLog_Finish
WPI_DataLog_SetMetadata
WPI_DataLog_Start
WPI_DataLog_Flush
WPI_DataLog_Pause
WPI_DataLog_Release
WPI_DataLog_Resume
WPI_DataLog_SetFilename
WPI_DataLog_SetMetadata
WPI_DataLog_Start
WPI_DataLog_SetBackgroundWriterFilename
WPI_DataLog_Stop
WPI_DestroyEvent
WPI_DestroySemaphore

3
wpilibc/src/main/native/cpp/DataLogManager.cpp
View File

@@ -13,6 +13,7 @@
#include <fmt/chrono.h>
#include <networktables/NetworkTableInstance.h>
#include <wpi/DataLog.h>
#include <wpi/DataLogBackgroundWriter.h>
#include <wpi/SafeThread.h>
#include <wpi/StringExtras.h>
#include <wpi/fs.h>
@@ -39,7 +40,7 @@ struct Thread final : public wpi::SafeThread {
std::string m_logDir;
bool m_filenameOverride;
wpi::log::DataLog m_log;
wpi::log::DataLogBackgroundWriter m_log;
bool m_ntLoggerEnabled = false;
NT_DataLogger m_ntEntryLogger = 0;
NT_ConnectionDataLogger m_ntConnLogger = 0;

5
wpilibj/src/main/java/edu/wpi/first/wpilibj/DataLogManager.java
View File

@@ -8,6 +8,7 @@ import edu.wpi.first.networktables.NetworkTableInstance;
import edu.wpi.first.util.WPIUtilJNI;
import edu.wpi.first.util.concurrent.Event;
import edu.wpi.first.util.datalog.DataLog;
import edu.wpi.first.util.datalog.DataLogBackgroundWriter;
import edu.wpi.first.util.datalog.IntegerLogEntry;
import edu.wpi.first.util.datalog.StringLogEntry;
import java.io.File;
@@ -40,7 +41,7 @@ import java.util.Random;
* <p>By default, all NetworkTables value changes are stored to the data log.
*/
public final class DataLogManager {
private static DataLog m_log;
private static DataLogBackgroundWriter m_log;
private static boolean m_stopped;
private static String m_logDir;
private static boolean m_filenameOverride;
@@ -113,7 +114,7 @@ public final class DataLogManager {
}
}
}
m_log = new DataLog(m_logDir, makeLogFilename(filename), period);
m_log = new DataLogBackgroundWriter(m_logDir, makeLogFilename(filename), period);
m_messageLog = new StringLogEntry(m_log, "messages");
// Log all NT entries and connections

4
wpiutil/examples/writelog/writelog.cpp
View File

@@ -8,7 +8,7 @@
#include <string>
#include <vector>
#include "wpi/DataLog.h"
#include "wpi/DataLogBackgroundWriter.h"
#include "wpi/print.h"
int main(int argc, char** argv) {
@@ -22,7 +22,7 @@ int main(int argc, char** argv) {
kNumRuns = std::stoi(argv[1]);
}
wpi::log::DataLog log;
wpi::log::DataLogBackgroundWriter log;
log.SetFilename("test.wpilog");
auto testVec =

78
wpiutil/src/main/java/edu/wpi/first/util/datalog/DataLog.java
View File

@@ -14,11 +14,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
/**
* A data log. The log file is created immediately upon construction with a temporary filename. The
* file may be renamed at any time using the setFilename() function.
*
* <p>The data log is periodically flushed to disk. It can also be explicitly flushed to disk by
* using the flush() function.
* A data log for high-speed writing of data values.
*
* <p>The finish() function is needed only to indicate in the log that a particular entry is no
* longer being used (it releases the name to ID mapping). The finish() function is not required to
@@ -31,66 +27,14 @@ import java.util.concurrent.ConcurrentMap;
* For this reason (as well as the fact that timestamps can be set to arbitrary values), records in
* the log are not guaranteed to be sorted by timestamp.
*/
public final class DataLog implements AutoCloseable {
public class DataLog implements AutoCloseable {
/**
* Construct a new Data Log. The log will be initially created with a temporary filename.
* Constructs.
*
* @param dir directory to store the log
* @param filename filename to use; if none provided, a random filename is generated of the form
* "wpilog_{}.wpilog"
* @param period time between automatic flushes to disk, in seconds; this is a time/storage
* tradeoff
* @param extraHeader extra header data
* @param impl implementation handle
*/
public DataLog(String dir, String filename, double period, String extraHeader) {
m_impl = DataLogJNI.create(dir, filename, period, extraHeader);
}
/**
* Construct a new Data Log. The log will be initially created with a temporary filename.
*
* @param dir directory to store the log
* @param filename filename to use; if none provided, a random filename is generated of the form
* "wpilog_{}.wpilog"
* @param period time between automatic flushes to disk, in seconds; this is a time/storage
* tradeoff
*/
public DataLog(String dir, String filename, double period) {
this(dir, filename, period, "");
}
/**
* Construct a new Data Log. The log will be initially created with a temporary filename.
*
* @param dir directory to store the log
* @param filename filename to use; if none provided, a random filename is generated of the form
* "wpilog_{}.wpilog"
*/
public DataLog(String dir, String filename) {
this(dir, filename, 0.25);
}
/**
* Construct a new Data Log. The log will be initially created with a temporary filename.
*
* @param dir directory to store the log
*/
public DataLog(String dir) {
this(dir, "", 0.25);
}
/** Construct a new Data Log. The log will be initially created with a temporary filename. */
public DataLog() {
this("");
}
/**
* Change log filename.
*
* @param filename filename
*/
public void setFilename(String filename) {
DataLogJNI.setFilename(m_impl, filename);
protected DataLog(long impl) {
m_impl = impl;
}
/** Explicitly flushes the log data to disk. */
@@ -106,11 +50,7 @@ public final class DataLog implements AutoCloseable {
DataLogJNI.pause(m_impl);
}
/**
* Resumes appending of data records to the log. If called after stop(), opens a new file (with
* random name if SetFilename was not called after stop()) and appends Start records and schema
* data values for all previously started entries and schemas.
*/
/** Resumes appending of data records to the log. */
public void resume() {
DataLogJNI.resume(m_impl);
}
@@ -518,6 +458,8 @@ public final class DataLog implements AutoCloseable {
seen.remove(typeString);
}
private long m_impl;
/** Implementation handle. */
protected long m_impl;
private final ConcurrentMap<String, Integer> m_schemaMap = new ConcurrentHashMap<>();
}

86
wpiutil/src/main/java/edu/wpi/first/util/datalog/DataLogBackgroundWriter.java Normal file
View File

@@ -0,0 +1,86 @@
// Copyright (c) FIRST and other WPILib contributors.
// Open Source Software; you can modify and/or share it under the terms of
// the WPILib BSD license file in the root directory of this project.
package edu.wpi.first.util.datalog;
/**
* A data log background writer that periodically flushes the data log on a background thread. The
* data log file is created immediately upon construction with a temporary filename. The file may be
* renamed at any time using the setFilename() function.
*
* <p>The data log is periodically flushed to disk. It can also be explicitly flushed to disk by
* using the flush() function. This operation is, however, non-blocking.
*/
public final class DataLogBackgroundWriter extends DataLog {
/**
* Construct a new Data Log. The log will be initially created with a temporary filename.
*
* @param dir directory to store the log
* @param filename filename to use; if none provided, a random filename is generated of the form
* "wpilog_{}.wpilog"
* @param period time between automatic flushes to disk, in seconds; this is a time/storage
* tradeoff
* @param extraHeader extra header data
*/
public DataLogBackgroundWriter(String dir, String filename, double period, String extraHeader) {
super(DataLogJNI.bgCreate(dir, filename, period, extraHeader));
}
/**
* Construct a new Data Log. The log will be initially created with a temporary filename.
*
* @param dir directory to store the log
* @param filename filename to use; if none provided, a random filename is generated of the form
* "wpilog_{}.wpilog"
* @param period time between automatic flushes to disk, in seconds; this is a time/storage
* tradeoff
*/
public DataLogBackgroundWriter(String dir, String filename, double period) {
this(dir, filename, period, "");
}
/**
* Construct a new Data Log. The log will be initially created with a temporary filename.
*
* @param dir directory to store the log
* @param filename filename to use; if none provided, a random filename is generated of the form
* "wpilog_{}.wpilog"
*/
public DataLogBackgroundWriter(String dir, String filename) {
this(dir, filename, 0.25);
}
/**
* Construct a new Data Log. The log will be initially created with a temporary filename.
*
* @param dir directory to store the log
*/
public DataLogBackgroundWriter(String dir) {
this(dir, "", 0.25);
}
/** Construct a new Data Log. The log will be initially created with a temporary filename. */
public DataLogBackgroundWriter() {
this("");
}
/**
* Change log filename.
*
* @param filename filename
*/
public void setFilename(String filename) {
DataLogJNI.bgSetFilename(m_impl, filename);
}
/**
* Resumes appending of data records to the log. If called after stop(), opens a new file (with
* random name if SetFilename was not called after stop()) and appends Start records and schema
* data values for all previously started entries and schemas.
*/
@Override
public void resume() {
DataLogJNI.resume(m_impl);
}
}

30
wpiutil/src/main/java/edu/wpi/first/util/datalog/DataLogJNI.java
View File

@@ -5,6 +5,7 @@
package edu.wpi.first.util.datalog;
import edu.wpi.first.util.WPIUtilJNI;
import java.io.IOException;
import java.nio.ByteBuffer;
/**
@@ -14,7 +15,8 @@ import java.nio.ByteBuffer;
*/
public class DataLogJNI extends WPIUtilJNI {
/**
* Create a new Data Log. The log will be initially created with a temporary filename.
* Create a new Data Log background writer. The log will be initially created with a temporary
* filename.
*
* @param dir directory to store the log
* @param filename filename to use; if none provided, a random filename is generated of the form
@@ -22,22 +24,32 @@ public class DataLogJNI extends WPIUtilJNI {
* @param period time between automatic flushes to disk, in seconds; this is a time/storage
* tradeoff
* @param extraHeader extra header data
* @return data log implementation handle
* @return data log background writer implementation handle
*/
static native long create(String dir, String filename, double period, String extraHeader);
static native long bgCreate(String dir, String filename, double period, String extraHeader);
/**
* Change log filename.
*
* @param impl data log implementation handle
* @param impl data log background writer implementation handle
* @param filename filename
*/
static native void setFilename(long impl, String filename);
static native void bgSetFilename(long impl, String filename);
/**
* Create a new Data Log foreground writer.
*
* @param filename filename to use
* @param extraHeader extra header data
* @return data log writer implementation handle
* @throws IOException if file cannot be opened
*/
static native long fgCreate(String filename, String extraHeader) throws IOException;
/**
* Explicitly flushes the log data to disk.
*
* @param impl data log implementation handle
* @param impl data log background writer implementation handle
*/
static native void flush(long impl);
@@ -45,7 +57,7 @@ public class DataLogJNI extends WPIUtilJNI {
* Pauses appending of data records to the log. While paused, no data records are saved (e.g.
* AppendX is a no-op). Has no effect on entry starts / finishes / metadata changes.
*
* @param impl data log implementation handle
* @param impl data log background writer implementation handle
*/
static native void pause(long impl);
@@ -54,14 +66,14 @@ public class DataLogJNI extends WPIUtilJNI {
* random name if SetFilename was not called after Stop()) and appends Start records and schema
* data values for all previously started entries and schemas.
*
* @param impl data log implementation handle
* @param impl data log background writer implementation handle
*/
static native void resume(long impl);
/**
* Stops appending all records to the log, and closes the log file.
*
* @param impl data log implementation handle
* @param impl data log background writer implementation handle
*/
static native void stop(long impl);

31
wpiutil/src/main/java/edu/wpi/first/util/datalog/DataLogWriter.java Normal file
View File

@@ -0,0 +1,31 @@
// Copyright (c) FIRST and other WPILib contributors.
// Open Source Software; you can modify and/or share it under the terms of
// the WPILib BSD license file in the root directory of this project.
package edu.wpi.first.util.datalog;
import java.io.IOException;
/** A data log writer that flushes the data log to a file when flush() is called. */
public class DataLogWriter extends DataLog {
/**
* Construct a new Data Log.
*
* @param filename filename to use
* @param extraHeader extra header data
* @throws IOException if file cannot be opened
*/
public DataLogWriter(String filename, String extraHeader) throws IOException {
super(DataLogJNI.fgCreate(filename, extraHeader));
}
/**
* Construct a new Data Log.
*
* @param filename filename to use
* @throws IOException if file cannot be opened
*/
public DataLogWriter(String filename) throws IOException {
this(filename, "");
}
}

643
wpiutil/src/main/native/cpp/DataLog.cpp
View File

@@ -4,60 +4,33 @@
#include "wpi/DataLog.h"
#include "wpi/Synchronization.h"
#ifndef _WIN32
#include <unistd.h>
#endif
#ifdef _WIN32
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif
#include <windows.h> // NOLINT(build/include_order)
#endif
#include <atomic>
#include <bit>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <random>
#include <vector>
#include "wpi/Endian.h"
#include "wpi/Logger.h"
#include "wpi/MathExtras.h"
#include "wpi/SmallString.h"
#include "wpi/fs.h"
#include "wpi/print.h"
#include "wpi/timestamp.h"
using namespace wpi::log;
static constexpr size_t kBlockSize = 16 * 1024;
static constexpr size_t kMaxBufferCount = 1024 * 1024 / kBlockSize;
static constexpr size_t kMaxFreeCount = 256 * 1024 / kBlockSize;
static constexpr size_t kRecordMaxHeaderSize = 17;
static constexpr uintmax_t kMinFreeSpace = 5 * 1024 * 1024;
static std::string FormatBytesSize(uintmax_t value) {
static constexpr uintmax_t kKiB = 1024;
static constexpr uintmax_t kMiB = kKiB * 1024;
static constexpr uintmax_t kGiB = kMiB * 1024;
if (value >= kGiB) {
return fmt::format("{:.1f} GiB", static_cast<double>(value) / kGiB);
} else if (value >= kMiB) {
return fmt::format("{:.1f} MiB", static_cast<double>(value) / kMiB);
} else if (value >= kKiB) {
return fmt::format("{:.1f} KiB", static_cast<double>(value) / kKiB);
} else {
return fmt::format("{} B", value);
static void DefaultLog(unsigned int level, const char* file, unsigned int line,
const char* msg) {
if (level > wpi::WPI_LOG_INFO) {
wpi::print(stderr, "DataLog: {}\n", msg);
} else if (level == wpi::WPI_LOG_INFO) {
wpi::print("DataLog: {}\n", msg);
}
}
wpi::Logger DataLog::s_defaultMessageLog{DefaultLog};
template <typename T>
static unsigned int WriteVarInt(uint8_t* buf, T val) {
unsigned int len = 0;
@@ -87,144 +60,72 @@ static unsigned int WriteRecordHeader(uint8_t* buf, uint32_t entry,
return buf - origbuf;
}
class DataLog::Buffer {
public:
explicit Buffer(size_t alloc = kBlockSize)
: m_buf{new uint8_t[alloc]}, m_maxLen{alloc} {}
~Buffer() { delete[] m_buf; }
Buffer(const Buffer&) = delete;
Buffer& operator=(const Buffer&) = delete;
Buffer(Buffer&& oth)
: m_buf{oth.m_buf}, m_len{oth.m_len}, m_maxLen{oth.m_maxLen} {
oth.m_buf = nullptr;
oth.m_len = 0;
oth.m_maxLen = 0;
void DataLog::StartFile() {
std::scoped_lock lock{m_mutex};
if (m_active) {
return;
}
Buffer& operator=(Buffer&& oth) {
if (m_buf) {
delete[] m_buf;
// Grab previously pending writes
std::vector<Buffer> bufs;
bufs.swap(m_outgoing);
m_outgoing.reserve(bufs.size() + 1);
// File header (version 1.0)
uint8_t* buf = Reserve(m_extraHeader.size() + 12);
static const uint8_t header[] = {'W', 'P', 'I', 'L', 'O', 'G', 0, 1};
std::memcpy(buf, header, 8);
support::endian::write32le(buf + 8, m_extraHeader.size());
std::memcpy(buf + 12, m_extraHeader.data(), m_extraHeader.size());
// Existing start and schema data records
for (auto&& entryInfo : m_entries) {
AppendStartRecord(entryInfo.second.id, entryInfo.first(),
entryInfo.second.type,
m_entryIds[entryInfo.second.id].metadata, 0);
if (!entryInfo.second.schemaData.empty()) {
StartRecord(entryInfo.second.id, 0, entryInfo.second.schemaData.size(),
0);
AppendImpl(entryInfo.second.schemaData);
}
m_buf = oth.m_buf;
m_len = oth.m_len;
m_maxLen = oth.m_maxLen;
oth.m_buf = nullptr;
oth.m_len = 0;
oth.m_maxLen = 0;
return *this;
}
uint8_t* Reserve(size_t size) {
assert(size <= GetRemaining());
uint8_t* rv = m_buf + m_len;
m_len += size;
return rv;
// Append previously pending writes
for (auto&& buf : bufs) {
m_outgoing.emplace_back(std::move(buf));
}
void Unreserve(size_t size) { m_len -= size; }
void Clear() { m_len = 0; }
size_t GetRemaining() const { return m_maxLen - m_len; }
std::span<uint8_t> GetData() { return {m_buf, m_len}; }
std::span<const uint8_t> GetData() const { return {m_buf, m_len}; }
private:
uint8_t* m_buf;
size_t m_len = 0;
size_t m_maxLen;
};
static void DefaultLog(unsigned int level, const char* file, unsigned int line,
const char* msg) {
if (level > wpi::WPI_LOG_INFO) {
wpi::print(stderr, "DataLog: {}\n", msg);
} else if (level == wpi::WPI_LOG_INFO) {
wpi::print("DataLog: {}\n", msg);
}
m_active = true;
}
static wpi::Logger defaultMessageLog{DefaultLog};
DataLog::DataLog(std::string_view dir, std::string_view filename, double period,
std::string_view extraHeader)
: DataLog{defaultMessageLog, dir, filename, period, extraHeader} {}
DataLog::DataLog(wpi::Logger& msglog, std::string_view dir,
std::string_view filename, double period,
std::string_view extraHeader)
: m_msglog{msglog},
m_period{period},
m_extraHeader{extraHeader},
m_newFilename{filename},
m_thread{[this, dir = std::string{dir}] { WriterThreadMain(dir); }} {}
DataLog::DataLog(std::function<void(std::span<const uint8_t> data)> write,
double period, std::string_view extraHeader)
: DataLog{defaultMessageLog, std::move(write), period, extraHeader} {}
DataLog::DataLog(wpi::Logger& msglog,
std::function<void(std::span<const uint8_t> data)> write,
double period, std::string_view extraHeader)
: m_msglog{msglog},
m_period{period},
m_extraHeader{extraHeader},
m_thread{[this, write = std::move(write)] {
WriterThreadMain(std::move(write));
}} {}
DataLog::~DataLog() {
{
std::scoped_lock lock{m_mutex};
m_shutdown = true;
m_doFlush = true;
}
m_cond.notify_all();
m_thread.join();
void DataLog::FlushBufs(std::vector<Buffer>* writeBufs) {
std::scoped_lock lock{m_mutex};
writeBufs->swap(m_outgoing);
DoReleaseBufs(&m_outgoing);
}
void DataLog::SetFilename(std::string_view filename) {
{
std::scoped_lock lock{m_mutex};
m_newFilename = filename;
}
m_cond.notify_all();
}
void DataLog::Flush() {
{
std::scoped_lock lock{m_mutex};
m_doFlush = true;
}
m_cond.notify_all();
void DataLog::ReleaseBufs(std::vector<Buffer>* bufs) {
std::scoped_lock lock{m_mutex};
DoReleaseBufs(bufs);
}
void DataLog::Pause() {
std::scoped_lock lock{m_mutex};
m_state = kPaused;
m_paused = true;
}
void DataLog::Resume() {
std::scoped_lock lock{m_mutex};
if (m_state == kPaused) {
m_state = kActive;
} else if (m_state == kStopped) {
m_state = kStart;
}
m_paused = false;
}
void DataLog::Stop() {
{
std::scoped_lock lock{m_mutex};
m_state = kStopped;
m_newFilename.clear();
}
m_cond.notify_all();
std::scoped_lock lock{m_mutex};
m_active = false;
}
void DataLog::BufferHalfFull() {}
bool DataLog::HasSchema(std::string_view name) const {
std::scoped_lock lock{m_mutex};
wpi::SmallString<128> fullName{"/.schema/"};
@@ -249,380 +150,13 @@ void DataLog::AddSchema(std::string_view name, std::string_view type,
if (entry <= 0) {
[[unlikely]] return; // should never happen, but check anyway
}
if (m_state != kActive && m_state != kPaused) {
if (!m_active) {
[[unlikely]] return;
}
StartRecord(entry, timestamp, schema.size(), 0);
AppendImpl(schema);
}
static void WriteToFile(fs::file_t f, std::span<const uint8_t> data,
std::string_view filename, wpi::Logger& msglog) {
do {
#ifdef _WIN32
DWORD ret;
if (!WriteFile(f, data.data(), data.size(), &ret, nullptr)) {
WPI_ERROR(msglog, "Error writing to log file '{}': {}", filename,
GetLastError());
break;
}
#else
ssize_t ret = ::write(f, data.data(), data.size());
if (ret < 0) {
// If it's a recoverable error, swallow it and retry the write
if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
continue;
}
// Otherwise it's a non-recoverable error; quit trying
WPI_ERROR(msglog, "Error writing to log file '{}': {}", filename,
std::strerror(errno));
break;
}
#endif
// The write may have written some or all of the data
data = data.subspan(ret);
} while (data.size() > 0);
}
static std::string MakeRandomFilename() {
// build random filename
static std::random_device dev;
static std::mt19937 rng(dev());
std::uniform_int_distribution<int> dist(0, 15);
const char* v = "0123456789abcdef";
std::string filename = "wpilog_";
for (int i = 0; i < 16; i++) {
filename += v[dist(rng)];
}
filename += ".wpilog";
return filename;
}
struct DataLog::WriterThreadState {
explicit WriterThreadState(std::string_view dir) : dirPath{dir} {}
WriterThreadState(const WriterThreadState&) = delete;
WriterThreadState& operator=(const WriterThreadState&) = delete;
~WriterThreadState() { Close(); }
void Close() {
if (f != fs::kInvalidFile) {
fs::CloseFile(f);
f = fs::kInvalidFile;
}
}
void SetFilename(std::string_view fn) {
baseFilename = fn;
filename = fn;
path = dirPath / filename;
segmentCount = 1;
}
void IncrementFilename() {
fs::path basePath{baseFilename};
filename = fmt::format("{}.{}{}", basePath.stem().string(), ++segmentCount,
basePath.extension().string());
path = dirPath / filename;
}
fs::path dirPath;
std::string baseFilename;
std::string filename;
fs::path path;
fs::file_t f = fs::kInvalidFile;
uintmax_t freeSpace = UINTMAX_MAX;
int segmentCount = 1;
};
void DataLog::StartLogFile(WriterThreadState& state) {
std::error_code ec;
if (state.filename.empty()) {
state.SetFilename(MakeRandomFilename());
}
// get free space
auto freeSpaceInfo = fs::space(state.dirPath, ec);
if (!ec) {
state.freeSpace = freeSpaceInfo.available;
} else {
state.freeSpace = UINTMAX_MAX;
}
if (state.freeSpace < kMinFreeSpace) {
WPI_ERROR(m_msglog,
"Insufficient free space ({} available), no log being saved",
FormatBytesSize(state.freeSpace));
m_state = kStopped;
} else {
// try preferred filename, or randomize it a few times, before giving up
for (int i = 0; i < 5; ++i) {
// open file for append
#ifdef _WIN32
// WIN32 doesn't allow combination of CreateNew and Append
state.f =
fs::OpenFileForWrite(state.path, ec, fs::CD_CreateNew, fs::OF_None);
#else
state.f =
fs::OpenFileForWrite(state.path, ec, fs::CD_CreateNew, fs::OF_Append);
#endif
if (ec) {
WPI_ERROR(m_msglog, "Could not open log file '{}': {}",
state.path.string(), ec.message());
// try again with random filename
state.SetFilename(MakeRandomFilename());
} else {
break;
}
}
if (state.f == fs::kInvalidFile) {
WPI_ERROR(m_msglog, "Could not open log file, no log being saved");
} else {
WPI_INFO(m_msglog, "Logging to '{}' ({} free space)", state.path.string(),
FormatBytesSize(state.freeSpace));
}
}
// write header (version 1.0)
if (state.f != fs::kInvalidFile) {
const uint8_t header[] = {'W', 'P', 'I', 'L', 'O', 'G', 0, 1};
WriteToFile(state.f, header, state.filename, m_msglog);
uint8_t extraLen[4];
support::endian::write32le(extraLen, m_extraHeader.size());
WriteToFile(state.f, extraLen, state.filename, m_msglog);
if (m_extraHeader.size() > 0) {
WriteToFile(state.f,
{reinterpret_cast<const uint8_t*>(m_extraHeader.data()),
m_extraHeader.size()},
state.filename, m_msglog);
}
}
}
void DataLog::WriterThreadMain(std::string_view dir) {
std::chrono::duration<double> periodTime{m_period};
WriterThreadState state{dir};
{
std::scoped_lock lock{m_mutex};
state.SetFilename(m_newFilename);
m_newFilename.clear();
}
StartLogFile(state);
std::error_code ec;
std::vector<Buffer> toWrite;
int freeSpaceCount = 0;
int checkExistCount = 0;
bool blocked = false;
uintmax_t written = 0;
std::unique_lock lock{m_mutex};
do {
bool doFlush = false;
auto timeoutTime = std::chrono::steady_clock::now() + periodTime;
if (m_cond.wait_until(lock, timeoutTime) == std::cv_status::timeout) {
doFlush = true;
}
if (m_state == kStopped) {
state.Close();
continue;
}
bool doStart = false;
// if file was deleted, recreate it with the same name
if (++checkExistCount >= 10) {
checkExistCount = 0;
lock.unlock();
bool exists = fs::exists(state.path, ec);
lock.lock();
if (!ec && !exists) {
state.Close();
state.IncrementFilename();
WPI_INFO(m_msglog, "Log file deleted, recreating as fresh log '{}'",
state.filename);
doStart = true;
}
}
// start new file if file exceeds 1.8 GB
if (written > 1800000000ull) {
state.Close();
state.IncrementFilename();
WPI_INFO(m_msglog, "Log file reached 1.8 GB, starting new file '{}'",
state.filename);
doStart = true;
}
if (m_state == kStart || doStart) {
lock.unlock();
StartLogFile(state);
lock.lock();
if (m_state == kStopped) {
continue;
}
if (state.f != fs::kInvalidFile) {
// Emit start and schema data records
for (auto&& entryInfo : m_entries) {
AppendStartRecord(entryInfo.second.id, entryInfo.first(),
entryInfo.second.type,
m_entryIds[entryInfo.second.id].metadata, 0);
if (!entryInfo.second.schemaData.empty()) {
StartRecord(entryInfo.second.id, 0,
entryInfo.second.schemaData.size(), 0);
AppendImpl(entryInfo.second.schemaData);
}
}
}
m_state = kActive;
written = 0;
}
if (!m_newFilename.empty() && state.f != fs::kInvalidFile) {
auto newFilename = std::move(m_newFilename);
m_newFilename.clear();
// rename
if (state.filename != newFilename) {
lock.unlock();
fs::rename(state.path, state.dirPath / newFilename, ec);
lock.lock();
}
if (ec) {
WPI_ERROR(m_msglog, "Could not rename log file from '{}' to '{}': {}",
state.filename, newFilename, ec.message());
} else {
WPI_INFO(m_msglog, "Renamed log file from '{}' to '{}'", state.filename,
newFilename);
}
state.SetFilename(newFilename);
}
if (doFlush || m_doFlush) {
// flush to file
m_doFlush = false;
if (m_outgoing.empty()) {
continue;
}
// swap outgoing with empty vector
toWrite.swap(m_outgoing);
if (state.f != fs::kInvalidFile && !blocked) {
lock.unlock();
// update free space every 10 flushes (in case other things are writing)
if (++freeSpaceCount >= 10) {
freeSpaceCount = 0;
auto freeSpaceInfo = fs::space(state.dirPath, ec);
if (!ec) {
state.freeSpace = freeSpaceInfo.available;
} else {
state.freeSpace = UINTMAX_MAX;
}
}
// write buffers to file
for (auto&& buf : toWrite) {
// stop writing when we go below the minimum free space
state.freeSpace -= buf.GetData().size();
written += buf.GetData().size();
if (state.freeSpace < kMinFreeSpace) {
[[unlikely]] WPI_ERROR(
m_msglog,
"Stopped logging due to low free space ({} available)",
FormatBytesSize(state.freeSpace));
blocked = true;
break;
}
WriteToFile(state.f, buf.GetData(), state.filename, m_msglog);
}
// sync to storage
#if defined(__linux__)
::fdatasync(state.f);
#elif defined(__APPLE__)
::fsync(state.f);
#endif
lock.lock();
if (blocked) {
[[unlikely]] m_state = kPaused;
}
}
// release buffers back to free list
for (auto&& buf : toWrite) {
buf.Clear();
if (m_free.size() < kMaxFreeCount) {
[[likely]] m_free.emplace_back(std::move(buf));
}
}
toWrite.resize(0);
}
} while (!m_shutdown);
}
void DataLog::WriterThreadMain(
std::function<void(std::span<const uint8_t> data)> write) {
std::chrono::duration<double> periodTime{m_period};
// write header (version 1.0)
{
const uint8_t header[] = {'W', 'P', 'I', 'L', 'O', 'G', 0, 1};
write(header);
uint8_t extraLen[4];
support::endian::write32le(extraLen, m_extraHeader.size());
write(extraLen);
if (m_extraHeader.size() > 0) {
write({reinterpret_cast<const uint8_t*>(m_extraHeader.data()),
m_extraHeader.size()});
}
}
std::vector<Buffer> toWrite;
std::unique_lock lock{m_mutex};
do {
bool doFlush = false;
auto timeoutTime = std::chrono::steady_clock::now() + periodTime;
if (m_cond.wait_until(lock, timeoutTime) == std::cv_status::timeout) {
doFlush = true;
}
if (doFlush || m_doFlush) {
// flush to file
m_doFlush = false;
if (m_outgoing.empty()) {
continue;
}
// swap outgoing with empty vector
toWrite.swap(m_outgoing);
lock.unlock();
// write buffers
for (auto&& buf : toWrite) {
if (!buf.GetData().empty()) {
write(buf.GetData());
}
}
lock.lock();
// release buffers back to free list
for (auto&& buf : toWrite) {
buf.Clear();
if (m_free.size() < kMaxFreeCount) {
[[likely]] m_free.emplace_back(std::move(buf));
}
}
toWrite.resize(0);
}
} while (!m_shutdown);
write({}); // indicate EOF
}
// Control records use the following format:
// 1-byte type
// 4-byte entry
@@ -654,7 +188,7 @@ int DataLog::StartImpl(std::string_view name, std::string_view type,
entryInfo.type = type;
entryInfo2.metadata = metadata;
if (m_state != kActive && m_state != kPaused) {
if (!m_active) {
[[unlikely]] return entryInfo.id;
}
@@ -674,6 +208,16 @@ void DataLog::AppendStartRecord(int id, std::string_view name,
AppendStringImpl(metadata);
}
void DataLog::DoReleaseBufs(std::vector<Buffer>* bufs) {
for (auto&& buf : *bufs) {
buf.Clear();
if (m_free.size() < kMaxFreeCount) {
[[likely]] m_free.emplace_back(std::move(buf));
}
}
bufs->resize(0);
}
void DataLog::Finish(int entry, int64_t timestamp) {
if (entry <= 0) {
return;
@@ -688,7 +232,7 @@ void DataLog::Finish(int entry, int64_t timestamp) {
return;
}
m_entryIds.erase(entry);
if (m_state != kActive && m_state != kPaused) {
if (!m_active) {
[[unlikely]] return;
}
uint8_t* buf = StartRecord(0, timestamp, 5, 5);
@@ -703,7 +247,7 @@ void DataLog::SetMetadata(int entry, std::string_view metadata,
}
std::scoped_lock lock{m_mutex};
m_entryIds[entry].metadata = metadata;
if (m_state != kActive && m_state != kPaused) {
if (!m_active) {
[[unlikely]] return;
}
uint8_t* buf = StartRecord(0, timestamp, 5 + 4 + metadata.size(), 5);
@@ -715,13 +259,15 @@ void DataLog::SetMetadata(int entry, std::string_view metadata,
uint8_t* DataLog::Reserve(size_t size) {
assert(size <= kBlockSize);
if (m_outgoing.empty() || size > m_outgoing.back().GetRemaining()) {
if (m_outgoing.size() == kMaxBufferCount / 2) {
[[unlikely]] BufferHalfFull();
}
if (m_free.empty()) {
if (m_outgoing.size() >= kMaxBufferCount) {
[[unlikely]] WPI_ERROR(
m_msglog,
"outgoing buffers exceeded threshold, pausing logging--"
"consider flushing to disk more frequently (smaller period)");
m_state = kPaused;
[[unlikely]]
if (BufferFull()) {
m_paused = true;
}
}
m_outgoing.emplace_back();
} else {
@@ -765,7 +311,7 @@ void DataLog::AppendRaw(int entry, std::span<const uint8_t> data,
return;
}
std::scoped_lock lock{m_mutex};
if (m_state != kActive) {
if (m_paused) {
[[unlikely]] return;
}
StartRecord(entry, timestamp, data.size(), 0);
@@ -779,7 +325,7 @@ void DataLog::AppendRaw2(int entry,
return;
}
std::scoped_lock lock{m_mutex};
if (m_state != kActive) {
if (m_paused) {
[[unlikely]] return;
}
size_t size = 0;
@@ -797,7 +343,7 @@ void DataLog::AppendBoolean(int entry, bool value, int64_t timestamp) {
return;
}
std::scoped_lock lock{m_mutex};
if (m_state != kActive) {
if (m_paused) {
[[unlikely]] return;
}
uint8_t* buf = StartRecord(entry, timestamp, 1, 1);
@@ -809,7 +355,7 @@ void DataLog::AppendInteger(int entry, int64_t value, int64_t timestamp) {
return;
}
std::scoped_lock lock{m_mutex};
if (m_state != kActive) {
if (m_paused) {
[[unlikely]] return;
}
uint8_t* buf = StartRecord(entry, timestamp, 8, 8);
@@ -821,7 +367,7 @@ void DataLog::AppendFloat(int entry, float value, int64_t timestamp) {
return;
}
std::scoped_lock lock{m_mutex};
if (m_state != kActive) {
if (m_paused) {
[[unlikely]] return;
}
uint8_t* buf = StartRecord(entry, timestamp, 4, 4);
@@ -837,7 +383,7 @@ void DataLog::AppendDouble(int entry, double value, int64_t timestamp) {
return;
}
std::scoped_lock lock{m_mutex};
if (m_state != kActive) {
if (m_paused) {
[[unlikely]] return;
}
uint8_t* buf = StartRecord(entry, timestamp, 8, 8);
@@ -861,7 +407,7 @@ void DataLog::AppendBooleanArray(int entry, std::span<const bool> arr,
return;
}
std::scoped_lock lock{m_mutex};
if (m_state != kActive) {
if (m_paused) {
[[unlikely]] return;
}
StartRecord(entry, timestamp, arr.size(), 0);
@@ -885,7 +431,7 @@ void DataLog::AppendBooleanArray(int entry, std::span<const int> arr,
return;
}
std::scoped_lock lock{m_mutex};
if (m_state != kActive) {
if (m_paused) {
[[unlikely]] return;
}
StartRecord(entry, timestamp, arr.size(), 0);
@@ -919,7 +465,7 @@ void DataLog::AppendIntegerArray(int entry, std::span<const int64_t> arr,
return;
}
std::scoped_lock lock{m_mutex};
if (m_state != kActive) {
if (m_paused) {
[[unlikely]] return;
}
StartRecord(entry, timestamp, arr.size() * 8, 0);
@@ -951,7 +497,7 @@ void DataLog::AppendFloatArray(int entry, std::span<const float> arr,
return;
}
std::scoped_lock lock{m_mutex};
if (m_state != kActive) {
if (m_paused) {
[[unlikely]] return;
}
StartRecord(entry, timestamp, arr.size() * 4, 0);
@@ -983,7 +529,7 @@ void DataLog::AppendDoubleArray(int entry, std::span<const double> arr,
return;
}
std::scoped_lock lock{m_mutex};
if (m_state != kActive) {
if (m_paused) {
[[unlikely]] return;
}
StartRecord(entry, timestamp, arr.size() * 8, 0);
@@ -1016,7 +562,7 @@ void DataLog::AppendStringArray(int entry, std::span<const std::string> arr,
size += 4 + str.size();
}
std::scoped_lock lock{m_mutex};
if (m_state != kActive) {
if (m_paused) {
[[unlikely]] return;
}
uint8_t* buf = StartRecord(entry, timestamp, size, 4);
@@ -1039,7 +585,7 @@ void DataLog::AppendStringArray(int entry,
size += 4 + str.size();
}
std::scoped_lock lock{m_mutex};
if (m_state != kActive) {
if (m_paused) {
[[unlikely]] return;
}
uint8_t* buf = StartRecord(entry, timestamp, size, 4);
@@ -1062,7 +608,7 @@ void DataLog::AppendStringArray(int entry,
size += 4 + str.len;
}
std::scoped_lock lock{m_mutex};
if (m_state != kActive) {
if (m_paused) {
[[unlikely]] return;
}
uint8_t* buf = StartRecord(entry, timestamp, size, 4);
@@ -1074,29 +620,10 @@ void DataLog::AppendStringArray(int entry,
extern "C" {
struct WPI_DataLog* WPI_DataLog_Create(const char* dir, const char* filename,
double period, const char* extraHeader) {
return reinterpret_cast<WPI_DataLog*>(
new DataLog{dir, filename, period, extraHeader});
}
struct WPI_DataLog* WPI_DataLog_Create_Func(
void (*write)(void* ptr, const uint8_t* data, size_t len), void* ptr,
double period, const char* extraHeader) {
return reinterpret_cast<WPI_DataLog*>(
new DataLog{[=](auto data) { write(ptr, data.data(), data.size()); },
period, extraHeader});
}
void WPI_DataLog_Release(struct WPI_DataLog* datalog) {
delete reinterpret_cast<DataLog*>(datalog);
}
void WPI_DataLog_SetFilename(struct WPI_DataLog* datalog,
const char* filename) {
reinterpret_cast<DataLog*>(datalog)->SetFilename(filename);
}
void WPI_DataLog_Flush(struct WPI_DataLog* datalog) {
reinterpret_cast<DataLog*>(datalog)->Flush();
}

483
wpiutil/src/main/native/cpp/DataLogBackgroundWriter.cpp Normal file
View File

@@ -0,0 +1,483 @@
// Copyright (c) FIRST and other WPILib contributors.
// Open Source Software; you can modify and/or share it under the terms of
// the WPILib BSD license file in the root directory of this project.
#include "wpi/DataLogBackgroundWriter.h"
#ifndef _WIN32
#include <unistd.h>
#endif
#ifdef _WIN32
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif
#include <windows.h> // NOLINT(build/include_order)
#endif
#include <random>
#include <fmt/format.h>
#include "wpi/Logger.h"
#include "wpi/fs.h"
using namespace wpi::log;
static constexpr uintmax_t kMinFreeSpace = 5 * 1024 * 1024;
static std::string FormatBytesSize(uintmax_t value) {
static constexpr uintmax_t kKiB = 1024;
static constexpr uintmax_t kMiB = kKiB * 1024;
static constexpr uintmax_t kGiB = kMiB * 1024;
if (value >= kGiB) {
return fmt::format("{:.1f} GiB", static_cast<double>(value) / kGiB);
} else if (value >= kMiB) {
return fmt::format("{:.1f} MiB", static_cast<double>(value) / kMiB);
} else if (value >= kKiB) {
return fmt::format("{:.1f} KiB", static_cast<double>(value) / kKiB);
} else {
return fmt::format("{} B", value);
}
}
DataLogBackgroundWriter::DataLogBackgroundWriter(std::string_view dir,
std::string_view filename,
double period,
std::string_view extraHeader)
: DataLogBackgroundWriter{s_defaultMessageLog, dir, filename, period,
extraHeader} {}
DataLogBackgroundWriter::DataLogBackgroundWriter(wpi::Logger& msglog,
std::string_view dir,
std::string_view filename,
double period,
std::string_view extraHeader)
: DataLog{msglog, extraHeader},
m_period{period},
m_newFilename{filename},
m_thread{[this, dir = std::string{dir}] { WriterThreadMain(dir); }} {}
DataLogBackgroundWriter::DataLogBackgroundWriter(
std::function<void(std::span<const uint8_t> data)> write, double period,
std::string_view extraHeader)
: DataLogBackgroundWriter{s_defaultMessageLog, std::move(write), period,
extraHeader} {}
DataLogBackgroundWriter::DataLogBackgroundWriter(
wpi::Logger& msglog,
std::function<void(std::span<const uint8_t> data)> write, double period,
std::string_view extraHeader)
: DataLog{msglog, extraHeader},
m_period{period},
m_thread{[this, write = std::move(write)] {
WriterThreadMain(std::move(write));
}} {}
DataLogBackgroundWriter::~DataLogBackgroundWriter() {
{
std::scoped_lock lock{m_mutex};
m_shutdown = true;
m_doFlush = true;
}
m_cond.notify_all();
m_thread.join();
}
void DataLogBackgroundWriter::SetFilename(std::string_view filename) {
{
std::scoped_lock lock{m_mutex};
m_newFilename = filename;
}
m_cond.notify_all();
}
void DataLogBackgroundWriter::Flush() {
{
std::scoped_lock lock{m_mutex};
m_doFlush = true;
}
m_cond.notify_all();
}
void DataLogBackgroundWriter::Pause() {
DataLog::Pause();
std::scoped_lock lock{m_mutex};
m_state = kPaused;
}
void DataLogBackgroundWriter::Resume() {
DataLog::Resume();
std::scoped_lock lock{m_mutex};
if (m_state == kPaused) {
m_state = kActive;
} else if (m_state == kStopped) {
m_state = kStart;
}
}
void DataLogBackgroundWriter::Stop() {
DataLog::Stop();
{
std::scoped_lock lock{m_mutex};
m_state = kStopped;
m_newFilename.clear();
}
m_cond.notify_all();
}
static void WriteToFile(fs::file_t f, std::span<const uint8_t> data,
std::string_view filename, wpi::Logger& msglog) {
do {
#ifdef _WIN32
DWORD ret;
if (!WriteFile(f, data.data(), data.size(), &ret, nullptr)) {
WPI_ERROR(msglog, "Error writing to log file '{}': {}", filename,
GetLastError());
break;
}
#else
ssize_t ret = ::write(f, data.data(), data.size());
if (ret < 0) {
// If it's a recoverable error, swallow it and retry the write
if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
continue;
}
// Otherwise it's a non-recoverable error; quit trying
WPI_ERROR(msglog, "Error writing to log file '{}': {}", filename,
std::strerror(errno));
break;
}
#endif
// The write may have written some or all of the data
data = data.subspan(ret);
} while (data.size() > 0);
}
static std::string MakeRandomFilename() {
// build random filename
static std::random_device dev;
static std::mt19937 rng(dev());
std::uniform_int_distribution<int> dist(0, 15);
const char* v = "0123456789abcdef";
std::string filename = "wpilog_";
for (int i = 0; i < 16; i++) {
filename += v[dist(rng)];
}
filename += ".wpilog";
return filename;
}
struct DataLogBackgroundWriter::WriterThreadState {
explicit WriterThreadState(std::string_view dir) : dirPath{dir} {}
WriterThreadState(const WriterThreadState&) = delete;
WriterThreadState& operator=(const WriterThreadState&) = delete;
~WriterThreadState() { Close(); }
void Close() {
if (f != fs::kInvalidFile) {
fs::CloseFile(f);
f = fs::kInvalidFile;
}
}
void SetFilename(std::string_view fn) {
baseFilename = fn;
filename = fn;
path = dirPath / filename;
segmentCount = 1;
}
void IncrementFilename() {
fs::path basePath{baseFilename};
filename = fmt::format("{}.{}{}", basePath.stem().string(), ++segmentCount,
basePath.extension().string());
path = dirPath / filename;
}
fs::path dirPath;
std::string baseFilename;
std::string filename;
fs::path path;
fs::file_t f = fs::kInvalidFile;
uintmax_t freeSpace = UINTMAX_MAX;
int segmentCount = 1;
};
void DataLogBackgroundWriter::BufferHalfFull() {
Flush();
}
bool DataLogBackgroundWriter::BufferFull() {
WPI_ERROR(m_msglog,
"outgoing buffers exceeded threshold, pausing logging--"
"consider flushing to disk more frequently (smaller period)");
return true;
}
void DataLogBackgroundWriter::StartLogFile(WriterThreadState& state) {
std::error_code ec;
if (state.filename.empty()) {
state.SetFilename(MakeRandomFilename());
}
// get free space
auto freeSpaceInfo = fs::space(state.dirPath, ec);
if (!ec) {
state.freeSpace = freeSpaceInfo.available;
} else {
state.freeSpace = UINTMAX_MAX;
}
if (state.freeSpace < kMinFreeSpace) {
WPI_ERROR(m_msglog,
"Insufficient free space ({} available), no log being saved",
FormatBytesSize(state.freeSpace));
} else {
// try preferred filename, or randomize it a few times, before giving up
for (int i = 0; i < 5; ++i) {
// open file for append
#ifdef _WIN32
// WIN32 doesn't allow combination of CreateNew and Append
state.f =
fs::OpenFileForWrite(state.path, ec, fs::CD_CreateNew, fs::OF_None);
#else
state.f =
fs::OpenFileForWrite(state.path, ec, fs::CD_CreateNew, fs::OF_Append);
#endif
if (ec) {
WPI_ERROR(m_msglog, "Could not open log file '{}': {}",
state.path.string(), ec.message());
// try again with random filename
state.SetFilename(MakeRandomFilename());
} else {
break;
}
}
if (state.f == fs::kInvalidFile) {
WPI_ERROR(m_msglog, "Could not open log file, no log being saved");
} else {
WPI_INFO(m_msglog, "Logging to '{}' ({} free space)", state.path.string(),
FormatBytesSize(state.freeSpace));
}
}
// start file
if (state.f != fs::kInvalidFile) {
StartFile();
}
}
void DataLogBackgroundWriter::WriterThreadMain(std::string_view dir) {
std::chrono::duration<double> periodTime{m_period};
WriterThreadState state{dir};
{
std::scoped_lock lock{m_mutex};
state.SetFilename(m_newFilename);
m_newFilename.clear();
}
StartLogFile(state);
std::error_code ec;
std::vector<DataLog::Buffer> toWrite;
int freeSpaceCount = 0;
int checkExistCount = 0;
bool blocked = false;
uintmax_t written = 0;
std::unique_lock lock{m_mutex};
do {
bool doFlush = false;
auto timeoutTime = std::chrono::steady_clock::now() + periodTime;
if (m_cond.wait_until(lock, timeoutTime) == std::cv_status::timeout) {
doFlush = true;
}
if (m_state == kStopped) {
state.Close();
continue;
}
bool doStart = false;
// if file was deleted, recreate it with the same name
if (++checkExistCount >= 10) {
checkExistCount = 0;
lock.unlock();
bool exists = fs::exists(state.path, ec);
lock.lock();
if (!ec && !exists) {
state.Close();
state.IncrementFilename();
WPI_INFO(m_msglog, "Log file deleted, recreating as fresh log '{}'",
state.filename);
doStart = true;
}
}
// start new file if file exceeds 1.8 GB
if (written > 1800000000ull) {
state.Close();
state.IncrementFilename();
WPI_INFO(m_msglog, "Log file reached 1.8 GB, starting new file '{}'",
state.filename);
doStart = true;
}
if (m_state == kStart || doStart) {
lock.unlock();
DataLog::Stop();
StartLogFile(state);
lock.lock();
if (m_state == kStopped) {
continue;
}
m_state = kActive;
written = 0;
}
if (!m_newFilename.empty() && state.f != fs::kInvalidFile) {
auto newFilename = std::move(m_newFilename);
m_newFilename.clear();
// rename
if (state.filename != newFilename) {
lock.unlock();
fs::rename(state.path, state.dirPath / newFilename, ec);
lock.lock();
}
if (ec) {
WPI_ERROR(m_msglog, "Could not rename log file from '{}' to '{}': {}",
state.filename, newFilename, ec.message());
} else {
WPI_INFO(m_msglog, "Renamed log file from '{}' to '{}'", state.filename,
newFilename);
}
state.SetFilename(newFilename);
}
if (doFlush || m_doFlush) {
// flush to file
m_doFlush = false;
DataLog::FlushBufs(&toWrite);
if (toWrite.empty()) {
continue;
}
if (state.f != fs::kInvalidFile && !blocked) {
lock.unlock();
// update free space every 10 flushes (in case other things are writing)
if (++freeSpaceCount >= 10) {
freeSpaceCount = 0;
auto freeSpaceInfo = fs::space(state.dirPath, ec);
if (!ec) {
state.freeSpace = freeSpaceInfo.available;
} else {
state.freeSpace = UINTMAX_MAX;
}
}
// write buffers to file
for (auto&& buf : toWrite) {
// stop writing when we go below the minimum free space
state.freeSpace -= buf.GetData().size();
written += buf.GetData().size();
if (state.freeSpace < kMinFreeSpace) {
[[unlikely]] WPI_ERROR(
m_msglog,
"Stopped logging due to low free space ({} available)",
FormatBytesSize(state.freeSpace));
blocked = true;
break;
}
WriteToFile(state.f, buf.GetData(), state.filename, m_msglog);
}
// sync to storage
#if defined(__linux__)
::fdatasync(state.f);
#elif defined(__APPLE__)
::fsync(state.f);
#endif
lock.lock();
if (blocked) {
[[unlikely]] m_state = kPaused;
}
}
// release buffers back to free list
ReleaseBufs(&toWrite);
}
} while (!m_shutdown);
}
void DataLogBackgroundWriter::WriterThreadMain(
std::function<void(std::span<const uint8_t> data)> write) {
std::chrono::duration<double> periodTime{m_period};
StartFile();
std::vector<DataLog::Buffer> toWrite;
std::unique_lock lock{m_mutex};
do {
bool doFlush = false;
auto timeoutTime = std::chrono::steady_clock::now() + periodTime;
if (m_cond.wait_until(lock, timeoutTime) == std::cv_status::timeout) {
doFlush = true;
}
if (doFlush || m_doFlush) {
// flush to file
m_doFlush = false;
DataLog::FlushBufs(&toWrite);
if (toWrite.empty()) {
continue;
}
lock.unlock();
// write buffers
for (auto&& buf : toWrite) {
if (!buf.GetData().empty()) {
write(buf.GetData());
}
}
lock.lock();
// release buffers back to free list
ReleaseBufs(&toWrite);
}
} while (!m_shutdown);
write({}); // indicate EOF
}
extern "C" {
struct WPI_DataLog* WPI_DataLog_CreateBackgroundWriter(
const char* dir, const char* filename, double period,
const char* extraHeader) {
return reinterpret_cast<WPI_DataLog*>(
new DataLogBackgroundWriter{dir, filename, period, extraHeader});
}
struct WPI_DataLog* WPI_DataLog_CreateBackgroundWriter_Func(
void (*write)(void* ptr, const uint8_t* data, size_t len), void* ptr,
double period, const char* extraHeader) {
return reinterpret_cast<WPI_DataLog*>(new DataLogBackgroundWriter{
[=](auto data) { write(ptr, data.data(), data.size()); }, period,
extraHeader});
}
void WPI_DataLog_SetBackgroundWriterFilename(struct WPI_DataLog* datalog,
const char* filename) {
reinterpret_cast<DataLogBackgroundWriter*>(datalog)->SetFilename(filename);
}
} // extern "C"

76
wpiutil/src/main/native/cpp/DataLogWriter.cpp Normal file
View File

@@ -0,0 +1,76 @@
// Copyright (c) FIRST and other WPILib contributors.
// Open Source Software; you can modify and/or share it under the terms of
// the WPILib BSD license file in the root directory of this project.
#include "wpi/DataLogWriter.h"
#include "wpi/raw_ostream.h"
using namespace wpi::log;
DataLogWriter::DataLogWriter(std::string_view filename, std::error_code& ec,
std::string_view extraHeader)
: DataLogWriter{s_defaultMessageLog, filename, ec, extraHeader} {}
DataLogWriter::DataLogWriter(wpi::Logger& msglog, std::string_view filename,
std::error_code& ec, std::string_view extraHeader)
: DataLogWriter{msglog, std::make_unique<raw_fd_ostream>(filename, ec),
extraHeader} {
if (ec) {
Stop();
}
}
DataLogWriter::DataLogWriter(std::unique_ptr<wpi::raw_ostream> os,
std::string_view extraHeader)
: DataLogWriter{s_defaultMessageLog, std::move(os), extraHeader} {}
DataLogWriter::DataLogWriter(wpi::Logger& msglog,
std::unique_ptr<wpi::raw_ostream> os,
std::string_view extraHeader)
: DataLog{msglog, extraHeader}, m_os{std::move(os)} {
StartFile();
}
DataLogWriter::~DataLogWriter() {
if (m_os) {
Flush();
}
}
void DataLogWriter::Flush() {
if (!m_os) {
return;
}
std::vector<Buffer> writeBufs;
FlushBufs(&writeBufs);
for (auto&& buf : writeBufs) {
(*m_os) << buf.GetData();
}
ReleaseBufs(&writeBufs);
}
void DataLogWriter::Stop() {
DataLog::Stop();
Flush();
m_os.reset();
}
bool DataLogWriter::BufferFull() {
Flush();
return false;
}
extern "C" {
struct WPI_DataLog* WPI_DataLog_CreateWriter(const char* filename,
int* errorCode,
const char* extraHeader) {
std::error_code ec;
auto rv = reinterpret_cast<WPI_DataLog*>(
new DataLogWriter{filename, ec, extraHeader});
*errorCode = ec.value();
return rv;
}
} // extern "C"

57
wpiutil/src/main/native/cpp/jni/DataLogJNI.cpp
View File

@@ -4,11 +4,15 @@
#include <jni.h>
#include <vector>
#include <fmt/format.h>
#include "WPIUtilJNI.h"
#include "edu_wpi_first_util_datalog_DataLogJNI.h"
#include "wpi/DataLog.h"
#include "wpi/DataLogBackgroundWriter.h"
#include "wpi/DataLogWriter.h"
#include "wpi/jni_util.h"
using namespace wpi::java;
@@ -18,11 +22,11 @@ extern "C" {
/*
* Class: edu_wpi_first_util_datalog_DataLogJNI
* Method: create
* Method: bgCreate
* Signature: (Ljava/lang/String;Ljava/lang/String;DLjava/lang/String;)J
*/
JNIEXPORT jlong JNICALL
Java_edu_wpi_first_util_datalog_DataLogJNI_create
Java_edu_wpi_first_util_datalog_DataLogJNI_bgCreate
(JNIEnv* env, jclass, jstring dir, jstring filename, jdouble period,
jstring extraHeader)
{
@@ -38,18 +42,18 @@ Java_edu_wpi_first_util_datalog_DataLogJNI_create
wpi::ThrowNullPointerException(env, "extraHeader is null");
return 0;
}
return reinterpret_cast<jlong>(new DataLog{JStringRef{env, dir},
JStringRef{env, filename}, period,
JStringRef{env, extraHeader}});
return reinterpret_cast<jlong>(new DataLogBackgroundWriter{
JStringRef{env, dir}, JStringRef{env, filename}, period,
JStringRef{env, extraHeader}});
}
/*
* Class: edu_wpi_first_util_datalog_DataLogJNI
* Method: setFilename
* Method: bgSetFilename
* Signature: (JLjava/lang/String;)V
*/
JNIEXPORT void JNICALL
Java_edu_wpi_first_util_datalog_DataLogJNI_setFilename
Java_edu_wpi_first_util_datalog_DataLogJNI_bgSetFilename
(JNIEnv* env, jclass, jlong impl, jstring filename)
{
if (impl == 0) {
@@ -60,7 +64,36 @@ Java_edu_wpi_first_util_datalog_DataLogJNI_setFilename
wpi::ThrowNullPointerException(env, "filename is null");
return;
}
reinterpret_cast<DataLog*>(impl)->SetFilename(JStringRef{env, filename});
reinterpret_cast<DataLogBackgroundWriter*>(impl)->SetFilename(
JStringRef{env, filename});
}
/*
* Class: edu_wpi_first_util_datalog_DataLogJNI
* Method: fgCreate
* Signature: (Ljava/lang/String;Ljava/lang/String;)J
*/
JNIEXPORT jlong JNICALL
Java_edu_wpi_first_util_datalog_DataLogJNI_fgCreate
(JNIEnv* env, jclass, jstring filename, jstring extraHeader)
{
if (!filename) {
wpi::ThrowNullPointerException(env, "filename is null");
return 0;
}
if (!extraHeader) {
wpi::ThrowNullPointerException(env, "extraHeader is null");
return 0;
}
std::error_code ec;
auto writer = new DataLogWriter{JStringRef{env, filename}, ec,
JStringRef{env, extraHeader}};
if (ec) {
wpi::ThrowIOException(env, ec.message());
delete writer;
return 0;
}
return reinterpret_cast<jlong>(writer);
}
/*
@@ -76,7 +109,7 @@ Java_edu_wpi_first_util_datalog_DataLogJNI_flush
wpi::ThrowNullPointerException(env, "impl is null");
return;
}
reinterpret_cast<DataLog*>(impl)->Flush();
reinterpret_cast<DataLogBackgroundWriter*>(impl)->Flush();
}
/*
@@ -92,7 +125,7 @@ Java_edu_wpi_first_util_datalog_DataLogJNI_pause
wpi::ThrowNullPointerException(env, "impl is null");
return;
}
reinterpret_cast<DataLog*>(impl)->Pause();
reinterpret_cast<DataLogBackgroundWriter*>(impl)->Pause();
}
/*
@@ -108,7 +141,7 @@ Java_edu_wpi_first_util_datalog_DataLogJNI_resume
wpi::ThrowNullPointerException(env, "impl is null");
return;
}
reinterpret_cast<DataLog*>(impl)->Resume();
reinterpret_cast<DataLogBackgroundWriter*>(impl)->Resume();
}
/*
@@ -124,7 +157,7 @@ Java_edu_wpi_first_util_datalog_DataLogJNI_stop
wpi::ThrowNullPointerException(env, "impl is null");
return;
}
reinterpret_cast<DataLog*>(impl)->Stop();
reinterpret_cast<DataLogBackgroundWriter*>(impl)->Stop();
}
/*

6
wpiutil/src/main/native/cpp/jni/WPIUtilJNI.cpp
View File

@@ -21,12 +21,14 @@ static uint64_t mockNow = 0;
static JException illegalArgEx;
static JException indexOobEx;
static JException interruptedEx;
static JException ioEx;
static JException nullPointerEx;
static const JExceptionInit exceptions[] = {
{"java/lang/IllegalArgumentException", &illegalArgEx},
{"java/lang/IndexOutOfBoundsException", &indexOobEx},
{"java/lang/InterruptedException", &interruptedEx},
{"java/io/IOException", &ioEx},
{"java/lang/NullPointerException", &nullPointerEx}};
void wpi::ThrowIllegalArgumentException(JNIEnv* env, std::string_view msg) {
@@ -37,6 +39,10 @@ void wpi::ThrowIndexOobException(JNIEnv* env, std::string_view msg) {
indexOobEx.Throw(env, msg);
}
void wpi::ThrowIOException(JNIEnv* env, std::string_view msg) {
ioEx.Throw(env, msg);
}
void wpi::ThrowNullPointerException(JNIEnv* env, std::string_view msg) {
nullPointerEx.Throw(env, msg);
}

1
wpiutil/src/main/native/cpp/jni/WPIUtilJNI.h
View File

@@ -12,6 +12,7 @@ namespace wpi {
void ThrowIllegalArgumentException(JNIEnv* env, std::string_view msg);
void ThrowIndexOobException(JNIEnv* env, std::string_view msg);
void ThrowIOException(JNIEnv* env, std::string_view msg);
void ThrowNullPointerException(JNIEnv* env, std::string_view msg);
} // namespace wpi

529
wpiutil/src/main/native/include/wpi/DataLog.h
View File

@@ -6,43 +6,25 @@
#include <stdint.h>
#ifdef __cplusplus
#include <concepts>
#include <functional>
#include <initializer_list>
#include <memory>
#include <ranges>
#include <span>
#include <string>
#include <string_view>
#include <thread>
#include <tuple>
#include <utility>
#include <vector>
#include <version>
#include "wpi/DataLog_c.h"
#include "wpi/DenseMap.h"
#include "wpi/SmallVector.h"
#include "wpi/StringMap.h"
#include "wpi/condition_variable.h"
#include "wpi/mutex.h"
#include "wpi/protobuf/Protobuf.h"
#include "wpi/struct/Struct.h"
#include "wpi/timestamp.h"
#endif // __cplusplus
/**
* A datalog string (for use with string array).
*/
struct WPI_DataLog_String {
/** Contents. */
const char* str;
/** Length. */
size_t len;
};
#ifdef __cplusplus
namespace wpi {
class Logger;
@@ -61,16 +43,11 @@ enum ControlRecordType {
} // namespace impl
/**
* A data log. The log file is created immediately upon construction with a
* temporary filename. The file may be renamed at any time using the
* SetFilename() function.
* A data log for high-speed writing of data values.
*
* The lifetime of the data log object must be longer than any data log entry
* objects that refer to it.
*
* The data log is periodically flushed to disk. It can also be explicitly
* flushed to disk by using the Flush() function.
*
* Finish() is needed only to indicate in the log that a particular entry is
* no longer being used (it releases the name to ID mapping). Finish() is not
* required to be called for data to be flushed to disk; entries in the log
@@ -87,105 +64,36 @@ enum ControlRecordType {
* arbitrary values), records in the log are not guaranteed to be sorted by
* timestamp.
*/
class DataLog final {
class DataLog {
public:
/**
* Construct a new Data Log. The log will be initially created with a
* temporary filename.
*
* @param dir directory to store the log
* @param filename filename to use; if none provided, a random filename is
* generated of the form "wpilog_{}.wpilog"
* @param period time between automatic flushes to disk, in seconds;
* this is a time/storage tradeoff
* @param extraHeader extra header data
*/
explicit DataLog(std::string_view dir = "", std::string_view filename = "",
double period = 0.25, std::string_view extraHeader = "");
virtual ~DataLog() = default;
/**
* Construct a new Data Log. The log will be initially created with a
* temporary filename.
*
* @param msglog message logger (will be called from separate thread)
* @param dir directory to store the log
* @param filename filename to use; if none provided, a random filename is
* generated of the form "wpilog_{}.wpilog"
* @param period time between automatic flushes to disk, in seconds;
* this is a time/storage tradeoff
* @param extraHeader extra header data
*/
explicit DataLog(wpi::Logger& msglog, std::string_view dir = "",
std::string_view filename = "", double period = 0.25,
std::string_view extraHeader = "");
/**
* Construct a new Data Log that passes its output to the provided function
* rather than a file. The write function will be called on a separate
* background thread and may block. The write function is called with an
* empty data array when the thread is terminating.
*
* @param write write function
* @param period time between automatic calls to write, in seconds;
* this is a time/storage tradeoff
* @param extraHeader extra header data
*/
explicit DataLog(std::function<void(std::span<const uint8_t> data)> write,
double period = 0.25, std::string_view extraHeader = "");
/**
* Construct a new Data Log that passes its output to the provided function
* rather than a file. The write function will be called on a separate
* background thread and may block. The write function is called with an
* empty data array when the thread is terminating.
*
* @param msglog message logger (will be called from separate thread)
* @param write write function
* @param period time between automatic calls to write, in seconds;
* this is a time/storage tradeoff
* @param extraHeader extra header data
*/
explicit DataLog(wpi::Logger& msglog,
std::function<void(std::span<const uint8_t> data)> write,
double period = 0.25, std::string_view extraHeader = "");
~DataLog();
DataLog(const DataLog&) = delete;
DataLog& operator=(const DataLog&) = delete;
DataLog(DataLog&&) = delete;
DataLog& operator=(const DataLog&&) = delete;
/**
* Change log filename.
*
* @param filename filename
*/
void SetFilename(std::string_view filename);
/**
* Explicitly flushes the log data to disk.
*/
void Flush();
virtual void Flush() = 0;
/**
* Pauses appending of data records to the log. While paused, no data records
* are saved (e.g. AppendX is a no-op). Has no effect on entry starts /
* finishes / metadata changes.
*/
void Pause();
virtual void Pause();
/**
* Resumes appending of data records to the log. If called after Stop(),
* opens a new file (with random name if SetFilename was not called after
* Stop()) and appends Start records and schema data values for all previously
* started entries and schemas.
* Resumes appending of data records to the log.
*/
void Resume();
virtual void Resume();
/**
* Stops appending all records to the log, and closes the log file.
* Stops appending start/metadata/schema records to the log.
*/
void Stop();
virtual void Stop();
/**
* Returns whether there is a data schema already registered with the given
@@ -468,13 +376,116 @@ class DataLog final {
void AppendStringArray(int entry, std::span<const WPI_DataLog_String> arr,
int64_t timestamp);
private:
struct WriterThreadState;
protected:
static constexpr size_t kBlockSize = 16 * 1024;
static wpi::Logger s_defaultMessageLog;
void StartLogFile(WriterThreadState& state);
void WriterThreadMain(std::string_view dir);
void WriterThreadMain(
std::function<void(std::span<const uint8_t> data)> write);
class Buffer {
public:
explicit Buffer(size_t alloc = kBlockSize)
: m_buf{new uint8_t[alloc]}, m_maxLen{alloc} {}
~Buffer() { delete[] m_buf; }
Buffer(const Buffer&) = delete;
Buffer& operator=(const Buffer&) = delete;
Buffer(Buffer&& oth)
: m_buf{oth.m_buf}, m_len{oth.m_len}, m_maxLen{oth.m_maxLen} {
oth.m_buf = nullptr;
oth.m_len = 0;
oth.m_maxLen = 0;
}
Buffer& operator=(Buffer&& oth) {
if (m_buf) {
delete[] m_buf;
}
m_buf = oth.m_buf;
m_len = oth.m_len;
m_maxLen = oth.m_maxLen;
oth.m_buf = nullptr;
oth.m_len = 0;
oth.m_maxLen = 0;
return *this;
}
uint8_t* Reserve(size_t size) {
assert(size <= GetRemaining());
uint8_t* rv = m_buf + m_len;
m_len += size;
return rv;
}
void Unreserve(size_t size) { m_len -= size; }
void Clear() { m_len = 0; }
size_t GetRemaining() const { return m_maxLen - m_len; }
std::span<uint8_t> GetData() { return {m_buf, m_len}; }
std::span<const uint8_t> GetData() const { return {m_buf, m_len}; }
private:
uint8_t* m_buf;
size_t m_len = 0;
size_t m_maxLen;
};
/**
* Constructs the log. StartFile() must be called to actually start the
* file output.
*
* @param msglog message logger (will be called from separate thread)
* @param extraHeader extra header metadata
*/
explicit DataLog(wpi::Logger& msglog, std::string_view extraHeader = "")
: m_msglog{msglog}, m_extraHeader{extraHeader} {}
/**
* Starts the log. Appends file header and Start records and schema data
* values for all previously started entries and schemas. No effect unless
* the data log is currently stopped.
*/
void StartFile();
/**
* Provides complete set of all buffers that need to be written.
*
* Any existing contents of writeBufs will be released as if ReleaseBufs()
* was called prior to this call.
*
* Returned buffers should provided back via ReleaseBufs() after the write is
* complete.
*
* @param writeBufs buffers to be written (output)
*/
void FlushBufs(std::vector<Buffer>* writeBufs);
/**
* Releases memory for a set of buffers back to the internal buffer pool.
*
* @param bufs buffers; empty on return
*/
void ReleaseBufs(std::vector<Buffer>* bufs);
/**
* Called when internal buffers are half the maximum count. Called with
* internal mutex held; do not call any other DataLog functions from this
* function.
*/
virtual void BufferHalfFull();
/**
* Called when internal buffers reach the maximum count. Called with internal
* mutex held; do not call any other DataLog functions from this function.
*
* @return true if log should be paused (don't call PauseLog)
*/
virtual bool BufferFull() = 0;
private:
static constexpr size_t kMaxBufferCount = 1024 * 1024 / kBlockSize;
static constexpr size_t kMaxFreeCount = 256 * 1024 / kBlockSize;
// must be called with m_mutex held
int StartImpl(std::string_view name, std::string_view type,
@@ -486,22 +497,16 @@ class DataLog final {
void AppendStringImpl(std::string_view str);
void AppendStartRecord(int id, std::string_view name, std::string_view type,
std::string_view metadata, int64_t timestamp);
void DoReleaseBufs(std::vector<Buffer>* bufs);
protected:
wpi::Logger& m_msglog;
private:
mutable wpi::mutex m_mutex;
wpi::condition_variable m_cond;
bool m_doFlush{false};
bool m_shutdown{false};
enum State {
kStart,
kActive,
kPaused,
kStopped,
} m_state = kActive;
double m_period;
bool m_active = false;
bool m_paused = false;
std::string m_extraHeader;
std::string m_newFilename;
class Buffer;
std::vector<Buffer> m_free;
std::vector<Buffer> m_outgoing;
struct EntryInfo {
@@ -516,7 +521,6 @@ class DataLog final {
};
wpi::DenseMap<int, EntryInfo2> m_entryIds;
int m_lastId = 0;
std::thread m_thread;
};
/**
@@ -1103,286 +1107,3 @@ class ProtobufLogEntry : public DataLogEntry {
};
} // namespace wpi::log
extern "C" {
#endif // __cplusplus
/** C-compatible data log (opaque struct). */
struct WPI_DataLog;
/**
* Construct a new Data Log. The log will be initially created with a
* temporary filename.
*
* @param dir directory to store the log
* @param filename filename to use; if none provided, a random filename is
* generated of the form "wpilog_{}.wpilog"
* @param period time between automatic flushes to disk, in seconds;
* this is a time/storage tradeoff
* @param extraHeader extra header data
*/
struct WPI_DataLog* WPI_DataLog_Create(const char* dir, const char* filename,
double period, const char* extraHeader);
/**
* Construct a new Data Log that passes its output to the provided function
* rather than a file. The write function will be called on a separate
* background thread and may block. The write function is called with an
* empty data array (data=NULL, len=0) when the thread is terminating.
*
* @param write write function
* @param ptr pointer to pass to write function ptr parameter
* @param period time between automatic calls to write, in seconds;
* this is a time/storage tradeoff
* @param extraHeader extra header data
*/
struct WPI_DataLog* WPI_DataLog_Create_Func(
void (*write)(void* ptr, const uint8_t* data, size_t len), void* ptr,
double period, const char* extraHeader);
/**
* Releases a data log object. Closes the file and returns resources to the
* system.
*
* @param datalog data log
*/
void WPI_DataLog_Release(struct WPI_DataLog* datalog);
/**
* Change log filename.
*
* @param datalog data log
* @param filename filename
*/
void WPI_DataLog_SetFilename(struct WPI_DataLog* datalog, const char* filename);
/**
* Explicitly flushes the log data to disk.
*
* @param datalog data log
*/
void WPI_DataLog_Flush(struct WPI_DataLog* datalog);
/**
* Pauses appending of data records to the log. While paused, no data records
* are saved (e.g. AppendX is a no-op). Has no effect on entry starts /
* finishes / metadata changes.
*
* @param datalog data log
*/
void WPI_DataLog_Pause(struct WPI_DataLog* datalog);
/**
* Resumes appending of data records to the log. If called after Stop(),
* opens a new file (with random name if SetFilename was not called after
* Stop()) and appends Start records and schema data values for all previously
* started entries and schemas.
*
* @param datalog data log
*/
void WPI_DataLog_Resume(struct WPI_DataLog* datalog);
/**
* Stops appending all records to the log, and closes the log file.
*
* @param datalog data log
*/
void WPI_DataLog_Stop(struct WPI_DataLog* datalog);
/**
* Start an entry. Duplicate names are allowed (with the same type), and
* result in the same index being returned (Start/Finish are reference
* counted). A duplicate name with a different type will result in an error
* message being printed to the console and 0 being returned (which will be
* ignored by the Append functions).
*
* @param datalog data log
* @param name Name
* @param type Data type
* @param metadata Initial metadata (e.g. data properties)
* @param timestamp Time stamp (may be 0 to indicate now)
*
* @return Entry index
*/
int WPI_DataLog_Start(struct WPI_DataLog* datalog, const char* name,
const char* type, const char* metadata,
int64_t timestamp);
/**
* Finish an entry.
*
* @param datalog data log
* @param entry Entry index
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_Finish(struct WPI_DataLog* datalog, int entry,
int64_t timestamp);
/**
* Updates the metadata for an entry.
*
* @param datalog data log
* @param entry Entry index
* @param metadata New metadata for the entry
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_SetMetadata(struct WPI_DataLog* datalog, int entry,
const char* metadata, int64_t timestamp);
/**
* Appends a raw record to the log.
*
* @param datalog data log
* @param entry Entry index, as returned by WPI_DataLog_Start()
* @param data Byte array to record
* @param len Length of byte array
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_AppendRaw(struct WPI_DataLog* datalog, int entry,
const uint8_t* data, size_t len, int64_t timestamp);
/**
* Appends a boolean record to the log.
*
* @param datalog data log
* @param entry Entry index, as returned by WPI_DataLog_Start()
* @param value Boolean value to record
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_AppendBoolean(struct WPI_DataLog* datalog, int entry,
int value, int64_t timestamp);
/**
* Appends an integer record to the log.
*
* @param datalog data log
* @param entry Entry index, as returned by WPI_DataLog_Start()
* @param value Integer value to record
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_AppendInteger(struct WPI_DataLog* datalog, int entry,
int64_t value, int64_t timestamp);
/**
* Appends a float record to the log.
*
* @param datalog data log
* @param entry Entry index, as returned by WPI_DataLog_Start()
* @param value Float value to record
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_AppendFloat(struct WPI_DataLog* datalog, int entry,
float value, int64_t timestamp);
/**
* Appends a double record to the log.
*
* @param datalog data log
* @param entry Entry index, as returned by WPI_DataLog_Start()
* @param value Double value to record
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_AppendDouble(struct WPI_DataLog* datalog, int entry,
double value, int64_t timestamp);
/**
* Appends a string record to the log.
*
* @param datalog data log
* @param entry Entry index, as returned by WPI_DataLog_Start()
* @param value String value to record
* @param len Length of string
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_AppendString(struct WPI_DataLog* datalog, int entry,
const char* value, size_t len, int64_t timestamp);
/**
* Appends a boolean array record to the log.
*
* @param datalog data log
* @param entry Entry index, as returned by WPI_DataLog_Start()
* @param arr Boolean array to record
* @param len Number of elements in array
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_AppendBooleanArray(struct WPI_DataLog* datalog, int entry,
const int* arr, size_t len,
int64_t timestamp);
/**
* Appends a boolean array record to the log.
*
* @param datalog data log
* @param entry Entry index, as returned by WPI_DataLog_Start()
* @param arr Boolean array to record
* @param len Number of elements in array
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_AppendBooleanArrayByte(struct WPI_DataLog* datalog, int entry,
const uint8_t* arr, size_t len,
int64_t timestamp);
/**
* Appends an integer array record to the log.
*
* @param datalog data log
* @param entry Entry index, as returned by WPI_DataLog_Start()
* @param arr Integer array to record
* @param len Number of elements in array
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_AppendIntegerArray(struct WPI_DataLog* datalog, int entry,
const int64_t* arr, size_t len,
int64_t timestamp);
/**
* Appends a float array record to the log.
*
* @param datalog data log
* @param entry Entry index, as returned by WPI_DataLog_Start()
* @param arr Float array to record
* @param len Number of elements in array
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_AppendFloatArray(struct WPI_DataLog* datalog, int entry,
const float* arr, size_t len,
int64_t timestamp);
/**
* Appends a double array record to the log.
*
* @param datalog data log
* @param entry Entry index, as returned by WPI_DataLog_Start()
* @param arr Double array to record
* @param len Number of elements in array
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_AppendDoubleArray(struct WPI_DataLog* datalog, int entry,
const double* arr, size_t len,
int64_t timestamp);
/**
* Appends a string array record to the log.
*
* @param datalog data log
* @param entry Entry index, as returned by WPI_DataLog_Start()
* @param arr String array to record
* @param len Number of elements in array
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_AppendStringArray(struct WPI_DataLog* datalog, int entry,
const WPI_DataLog_String* arr, size_t len,
int64_t timestamp);
void WPI_DataLog_AddSchemaString(struct WPI_DataLog* datalog, const char* name,
const char* type, const char* schema,
int64_t timestamp);
void WPI_DataLog_AddSchema(struct WPI_DataLog* datalog, const char* name,
const char* type, const uint8_t* schema,
size_t schema_len, int64_t timestamp);
#ifdef __cplusplus
} // extern "C"
#endif // __cplusplus

170
wpiutil/src/main/native/include/wpi/DataLogBackgroundWriter.h Normal file
View File

@@ -0,0 +1,170 @@
// Copyright (c) FIRST and other WPILib contributors.
// Open Source Software; you can modify and/or share it under the terms of
// the WPILib BSD license file in the root directory of this project.
#pragma once
#include <stdint.h>
#include <functional>
#include <span>
#include <string>
#include <string_view>
#include <thread>
#include "wpi/DataLog.h"
#include "wpi/condition_variable.h"
#include "wpi/mutex.h"
namespace wpi {
class Logger;
} // namespace wpi
namespace wpi::log {
/**
* A data log background writer that periodically flushes the data log on a
* background thread. The data log file is created immediately upon
* construction with a temporary filename. The file may be renamed at any time
* using the SetFilename() function.
*
* The lifetime of this object must be longer than any data log entry objects
* that refer to it.
*
* The data log is periodically flushed to disk. It can also be explicitly
* flushed to disk by using the Flush() function. This operation is, however,
* non-blocking.
*/
class DataLogBackgroundWriter final : public DataLog {
public:
/**
* Construct a new Data Log. The log will be initially created with a
* temporary filename.
*
* @param dir directory to store the log
* @param filename filename to use; if none provided, a random filename is
* generated of the form "wpilog_{}.wpilog"
* @param period time between automatic flushes to disk, in seconds;
* this is a time/storage tradeoff
* @param extraHeader extra header data
*/
explicit DataLogBackgroundWriter(std::string_view dir = "",
std::string_view filename = "",
double period = 0.25,
std::string_view extraHeader = "");
/**
* Construct a new Data Log. The log will be initially created with a
* temporary filename.
*
* @param msglog message logger (will be called from separate thread)
* @param dir directory to store the log
* @param filename filename to use; if none provided, a random filename is
* generated of the form "wpilog_{}.wpilog"
* @param period time between automatic flushes to disk, in seconds;
* this is a time/storage tradeoff
* @param extraHeader extra header data
*/
explicit DataLogBackgroundWriter(wpi::Logger& msglog,
std::string_view dir = "",
std::string_view filename = "",
double period = 0.25,
std::string_view extraHeader = "");
/**
* Construct a new Data Log that passes its output to the provided function
* rather than a file. The write function will be called on a separate
* background thread and may block. The write function is called with an
* empty data array when the thread is terminating.
*
* @param write write function
* @param period time between automatic calls to write, in seconds;
* this is a time/storage tradeoff
* @param extraHeader extra header data
*/
explicit DataLogBackgroundWriter(
std::function<void(std::span<const uint8_t> data)> write,
double period = 0.25, std::string_view extraHeader = "");
/**
* Construct a new Data Log that passes its output to the provided function
* rather than a file. The write function will be called on a separate
* background thread and may block. The write function is called with an
* empty data array when the thread is terminating.
*
* @param msglog message logger (will be called from separate thread)
* @param write write function
* @param period time between automatic calls to write, in seconds;
* this is a time/storage tradeoff
* @param extraHeader extra header data
*/
explicit DataLogBackgroundWriter(
wpi::Logger& msglog,
std::function<void(std::span<const uint8_t> data)> write,
double period = 0.25, std::string_view extraHeader = "");
~DataLogBackgroundWriter() final;
DataLogBackgroundWriter(const DataLogBackgroundWriter&) = delete;
DataLogBackgroundWriter& operator=(const DataLogBackgroundWriter&) = delete;
DataLogBackgroundWriter(DataLogBackgroundWriter&&) = delete;
DataLogBackgroundWriter& operator=(const DataLogBackgroundWriter&&) = delete;
/**
* Change log filename.
*
* @param filename filename
*/
void SetFilename(std::string_view filename);
/**
* Explicitly flushes the log data to disk.
*/
void Flush() final;
/**
* Pauses appending of data records to the log. While paused, no data records
* are saved (e.g. AppendX is a no-op). Has no effect on entry starts /
* finishes / metadata changes.
*/
void Pause() final;
/**
* Resumes appending of data records to the log. If called after Stop(),
* opens a new file (with random name if SetFilename was not called after
* Stop()) and appends Start records and schema data values for all previously
* started entries and schemas.
*/
void Resume() final;
/**
* Stops appending all records to the log, and closes the log file.
*/
void Stop() final;
private:
struct WriterThreadState;
void BufferHalfFull() final;
bool BufferFull() final;
void StartLogFile(WriterThreadState& state);
void WriterThreadMain(std::string_view dir);
void WriterThreadMain(
std::function<void(std::span<const uint8_t> data)> write);
mutable wpi::mutex m_mutex;
wpi::condition_variable m_cond;
bool m_doFlush{false};
bool m_shutdown{false};
enum State {
kStart,
kActive,
kPaused,
kStopped,
} m_state = kActive;
double m_period;
std::string m_newFilename;
std::thread m_thread;
};
} // namespace wpi::log

90
wpiutil/src/main/native/include/wpi/DataLogWriter.h Normal file
View File

@@ -0,0 +1,90 @@
// Copyright (c) FIRST and other WPILib contributors.
// Open Source Software; you can modify and/or share it under the terms of
// the WPILib BSD license file in the root directory of this project.
#pragma once
#include <memory>
#include <string_view>
#include <system_error>
#include "wpi/DataLog.h"
namespace wpi {
class raw_ostream;
class Logger;
} // namespace wpi
namespace wpi::log {
/**
* A data log writer that flushes the data log to a file when Flush() is called.
*
* The lifetime of this object must be longer than any data log entry objects
* that refer to it.
*/
class DataLogWriter final : public DataLog {
public:
/**
* Constructs with a filename.
*
* @param filename filename to use
* @param ec error code if failed to open file (output)
* @param extraHeader extra header data
*/
explicit DataLogWriter(std::string_view filename, std::error_code& ec,
std::string_view extraHeader = "");
/**
* Construct with a filename.
*
* @param msglog message logger
* @param filename filename to use
* @param ec error code if failed to open file (output)
* @param extraHeader extra header data
*/
DataLogWriter(wpi::Logger& msglog, std::string_view filename,
std::error_code& ec, std::string_view extraHeader = "");
/**
* Constructs with an output stream.
*
* @param os output stream
* @param extraHeader extra header data
*/
explicit DataLogWriter(std::unique_ptr<wpi::raw_ostream> os,
std::string_view extraHeader = "");
/**
* Constructs with an output stream.
*
* @param msglog message logger
* @param os output stream
* @param extraHeader extra header data
*/
DataLogWriter(wpi::Logger& msglog, std::unique_ptr<wpi::raw_ostream> os,
std::string_view extraHeader = "");
~DataLogWriter() final;
DataLogWriter(const DataLogWriter&) = delete;
DataLogWriter& operator=(const DataLogWriter&) = delete;
DataLogWriter(DataLogWriter&&) = delete;
DataLogWriter& operator=(const DataLogWriter&&) = delete;
/**
* Flushes the log data to disk.
*/
void Flush() final;
/**
* Stops appending all records to the log, and closes the log file.
*/
void Stop() final;
private:
bool BufferFull() final;
std::unique_ptr<wpi::raw_ostream> m_os;
};
} // namespace wpi::log

318
wpiutil/src/main/native/include/wpi/DataLog_c.h Normal file
View File

@@ -0,0 +1,318 @@
// Copyright (c) FIRST and other WPILib contributors.
// Open Source Software; you can modify and/or share it under the terms of
// the WPILib BSD license file in the root directory of this project.
#pragma once
#include <stddef.h> // NOLINT
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
/**
* A datalog string (for use with string array).
*/
struct WPI_DataLog_String {
/** Contents. */
const char* str;
/** Length. */
size_t len;
};
/** C-compatible data log (opaque struct). */
struct WPI_DataLog;
/**
* Construct a new Data Log.
*
* @param filename filename to use
* @param errorCode error if file failed to open (output)
* @param extraHeader extra header data
*/
struct WPI_DataLog* WPI_DataLog_CreateWriter(const char* filename,
int* errorCode,
const char* extraHeader);
/**
* Construct a new Data Log background writer. The log will be initially
* created with a temporary filename.
*
* @param dir directory to store the log
* @param filename filename to use; if none provided, a random filename is
* generated of the form "wpilog_{}.wpilog"
* @param period time between automatic flushes to disk, in seconds;
* this is a time/storage tradeoff
* @param extraHeader extra header data
*/
struct WPI_DataLog* WPI_DataLog_CreateBackgroundWriter(const char* dir,
const char* filename,
double period,
const char* extraHeader);
/**
* Construct a new Data Log background writer that passes its output to the
* provided function rather than a file. The write function will be called on a
* separate background thread and may block. The write function is called with
* an empty data array (data=NULL, len=0) when the thread is terminating.
*
* @param write write function
* @param ptr pointer to pass to write function ptr parameter
* @param period time between automatic calls to write, in seconds;
* this is a time/storage tradeoff
* @param extraHeader extra header data
*/
struct WPI_DataLog* WPI_DataLog_CreateBackgroundWriter_Func(
void (*write)(void* ptr, const uint8_t* data, size_t len), void* ptr,
double period, const char* extraHeader);
/**
* Change log filename. Can only be used on background writer data logs.
*
* @param datalog data log
* @param filename filename
*/
void WPI_DataLog_SetBackgroundWriterFilename(struct WPI_DataLog* datalog,
const char* filename);
/**
* Releases a data log object. Closes the file and returns resources to the
* system.
*
* @param datalog data log
*/
void WPI_DataLog_Release(struct WPI_DataLog* datalog);
/**
* Explicitly flushes the log data to disk.
*
* @param datalog data log
*/
void WPI_DataLog_Flush(struct WPI_DataLog* datalog);
/**
* Pauses appending of data records to the log. While paused, no data records
* are saved (e.g. AppendX is a no-op). Has no effect on entry starts /
* finishes / metadata changes.
*
* @param datalog data log
*/
void WPI_DataLog_Pause(struct WPI_DataLog* datalog);
/**
* Resumes appending of data records to the log. If called after Stop(),
* opens a new file (with random name if SetFilename was not called after
* Stop()) and appends Start records and schema data values for all previously
* started entries and schemas.
*
* @param datalog data log
*/
void WPI_DataLog_Resume(struct WPI_DataLog* datalog);
/**
* Stops appending all records to the log, and closes the log file.
*
* @param datalog data log
*/
void WPI_DataLog_Stop(struct WPI_DataLog* datalog);
/**
* Start an entry. Duplicate names are allowed (with the same type), and
* result in the same index being returned (Start/Finish are reference
* counted). A duplicate name with a different type will result in an error
* message being printed to the console and 0 being returned (which will be
* ignored by the Append functions).
*
* @param datalog data log
* @param name Name
* @param type Data type
* @param metadata Initial metadata (e.g. data properties)
* @param timestamp Time stamp (may be 0 to indicate now)
*
* @return Entry index
*/
int WPI_DataLog_Start(struct WPI_DataLog* datalog, const char* name,
const char* type, const char* metadata,
int64_t timestamp);
/**
* Finish an entry.
*
* @param datalog data log
* @param entry Entry index
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_Finish(struct WPI_DataLog* datalog, int entry,
int64_t timestamp);
/**
* Updates the metadata for an entry.
*
* @param datalog data log
* @param entry Entry index
* @param metadata New metadata for the entry
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_SetMetadata(struct WPI_DataLog* datalog, int entry,
const char* metadata, int64_t timestamp);
/**
* Appends a raw record to the log.
*
* @param datalog data log
* @param entry Entry index, as returned by WPI_DataLog_Start()
* @param data Byte array to record
* @param len Length of byte array
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_AppendRaw(struct WPI_DataLog* datalog, int entry,
const uint8_t* data, size_t len, int64_t timestamp);
/**
* Appends a boolean record to the log.
*
* @param datalog data log
* @param entry Entry index, as returned by WPI_DataLog_Start()
* @param value Boolean value to record
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_AppendBoolean(struct WPI_DataLog* datalog, int entry,
int value, int64_t timestamp);
/**
* Appends an integer record to the log.
*
* @param datalog data log
* @param entry Entry index, as returned by WPI_DataLog_Start()
* @param value Integer value to record
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_AppendInteger(struct WPI_DataLog* datalog, int entry,
int64_t value, int64_t timestamp);
/**
* Appends a float record to the log.
*
* @param datalog data log
* @param entry Entry index, as returned by WPI_DataLog_Start()
* @param value Float value to record
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_AppendFloat(struct WPI_DataLog* datalog, int entry,
float value, int64_t timestamp);
/**
* Appends a double record to the log.
*
* @param datalog data log
* @param entry Entry index, as returned by WPI_DataLog_Start()
* @param value Double value to record
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_AppendDouble(struct WPI_DataLog* datalog, int entry,
double value, int64_t timestamp);
/**
* Appends a string record to the log.
*
* @param datalog data log
* @param entry Entry index, as returned by WPI_DataLog_Start()
* @param value String value to record
* @param len Length of string
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_AppendString(struct WPI_DataLog* datalog, int entry,
const char* value, size_t len, int64_t timestamp);
/**
* Appends a boolean array record to the log.
*
* @param datalog data log
* @param entry Entry index, as returned by WPI_DataLog_Start()
* @param arr Boolean array to record
* @param len Number of elements in array
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_AppendBooleanArray(struct WPI_DataLog* datalog, int entry,
const int* arr, size_t len,
int64_t timestamp);
/**
* Appends a boolean array record to the log.
*
* @param datalog data log
* @param entry Entry index, as returned by WPI_DataLog_Start()
* @param arr Boolean array to record
* @param len Number of elements in array
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_AppendBooleanArrayByte(struct WPI_DataLog* datalog, int entry,
const uint8_t* arr, size_t len,
int64_t timestamp);
/**
* Appends an integer array record to the log.
*
* @param datalog data log
* @param entry Entry index, as returned by WPI_DataLog_Start()
* @param arr Integer array to record
* @param len Number of elements in array
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_AppendIntegerArray(struct WPI_DataLog* datalog, int entry,
const int64_t* arr, size_t len,
int64_t timestamp);
/**
* Appends a float array record to the log.
*
* @param datalog data log
* @param entry Entry index, as returned by WPI_DataLog_Start()
* @param arr Float array to record
* @param len Number of elements in array
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_AppendFloatArray(struct WPI_DataLog* datalog, int entry,
const float* arr, size_t len,
int64_t timestamp);
/**
* Appends a double array record to the log.
*
* @param datalog data log
* @param entry Entry index, as returned by WPI_DataLog_Start()
* @param arr Double array to record
* @param len Number of elements in array
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_AppendDoubleArray(struct WPI_DataLog* datalog, int entry,
const double* arr, size_t len,
int64_t timestamp);
/**
* Appends a string array record to the log.
*
* @param datalog data log
* @param entry Entry index, as returned by WPI_DataLog_Start()
* @param arr String array to record
* @param len Number of elements in array
* @param timestamp Time stamp (may be 0 to indicate now)
*/
void WPI_DataLog_AppendStringArray(struct WPI_DataLog* datalog, int entry,
const WPI_DataLog_String* arr, size_t len,
int64_t timestamp);
void WPI_DataLog_AddSchemaString(struct WPI_DataLog* datalog, const char* name,
const char* type, const char* schema,
int64_t timestamp);
void WPI_DataLog_AddSchema(struct WPI_DataLog* datalog, const char* name,
const char* type, const uint8_t* schema,
size_t schema_len, int64_t timestamp);
#ifdef __cplusplus
} // extern "C"
#endif

50
wpiutil/src/test/native/cpp/DataLogTest.cpp
View File

@@ -3,10 +3,13 @@
// the WPILib BSD license file in the root directory of this project.
#include <array>
#include <memory>
#include <gtest/gtest.h>
#include "wpi/DataLog.h"
#include "wpi/DataLogWriter.h"
#include "wpi/Logger.h"
#include "wpi/raw_ostream.h"
namespace {
struct ThingA {
@@ -118,19 +121,22 @@ static_assert(wpi::StructSerializable<ThingC>);
static_assert(wpi::StructSerializable<ThingC, Info1>);
static_assert(wpi::StructSerializable<ThingC, Info2>);
TEST(DataLogTest, SimpleInt) {
class DataLogTest : public ::testing::Test {
public:
wpi::Logger msglog;
std::vector<uint8_t> data;
{
wpi::log::DataLog log{
[&](auto out) { data.insert(data.end(), out.begin(), out.end()); }};
int entry = log.Start("test", "int64");
log.AppendInteger(entry, 1, 0);
}
wpi::log::DataLogWriter log{msglog,
std::make_unique<wpi::raw_uvector_ostream>(data)};
};
TEST_F(DataLogTest, SimpleInt) {
int entry = log.Start("test", "int64");
log.AppendInteger(entry, 1, 0);
log.Flush();
ASSERT_EQ(data.size(), 66u);
}
TEST(DataLogTest, StructA) {
wpi::log::DataLog log{[](auto) {}};
TEST_F(DataLogTest, StructA) {
[[maybe_unused]]
wpi::log::StructLogEntry<ThingA> entry0;
wpi::log::StructLogEntry<ThingA> entry{log, "a", 5};
@@ -138,8 +144,7 @@ TEST(DataLogTest, StructA) {
entry.Append(ThingA{}, 7);
}
TEST(DataLogTest, StructArrayA) {
wpi::log::DataLog log{[](auto) {}};
TEST_F(DataLogTest, StructArrayA) {
[[maybe_unused]]
wpi::log::StructArrayLogEntry<ThingA> entry0;
wpi::log::StructArrayLogEntry<ThingA> entry{log, "a", 5};
@@ -147,8 +152,7 @@ TEST(DataLogTest, StructArrayA) {
entry.Append({{ThingA{}, ThingA{}}}, 7);
}
TEST(DataLogTest, StructFixedArrayA) {
wpi::log::DataLog log{[](auto) {}};
TEST_F(DataLogTest, StructFixedArrayA) {
[[maybe_unused]]
wpi::log::StructArrayLogEntry<std::array<ThingA, 2>> entry0;
wpi::log::StructLogEntry<std::array<ThingA, 2>> entry{log, "a", 5};
@@ -157,8 +161,7 @@ TEST(DataLogTest, StructFixedArrayA) {
entry.Append(arr, 7);
}
TEST(DataLogTest, StructB) {
wpi::log::DataLog log{[](auto) {}};
TEST_F(DataLogTest, StructB) {
Info1 info;
[[maybe_unused]]
wpi::log::StructLogEntry<ThingB, Info1> entry0;
@@ -167,8 +170,7 @@ TEST(DataLogTest, StructB) {
entry.Append(ThingB{}, 7);
}
TEST(DataLogTest, StructArrayB) {
wpi::log::DataLog log{[](auto) {}};
TEST_F(DataLogTest, StructArrayB) {
Info1 info;
[[maybe_unused]]
wpi::log::StructArrayLogEntry<ThingB, Info1> entry0;
@@ -177,8 +179,7 @@ TEST(DataLogTest, StructArrayB) {
entry.Append({{ThingB{}, ThingB{}}}, 7);
}
TEST(DataLogTest, StructFixedArrayB) {
wpi::log::DataLog log{[](auto) {}};
TEST_F(DataLogTest, StructFixedArrayB) {
Info1 info;
wpi::log::StructLogEntry<std::array<ThingB, 2>, Info1> entry{log, "a", info,
5};
@@ -187,8 +188,7 @@ TEST(DataLogTest, StructFixedArrayB) {
entry.Append(arr, 7);
}
TEST(DataLogTest, StructC) {
wpi::log::DataLog log{[](auto) {}};
TEST_F(DataLogTest, StructC) {
{
wpi::log::StructLogEntry<ThingC> entry{log, "c", 5};
entry.Append(ThingC{});
@@ -208,8 +208,7 @@ TEST(DataLogTest, StructC) {
}
}
TEST(DataLogTest, StructArrayC) {
wpi::log::DataLog log{[](auto) {}};
TEST_F(DataLogTest, StructArrayC) {
{
wpi::log::StructArrayLogEntry<ThingC> entry{log, "c", 5};
entry.Append({{ThingC{}, ThingC{}}});
@@ -229,8 +228,7 @@ TEST(DataLogTest, StructArrayC) {
}
}
TEST(DataLogTest, StructFixedArrayC) {
wpi::log::DataLog log{[](auto) {}};
TEST_F(DataLogTest, StructFixedArrayC) {
std::array<ThingC, 2> arr;
{
wpi::log::StructLogEntry<std::array<ThingC, 2>> entry{log, "c", 5};