diff --git a/ntcoreffi/src/main/native/cpp/DataLogManager.cpp b/ntcoreffi/src/main/native/cpp/DataLogManager.cpp index 631ddd0f68..2692b8781b 100644 --- a/ntcoreffi/src/main/native/cpp/DataLogManager.cpp +++ b/ntcoreffi/src/main/native/cpp/DataLogManager.cpp @@ -7,11 +7,13 @@ #include #include #include +#include +#include #include #include #include -#include +#include #include #include #include @@ -193,7 +195,7 @@ struct Thread final : public wpi::SafeThread { std::string m_logDir; bool m_filenameOverride; - wpi::log::DataLog m_log; + wpi::log::DataLogBackgroundWriter m_log; bool m_ntLoggerEnabled = false; NT_DataLogger m_ntEntryLogger = 0; NT_ConnectionDataLogger m_ntConnLogger = 0; diff --git a/ntcoreffi/src/main/native/symbols.txt b/ntcoreffi/src/main/native/symbols.txt index 6700936804..2a09552940 100644 --- a/ntcoreffi/src/main/native/symbols.txt +++ b/ntcoreffi/src/main/native/symbols.txt @@ -236,16 +236,17 @@ WPI_DataLog_AppendIntegerArray WPI_DataLog_AppendRaw WPI_DataLog_AppendString WPI_DataLog_AppendStringArray -WPI_DataLog_Create -WPI_DataLog_Create_Func +WPI_DataLog_CreateBackgroundWriter +WPI_DataLog_CreateBackgroundWriter_Func +WPI_DataLog_CreateWriter WPI_DataLog_Finish +WPI_DataLog_SetMetadata +WPI_DataLog_Start WPI_DataLog_Flush WPI_DataLog_Pause WPI_DataLog_Release WPI_DataLog_Resume -WPI_DataLog_SetFilename -WPI_DataLog_SetMetadata -WPI_DataLog_Start +WPI_DataLog_SetBackgroundWriterFilename WPI_DataLog_Stop WPI_DestroyEvent WPI_DestroySemaphore diff --git a/wpilibc/src/main/native/cpp/DataLogManager.cpp b/wpilibc/src/main/native/cpp/DataLogManager.cpp index 8e5057a1d8..43870728d4 100644 --- a/wpilibc/src/main/native/cpp/DataLogManager.cpp +++ b/wpilibc/src/main/native/cpp/DataLogManager.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -39,7 +40,7 @@ struct Thread final : public wpi::SafeThread { std::string m_logDir; bool m_filenameOverride; - wpi::log::DataLog m_log; + wpi::log::DataLogBackgroundWriter m_log; bool m_ntLoggerEnabled = false; NT_DataLogger m_ntEntryLogger = 0; NT_ConnectionDataLogger m_ntConnLogger = 0; diff --git a/wpilibj/src/main/java/edu/wpi/first/wpilibj/DataLogManager.java b/wpilibj/src/main/java/edu/wpi/first/wpilibj/DataLogManager.java index 770db82716..8543b05cc1 100644 --- a/wpilibj/src/main/java/edu/wpi/first/wpilibj/DataLogManager.java +++ b/wpilibj/src/main/java/edu/wpi/first/wpilibj/DataLogManager.java @@ -8,6 +8,7 @@ import edu.wpi.first.networktables.NetworkTableInstance; import edu.wpi.first.util.WPIUtilJNI; import edu.wpi.first.util.concurrent.Event; import edu.wpi.first.util.datalog.DataLog; +import edu.wpi.first.util.datalog.DataLogBackgroundWriter; import edu.wpi.first.util.datalog.IntegerLogEntry; import edu.wpi.first.util.datalog.StringLogEntry; import java.io.File; @@ -40,7 +41,7 @@ import java.util.Random; *

By default, all NetworkTables value changes are stored to the data log. */ public final class DataLogManager { - private static DataLog m_log; + private static DataLogBackgroundWriter m_log; private static boolean m_stopped; private static String m_logDir; private static boolean m_filenameOverride; @@ -113,7 +114,7 @@ public final class DataLogManager { } } } - m_log = new DataLog(m_logDir, makeLogFilename(filename), period); + m_log = new DataLogBackgroundWriter(m_logDir, makeLogFilename(filename), period); m_messageLog = new StringLogEntry(m_log, "messages"); // Log all NT entries and connections diff --git a/wpiutil/examples/writelog/writelog.cpp b/wpiutil/examples/writelog/writelog.cpp index 309181aa47..0d8283ca12 100644 --- a/wpiutil/examples/writelog/writelog.cpp +++ b/wpiutil/examples/writelog/writelog.cpp @@ -8,7 +8,7 @@ #include #include -#include "wpi/DataLog.h" +#include "wpi/DataLogBackgroundWriter.h" #include "wpi/print.h" int main(int argc, char** argv) { @@ -22,7 +22,7 @@ int main(int argc, char** argv) { kNumRuns = std::stoi(argv[1]); } - wpi::log::DataLog log; + wpi::log::DataLogBackgroundWriter log; log.SetFilename("test.wpilog"); auto testVec = diff --git a/wpiutil/src/main/java/edu/wpi/first/util/datalog/DataLog.java b/wpiutil/src/main/java/edu/wpi/first/util/datalog/DataLog.java index f48b0815cf..bec3a7bde6 100644 --- a/wpiutil/src/main/java/edu/wpi/first/util/datalog/DataLog.java +++ b/wpiutil/src/main/java/edu/wpi/first/util/datalog/DataLog.java @@ -14,11 +14,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; /** - * A data log. The log file is created immediately upon construction with a temporary filename. The - * file may be renamed at any time using the setFilename() function. - * - *

The data log is periodically flushed to disk. It can also be explicitly flushed to disk by - * using the flush() function. + * A data log for high-speed writing of data values. * *

The finish() function is needed only to indicate in the log that a particular entry is no * longer being used (it releases the name to ID mapping). The finish() function is not required to @@ -31,66 +27,14 @@ import java.util.concurrent.ConcurrentMap; * For this reason (as well as the fact that timestamps can be set to arbitrary values), records in * the log are not guaranteed to be sorted by timestamp. */ -public final class DataLog implements AutoCloseable { +public class DataLog implements AutoCloseable { /** - * Construct a new Data Log. The log will be initially created with a temporary filename. + * Constructs. * - * @param dir directory to store the log - * @param filename filename to use; if none provided, a random filename is generated of the form - * "wpilog_{}.wpilog" - * @param period time between automatic flushes to disk, in seconds; this is a time/storage - * tradeoff - * @param extraHeader extra header data + * @param impl implementation handle */ - public DataLog(String dir, String filename, double period, String extraHeader) { - m_impl = DataLogJNI.create(dir, filename, period, extraHeader); - } - - /** - * Construct a new Data Log. The log will be initially created with a temporary filename. - * - * @param dir directory to store the log - * @param filename filename to use; if none provided, a random filename is generated of the form - * "wpilog_{}.wpilog" - * @param period time between automatic flushes to disk, in seconds; this is a time/storage - * tradeoff - */ - public DataLog(String dir, String filename, double period) { - this(dir, filename, period, ""); - } - - /** - * Construct a new Data Log. The log will be initially created with a temporary filename. - * - * @param dir directory to store the log - * @param filename filename to use; if none provided, a random filename is generated of the form - * "wpilog_{}.wpilog" - */ - public DataLog(String dir, String filename) { - this(dir, filename, 0.25); - } - - /** - * Construct a new Data Log. The log will be initially created with a temporary filename. - * - * @param dir directory to store the log - */ - public DataLog(String dir) { - this(dir, "", 0.25); - } - - /** Construct a new Data Log. The log will be initially created with a temporary filename. */ - public DataLog() { - this(""); - } - - /** - * Change log filename. - * - * @param filename filename - */ - public void setFilename(String filename) { - DataLogJNI.setFilename(m_impl, filename); + protected DataLog(long impl) { + m_impl = impl; } /** Explicitly flushes the log data to disk. */ @@ -106,11 +50,7 @@ public final class DataLog implements AutoCloseable { DataLogJNI.pause(m_impl); } - /** - * Resumes appending of data records to the log. If called after stop(), opens a new file (with - * random name if SetFilename was not called after stop()) and appends Start records and schema - * data values for all previously started entries and schemas. - */ + /** Resumes appending of data records to the log. */ public void resume() { DataLogJNI.resume(m_impl); } @@ -518,6 +458,8 @@ public final class DataLog implements AutoCloseable { seen.remove(typeString); } - private long m_impl; + /** Implementation handle. */ + protected long m_impl; + private final ConcurrentMap m_schemaMap = new ConcurrentHashMap<>(); } diff --git a/wpiutil/src/main/java/edu/wpi/first/util/datalog/DataLogBackgroundWriter.java b/wpiutil/src/main/java/edu/wpi/first/util/datalog/DataLogBackgroundWriter.java new file mode 100644 index 0000000000..18a6e5e63c --- /dev/null +++ b/wpiutil/src/main/java/edu/wpi/first/util/datalog/DataLogBackgroundWriter.java @@ -0,0 +1,86 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +package edu.wpi.first.util.datalog; + +/** + * A data log background writer that periodically flushes the data log on a background thread. The + * data log file is created immediately upon construction with a temporary filename. The file may be + * renamed at any time using the setFilename() function. + * + *

The data log is periodically flushed to disk. It can also be explicitly flushed to disk by + * using the flush() function. This operation is, however, non-blocking. + */ +public final class DataLogBackgroundWriter extends DataLog { + /** + * Construct a new Data Log. The log will be initially created with a temporary filename. + * + * @param dir directory to store the log + * @param filename filename to use; if none provided, a random filename is generated of the form + * "wpilog_{}.wpilog" + * @param period time between automatic flushes to disk, in seconds; this is a time/storage + * tradeoff + * @param extraHeader extra header data + */ + public DataLogBackgroundWriter(String dir, String filename, double period, String extraHeader) { + super(DataLogJNI.bgCreate(dir, filename, period, extraHeader)); + } + + /** + * Construct a new Data Log. The log will be initially created with a temporary filename. + * + * @param dir directory to store the log + * @param filename filename to use; if none provided, a random filename is generated of the form + * "wpilog_{}.wpilog" + * @param period time between automatic flushes to disk, in seconds; this is a time/storage + * tradeoff + */ + public DataLogBackgroundWriter(String dir, String filename, double period) { + this(dir, filename, period, ""); + } + + /** + * Construct a new Data Log. The log will be initially created with a temporary filename. + * + * @param dir directory to store the log + * @param filename filename to use; if none provided, a random filename is generated of the form + * "wpilog_{}.wpilog" + */ + public DataLogBackgroundWriter(String dir, String filename) { + this(dir, filename, 0.25); + } + + /** + * Construct a new Data Log. The log will be initially created with a temporary filename. + * + * @param dir directory to store the log + */ + public DataLogBackgroundWriter(String dir) { + this(dir, "", 0.25); + } + + /** Construct a new Data Log. The log will be initially created with a temporary filename. */ + public DataLogBackgroundWriter() { + this(""); + } + + /** + * Change log filename. + * + * @param filename filename + */ + public void setFilename(String filename) { + DataLogJNI.bgSetFilename(m_impl, filename); + } + + /** + * Resumes appending of data records to the log. If called after stop(), opens a new file (with + * random name if SetFilename was not called after stop()) and appends Start records and schema + * data values for all previously started entries and schemas. + */ + @Override + public void resume() { + DataLogJNI.resume(m_impl); + } +} diff --git a/wpiutil/src/main/java/edu/wpi/first/util/datalog/DataLogJNI.java b/wpiutil/src/main/java/edu/wpi/first/util/datalog/DataLogJNI.java index c7643390f1..ea040edb71 100644 --- a/wpiutil/src/main/java/edu/wpi/first/util/datalog/DataLogJNI.java +++ b/wpiutil/src/main/java/edu/wpi/first/util/datalog/DataLogJNI.java @@ -5,6 +5,7 @@ package edu.wpi.first.util.datalog; import edu.wpi.first.util.WPIUtilJNI; +import java.io.IOException; import java.nio.ByteBuffer; /** @@ -14,7 +15,8 @@ import java.nio.ByteBuffer; */ public class DataLogJNI extends WPIUtilJNI { /** - * Create a new Data Log. The log will be initially created with a temporary filename. + * Create a new Data Log background writer. The log will be initially created with a temporary + * filename. * * @param dir directory to store the log * @param filename filename to use; if none provided, a random filename is generated of the form @@ -22,22 +24,32 @@ public class DataLogJNI extends WPIUtilJNI { * @param period time between automatic flushes to disk, in seconds; this is a time/storage * tradeoff * @param extraHeader extra header data - * @return data log implementation handle + * @return data log background writer implementation handle */ - static native long create(String dir, String filename, double period, String extraHeader); + static native long bgCreate(String dir, String filename, double period, String extraHeader); /** * Change log filename. * - * @param impl data log implementation handle + * @param impl data log background writer implementation handle * @param filename filename */ - static native void setFilename(long impl, String filename); + static native void bgSetFilename(long impl, String filename); + + /** + * Create a new Data Log foreground writer. + * + * @param filename filename to use + * @param extraHeader extra header data + * @return data log writer implementation handle + * @throws IOException if file cannot be opened + */ + static native long fgCreate(String filename, String extraHeader) throws IOException; /** * Explicitly flushes the log data to disk. * - * @param impl data log implementation handle + * @param impl data log background writer implementation handle */ static native void flush(long impl); @@ -45,7 +57,7 @@ public class DataLogJNI extends WPIUtilJNI { * Pauses appending of data records to the log. While paused, no data records are saved (e.g. * AppendX is a no-op). Has no effect on entry starts / finishes / metadata changes. * - * @param impl data log implementation handle + * @param impl data log background writer implementation handle */ static native void pause(long impl); @@ -54,14 +66,14 @@ public class DataLogJNI extends WPIUtilJNI { * random name if SetFilename was not called after Stop()) and appends Start records and schema * data values for all previously started entries and schemas. * - * @param impl data log implementation handle + * @param impl data log background writer implementation handle */ static native void resume(long impl); /** * Stops appending all records to the log, and closes the log file. * - * @param impl data log implementation handle + * @param impl data log background writer implementation handle */ static native void stop(long impl); diff --git a/wpiutil/src/main/java/edu/wpi/first/util/datalog/DataLogWriter.java b/wpiutil/src/main/java/edu/wpi/first/util/datalog/DataLogWriter.java new file mode 100644 index 0000000000..2b11b9a4d2 --- /dev/null +++ b/wpiutil/src/main/java/edu/wpi/first/util/datalog/DataLogWriter.java @@ -0,0 +1,31 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +package edu.wpi.first.util.datalog; + +import java.io.IOException; + +/** A data log writer that flushes the data log to a file when flush() is called. */ +public class DataLogWriter extends DataLog { + /** + * Construct a new Data Log. + * + * @param filename filename to use + * @param extraHeader extra header data + * @throws IOException if file cannot be opened + */ + public DataLogWriter(String filename, String extraHeader) throws IOException { + super(DataLogJNI.fgCreate(filename, extraHeader)); + } + + /** + * Construct a new Data Log. + * + * @param filename filename to use + * @throws IOException if file cannot be opened + */ + public DataLogWriter(String filename) throws IOException { + this(filename, ""); + } +} diff --git a/wpiutil/src/main/native/cpp/DataLog.cpp b/wpiutil/src/main/native/cpp/DataLog.cpp index ff98fd25e5..befcc7513a 100644 --- a/wpiutil/src/main/native/cpp/DataLog.cpp +++ b/wpiutil/src/main/native/cpp/DataLog.cpp @@ -4,60 +4,33 @@ #include "wpi/DataLog.h" -#include "wpi/Synchronization.h" - -#ifndef _WIN32 -#include -#endif - -#ifdef _WIN32 -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN -#endif - -#include // NOLINT(build/include_order) - -#endif - -#include #include #include #include #include -#include #include #include "wpi/Endian.h" #include "wpi/Logger.h" -#include "wpi/MathExtras.h" #include "wpi/SmallString.h" -#include "wpi/fs.h" #include "wpi/print.h" #include "wpi/timestamp.h" using namespace wpi::log; -static constexpr size_t kBlockSize = 16 * 1024; -static constexpr size_t kMaxBufferCount = 1024 * 1024 / kBlockSize; -static constexpr size_t kMaxFreeCount = 256 * 1024 / kBlockSize; static constexpr size_t kRecordMaxHeaderSize = 17; -static constexpr uintmax_t kMinFreeSpace = 5 * 1024 * 1024; -static std::string FormatBytesSize(uintmax_t value) { - static constexpr uintmax_t kKiB = 1024; - static constexpr uintmax_t kMiB = kKiB * 1024; - static constexpr uintmax_t kGiB = kMiB * 1024; - if (value >= kGiB) { - return fmt::format("{:.1f} GiB", static_cast(value) / kGiB); - } else if (value >= kMiB) { - return fmt::format("{:.1f} MiB", static_cast(value) / kMiB); - } else if (value >= kKiB) { - return fmt::format("{:.1f} KiB", static_cast(value) / kKiB); - } else { - return fmt::format("{} B", value); +static void DefaultLog(unsigned int level, const char* file, unsigned int line, + const char* msg) { + if (level > wpi::WPI_LOG_INFO) { + wpi::print(stderr, "DataLog: {}\n", msg); + } else if (level == wpi::WPI_LOG_INFO) { + wpi::print("DataLog: {}\n", msg); } } +wpi::Logger DataLog::s_defaultMessageLog{DefaultLog}; + template static unsigned int WriteVarInt(uint8_t* buf, T val) { unsigned int len = 0; @@ -87,144 +60,72 @@ static unsigned int WriteRecordHeader(uint8_t* buf, uint32_t entry, return buf - origbuf; } -class DataLog::Buffer { - public: - explicit Buffer(size_t alloc = kBlockSize) - : m_buf{new uint8_t[alloc]}, m_maxLen{alloc} {} - ~Buffer() { delete[] m_buf; } - - Buffer(const Buffer&) = delete; - Buffer& operator=(const Buffer&) = delete; - - Buffer(Buffer&& oth) - : m_buf{oth.m_buf}, m_len{oth.m_len}, m_maxLen{oth.m_maxLen} { - oth.m_buf = nullptr; - oth.m_len = 0; - oth.m_maxLen = 0; +void DataLog::StartFile() { + std::scoped_lock lock{m_mutex}; + if (m_active) { + return; } - Buffer& operator=(Buffer&& oth) { - if (m_buf) { - delete[] m_buf; + // Grab previously pending writes + std::vector bufs; + bufs.swap(m_outgoing); + m_outgoing.reserve(bufs.size() + 1); + + // File header (version 1.0) + uint8_t* buf = Reserve(m_extraHeader.size() + 12); + static const uint8_t header[] = {'W', 'P', 'I', 'L', 'O', 'G', 0, 1}; + std::memcpy(buf, header, 8); + support::endian::write32le(buf + 8, m_extraHeader.size()); + std::memcpy(buf + 12, m_extraHeader.data(), m_extraHeader.size()); + + // Existing start and schema data records + for (auto&& entryInfo : m_entries) { + AppendStartRecord(entryInfo.second.id, entryInfo.first(), + entryInfo.second.type, + m_entryIds[entryInfo.second.id].metadata, 0); + if (!entryInfo.second.schemaData.empty()) { + StartRecord(entryInfo.second.id, 0, entryInfo.second.schemaData.size(), + 0); + AppendImpl(entryInfo.second.schemaData); } - m_buf = oth.m_buf; - m_len = oth.m_len; - m_maxLen = oth.m_maxLen; - oth.m_buf = nullptr; - oth.m_len = 0; - oth.m_maxLen = 0; - return *this; } - uint8_t* Reserve(size_t size) { - assert(size <= GetRemaining()); - uint8_t* rv = m_buf + m_len; - m_len += size; - return rv; + // Append previously pending writes + for (auto&& buf : bufs) { + m_outgoing.emplace_back(std::move(buf)); } - void Unreserve(size_t size) { m_len -= size; } - - void Clear() { m_len = 0; } - - size_t GetRemaining() const { return m_maxLen - m_len; } - - std::span GetData() { return {m_buf, m_len}; } - std::span GetData() const { return {m_buf, m_len}; } - - private: - uint8_t* m_buf; - size_t m_len = 0; - size_t m_maxLen; -}; - -static void DefaultLog(unsigned int level, const char* file, unsigned int line, - const char* msg) { - if (level > wpi::WPI_LOG_INFO) { - wpi::print(stderr, "DataLog: {}\n", msg); - } else if (level == wpi::WPI_LOG_INFO) { - wpi::print("DataLog: {}\n", msg); - } + m_active = true; } -static wpi::Logger defaultMessageLog{DefaultLog}; - -DataLog::DataLog(std::string_view dir, std::string_view filename, double period, - std::string_view extraHeader) - : DataLog{defaultMessageLog, dir, filename, period, extraHeader} {} - -DataLog::DataLog(wpi::Logger& msglog, std::string_view dir, - std::string_view filename, double period, - std::string_view extraHeader) - : m_msglog{msglog}, - m_period{period}, - m_extraHeader{extraHeader}, - m_newFilename{filename}, - m_thread{[this, dir = std::string{dir}] { WriterThreadMain(dir); }} {} - -DataLog::DataLog(std::function data)> write, - double period, std::string_view extraHeader) - : DataLog{defaultMessageLog, std::move(write), period, extraHeader} {} - -DataLog::DataLog(wpi::Logger& msglog, - std::function data)> write, - double period, std::string_view extraHeader) - : m_msglog{msglog}, - m_period{period}, - m_extraHeader{extraHeader}, - m_thread{[this, write = std::move(write)] { - WriterThreadMain(std::move(write)); - }} {} - -DataLog::~DataLog() { - { - std::scoped_lock lock{m_mutex}; - m_shutdown = true; - m_doFlush = true; - } - m_cond.notify_all(); - m_thread.join(); +void DataLog::FlushBufs(std::vector* writeBufs) { + std::scoped_lock lock{m_mutex}; + writeBufs->swap(m_outgoing); + DoReleaseBufs(&m_outgoing); } -void DataLog::SetFilename(std::string_view filename) { - { - std::scoped_lock lock{m_mutex}; - m_newFilename = filename; - } - m_cond.notify_all(); -} - -void DataLog::Flush() { - { - std::scoped_lock lock{m_mutex}; - m_doFlush = true; - } - m_cond.notify_all(); +void DataLog::ReleaseBufs(std::vector* bufs) { + std::scoped_lock lock{m_mutex}; + DoReleaseBufs(bufs); } void DataLog::Pause() { std::scoped_lock lock{m_mutex}; - m_state = kPaused; + m_paused = true; } void DataLog::Resume() { std::scoped_lock lock{m_mutex}; - if (m_state == kPaused) { - m_state = kActive; - } else if (m_state == kStopped) { - m_state = kStart; - } + m_paused = false; } void DataLog::Stop() { - { - std::scoped_lock lock{m_mutex}; - m_state = kStopped; - m_newFilename.clear(); - } - m_cond.notify_all(); + std::scoped_lock lock{m_mutex}; + m_active = false; } +void DataLog::BufferHalfFull() {} + bool DataLog::HasSchema(std::string_view name) const { std::scoped_lock lock{m_mutex}; wpi::SmallString<128> fullName{"/.schema/"}; @@ -249,380 +150,13 @@ void DataLog::AddSchema(std::string_view name, std::string_view type, if (entry <= 0) { [[unlikely]] return; // should never happen, but check anyway } - if (m_state != kActive && m_state != kPaused) { + if (!m_active) { [[unlikely]] return; } StartRecord(entry, timestamp, schema.size(), 0); AppendImpl(schema); } -static void WriteToFile(fs::file_t f, std::span data, - std::string_view filename, wpi::Logger& msglog) { - do { -#ifdef _WIN32 - DWORD ret; - if (!WriteFile(f, data.data(), data.size(), &ret, nullptr)) { - WPI_ERROR(msglog, "Error writing to log file '{}': {}", filename, - GetLastError()); - break; - } -#else - ssize_t ret = ::write(f, data.data(), data.size()); - if (ret < 0) { - // If it's a recoverable error, swallow it and retry the write - if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { - continue; - } - - // Otherwise it's a non-recoverable error; quit trying - WPI_ERROR(msglog, "Error writing to log file '{}': {}", filename, - std::strerror(errno)); - break; - } -#endif - - // The write may have written some or all of the data - data = data.subspan(ret); - } while (data.size() > 0); -} - -static std::string MakeRandomFilename() { - // build random filename - static std::random_device dev; - static std::mt19937 rng(dev()); - std::uniform_int_distribution dist(0, 15); - const char* v = "0123456789abcdef"; - std::string filename = "wpilog_"; - for (int i = 0; i < 16; i++) { - filename += v[dist(rng)]; - } - filename += ".wpilog"; - return filename; -} - -struct DataLog::WriterThreadState { - explicit WriterThreadState(std::string_view dir) : dirPath{dir} {} - WriterThreadState(const WriterThreadState&) = delete; - WriterThreadState& operator=(const WriterThreadState&) = delete; - ~WriterThreadState() { Close(); } - - void Close() { - if (f != fs::kInvalidFile) { - fs::CloseFile(f); - f = fs::kInvalidFile; - } - } - - void SetFilename(std::string_view fn) { - baseFilename = fn; - filename = fn; - path = dirPath / filename; - segmentCount = 1; - } - - void IncrementFilename() { - fs::path basePath{baseFilename}; - filename = fmt::format("{}.{}{}", basePath.stem().string(), ++segmentCount, - basePath.extension().string()); - path = dirPath / filename; - } - - fs::path dirPath; - std::string baseFilename; - std::string filename; - fs::path path; - fs::file_t f = fs::kInvalidFile; - uintmax_t freeSpace = UINTMAX_MAX; - int segmentCount = 1; -}; - -void DataLog::StartLogFile(WriterThreadState& state) { - std::error_code ec; - - if (state.filename.empty()) { - state.SetFilename(MakeRandomFilename()); - } - - // get free space - auto freeSpaceInfo = fs::space(state.dirPath, ec); - if (!ec) { - state.freeSpace = freeSpaceInfo.available; - } else { - state.freeSpace = UINTMAX_MAX; - } - if (state.freeSpace < kMinFreeSpace) { - WPI_ERROR(m_msglog, - "Insufficient free space ({} available), no log being saved", - FormatBytesSize(state.freeSpace)); - m_state = kStopped; - } else { - // try preferred filename, or randomize it a few times, before giving up - for (int i = 0; i < 5; ++i) { - // open file for append -#ifdef _WIN32 - // WIN32 doesn't allow combination of CreateNew and Append - state.f = - fs::OpenFileForWrite(state.path, ec, fs::CD_CreateNew, fs::OF_None); -#else - state.f = - fs::OpenFileForWrite(state.path, ec, fs::CD_CreateNew, fs::OF_Append); -#endif - if (ec) { - WPI_ERROR(m_msglog, "Could not open log file '{}': {}", - state.path.string(), ec.message()); - // try again with random filename - state.SetFilename(MakeRandomFilename()); - } else { - break; - } - } - - if (state.f == fs::kInvalidFile) { - WPI_ERROR(m_msglog, "Could not open log file, no log being saved"); - } else { - WPI_INFO(m_msglog, "Logging to '{}' ({} free space)", state.path.string(), - FormatBytesSize(state.freeSpace)); - } - } - - // write header (version 1.0) - if (state.f != fs::kInvalidFile) { - const uint8_t header[] = {'W', 'P', 'I', 'L', 'O', 'G', 0, 1}; - WriteToFile(state.f, header, state.filename, m_msglog); - uint8_t extraLen[4]; - support::endian::write32le(extraLen, m_extraHeader.size()); - WriteToFile(state.f, extraLen, state.filename, m_msglog); - if (m_extraHeader.size() > 0) { - WriteToFile(state.f, - {reinterpret_cast(m_extraHeader.data()), - m_extraHeader.size()}, - state.filename, m_msglog); - } - } -} - -void DataLog::WriterThreadMain(std::string_view dir) { - std::chrono::duration periodTime{m_period}; - - WriterThreadState state{dir}; - { - std::scoped_lock lock{m_mutex}; - state.SetFilename(m_newFilename); - m_newFilename.clear(); - } - StartLogFile(state); - - std::error_code ec; - std::vector toWrite; - int freeSpaceCount = 0; - int checkExistCount = 0; - bool blocked = false; - uintmax_t written = 0; - - std::unique_lock lock{m_mutex}; - do { - bool doFlush = false; - auto timeoutTime = std::chrono::steady_clock::now() + periodTime; - if (m_cond.wait_until(lock, timeoutTime) == std::cv_status::timeout) { - doFlush = true; - } - - if (m_state == kStopped) { - state.Close(); - continue; - } - - bool doStart = false; - - // if file was deleted, recreate it with the same name - if (++checkExistCount >= 10) { - checkExistCount = 0; - lock.unlock(); - bool exists = fs::exists(state.path, ec); - lock.lock(); - if (!ec && !exists) { - state.Close(); - state.IncrementFilename(); - WPI_INFO(m_msglog, "Log file deleted, recreating as fresh log '{}'", - state.filename); - doStart = true; - } - } - - // start new file if file exceeds 1.8 GB - if (written > 1800000000ull) { - state.Close(); - state.IncrementFilename(); - WPI_INFO(m_msglog, "Log file reached 1.8 GB, starting new file '{}'", - state.filename); - doStart = true; - } - - if (m_state == kStart || doStart) { - lock.unlock(); - StartLogFile(state); - lock.lock(); - if (m_state == kStopped) { - continue; - } - if (state.f != fs::kInvalidFile) { - // Emit start and schema data records - for (auto&& entryInfo : m_entries) { - AppendStartRecord(entryInfo.second.id, entryInfo.first(), - entryInfo.second.type, - m_entryIds[entryInfo.second.id].metadata, 0); - if (!entryInfo.second.schemaData.empty()) { - StartRecord(entryInfo.second.id, 0, - entryInfo.second.schemaData.size(), 0); - AppendImpl(entryInfo.second.schemaData); - } - } - } - m_state = kActive; - written = 0; - } - - if (!m_newFilename.empty() && state.f != fs::kInvalidFile) { - auto newFilename = std::move(m_newFilename); - m_newFilename.clear(); - // rename - if (state.filename != newFilename) { - lock.unlock(); - fs::rename(state.path, state.dirPath / newFilename, ec); - lock.lock(); - } - if (ec) { - WPI_ERROR(m_msglog, "Could not rename log file from '{}' to '{}': {}", - state.filename, newFilename, ec.message()); - } else { - WPI_INFO(m_msglog, "Renamed log file from '{}' to '{}'", state.filename, - newFilename); - } - state.SetFilename(newFilename); - } - - if (doFlush || m_doFlush) { - // flush to file - m_doFlush = false; - if (m_outgoing.empty()) { - continue; - } - // swap outgoing with empty vector - toWrite.swap(m_outgoing); - - if (state.f != fs::kInvalidFile && !blocked) { - lock.unlock(); - - // update free space every 10 flushes (in case other things are writing) - if (++freeSpaceCount >= 10) { - freeSpaceCount = 0; - auto freeSpaceInfo = fs::space(state.dirPath, ec); - if (!ec) { - state.freeSpace = freeSpaceInfo.available; - } else { - state.freeSpace = UINTMAX_MAX; - } - } - - // write buffers to file - for (auto&& buf : toWrite) { - // stop writing when we go below the minimum free space - state.freeSpace -= buf.GetData().size(); - written += buf.GetData().size(); - if (state.freeSpace < kMinFreeSpace) { - [[unlikely]] WPI_ERROR( - m_msglog, - "Stopped logging due to low free space ({} available)", - FormatBytesSize(state.freeSpace)); - blocked = true; - break; - } - WriteToFile(state.f, buf.GetData(), state.filename, m_msglog); - } - - // sync to storage -#if defined(__linux__) - ::fdatasync(state.f); -#elif defined(__APPLE__) - ::fsync(state.f); -#endif - lock.lock(); - if (blocked) { - [[unlikely]] m_state = kPaused; - } - } - - // release buffers back to free list - for (auto&& buf : toWrite) { - buf.Clear(); - if (m_free.size() < kMaxFreeCount) { - [[likely]] m_free.emplace_back(std::move(buf)); - } - } - toWrite.resize(0); - } - } while (!m_shutdown); -} - -void DataLog::WriterThreadMain( - std::function data)> write) { - std::chrono::duration periodTime{m_period}; - - // write header (version 1.0) - { - const uint8_t header[] = {'W', 'P', 'I', 'L', 'O', 'G', 0, 1}; - write(header); - uint8_t extraLen[4]; - support::endian::write32le(extraLen, m_extraHeader.size()); - write(extraLen); - if (m_extraHeader.size() > 0) { - write({reinterpret_cast(m_extraHeader.data()), - m_extraHeader.size()}); - } - } - - std::vector toWrite; - - std::unique_lock lock{m_mutex}; - do { - bool doFlush = false; - auto timeoutTime = std::chrono::steady_clock::now() + periodTime; - if (m_cond.wait_until(lock, timeoutTime) == std::cv_status::timeout) { - doFlush = true; - } - - if (doFlush || m_doFlush) { - // flush to file - m_doFlush = false; - if (m_outgoing.empty()) { - continue; - } - // swap outgoing with empty vector - toWrite.swap(m_outgoing); - - lock.unlock(); - // write buffers - for (auto&& buf : toWrite) { - if (!buf.GetData().empty()) { - write(buf.GetData()); - } - } - lock.lock(); - - // release buffers back to free list - for (auto&& buf : toWrite) { - buf.Clear(); - if (m_free.size() < kMaxFreeCount) { - [[likely]] m_free.emplace_back(std::move(buf)); - } - } - toWrite.resize(0); - } - } while (!m_shutdown); - - write({}); // indicate EOF -} - // Control records use the following format: // 1-byte type // 4-byte entry @@ -654,7 +188,7 @@ int DataLog::StartImpl(std::string_view name, std::string_view type, entryInfo.type = type; entryInfo2.metadata = metadata; - if (m_state != kActive && m_state != kPaused) { + if (!m_active) { [[unlikely]] return entryInfo.id; } @@ -674,6 +208,16 @@ void DataLog::AppendStartRecord(int id, std::string_view name, AppendStringImpl(metadata); } +void DataLog::DoReleaseBufs(std::vector* bufs) { + for (auto&& buf : *bufs) { + buf.Clear(); + if (m_free.size() < kMaxFreeCount) { + [[likely]] m_free.emplace_back(std::move(buf)); + } + } + bufs->resize(0); +} + void DataLog::Finish(int entry, int64_t timestamp) { if (entry <= 0) { return; @@ -688,7 +232,7 @@ void DataLog::Finish(int entry, int64_t timestamp) { return; } m_entryIds.erase(entry); - if (m_state != kActive && m_state != kPaused) { + if (!m_active) { [[unlikely]] return; } uint8_t* buf = StartRecord(0, timestamp, 5, 5); @@ -703,7 +247,7 @@ void DataLog::SetMetadata(int entry, std::string_view metadata, } std::scoped_lock lock{m_mutex}; m_entryIds[entry].metadata = metadata; - if (m_state != kActive && m_state != kPaused) { + if (!m_active) { [[unlikely]] return; } uint8_t* buf = StartRecord(0, timestamp, 5 + 4 + metadata.size(), 5); @@ -715,13 +259,15 @@ void DataLog::SetMetadata(int entry, std::string_view metadata, uint8_t* DataLog::Reserve(size_t size) { assert(size <= kBlockSize); if (m_outgoing.empty() || size > m_outgoing.back().GetRemaining()) { + if (m_outgoing.size() == kMaxBufferCount / 2) { + [[unlikely]] BufferHalfFull(); + } if (m_free.empty()) { if (m_outgoing.size() >= kMaxBufferCount) { - [[unlikely]] WPI_ERROR( - m_msglog, - "outgoing buffers exceeded threshold, pausing logging--" - "consider flushing to disk more frequently (smaller period)"); - m_state = kPaused; + [[unlikely]] + if (BufferFull()) { + m_paused = true; + } } m_outgoing.emplace_back(); } else { @@ -765,7 +311,7 @@ void DataLog::AppendRaw(int entry, std::span data, return; } std::scoped_lock lock{m_mutex}; - if (m_state != kActive) { + if (m_paused) { [[unlikely]] return; } StartRecord(entry, timestamp, data.size(), 0); @@ -779,7 +325,7 @@ void DataLog::AppendRaw2(int entry, return; } std::scoped_lock lock{m_mutex}; - if (m_state != kActive) { + if (m_paused) { [[unlikely]] return; } size_t size = 0; @@ -797,7 +343,7 @@ void DataLog::AppendBoolean(int entry, bool value, int64_t timestamp) { return; } std::scoped_lock lock{m_mutex}; - if (m_state != kActive) { + if (m_paused) { [[unlikely]] return; } uint8_t* buf = StartRecord(entry, timestamp, 1, 1); @@ -809,7 +355,7 @@ void DataLog::AppendInteger(int entry, int64_t value, int64_t timestamp) { return; } std::scoped_lock lock{m_mutex}; - if (m_state != kActive) { + if (m_paused) { [[unlikely]] return; } uint8_t* buf = StartRecord(entry, timestamp, 8, 8); @@ -821,7 +367,7 @@ void DataLog::AppendFloat(int entry, float value, int64_t timestamp) { return; } std::scoped_lock lock{m_mutex}; - if (m_state != kActive) { + if (m_paused) { [[unlikely]] return; } uint8_t* buf = StartRecord(entry, timestamp, 4, 4); @@ -837,7 +383,7 @@ void DataLog::AppendDouble(int entry, double value, int64_t timestamp) { return; } std::scoped_lock lock{m_mutex}; - if (m_state != kActive) { + if (m_paused) { [[unlikely]] return; } uint8_t* buf = StartRecord(entry, timestamp, 8, 8); @@ -861,7 +407,7 @@ void DataLog::AppendBooleanArray(int entry, std::span arr, return; } std::scoped_lock lock{m_mutex}; - if (m_state != kActive) { + if (m_paused) { [[unlikely]] return; } StartRecord(entry, timestamp, arr.size(), 0); @@ -885,7 +431,7 @@ void DataLog::AppendBooleanArray(int entry, std::span arr, return; } std::scoped_lock lock{m_mutex}; - if (m_state != kActive) { + if (m_paused) { [[unlikely]] return; } StartRecord(entry, timestamp, arr.size(), 0); @@ -919,7 +465,7 @@ void DataLog::AppendIntegerArray(int entry, std::span arr, return; } std::scoped_lock lock{m_mutex}; - if (m_state != kActive) { + if (m_paused) { [[unlikely]] return; } StartRecord(entry, timestamp, arr.size() * 8, 0); @@ -951,7 +497,7 @@ void DataLog::AppendFloatArray(int entry, std::span arr, return; } std::scoped_lock lock{m_mutex}; - if (m_state != kActive) { + if (m_paused) { [[unlikely]] return; } StartRecord(entry, timestamp, arr.size() * 4, 0); @@ -983,7 +529,7 @@ void DataLog::AppendDoubleArray(int entry, std::span arr, return; } std::scoped_lock lock{m_mutex}; - if (m_state != kActive) { + if (m_paused) { [[unlikely]] return; } StartRecord(entry, timestamp, arr.size() * 8, 0); @@ -1016,7 +562,7 @@ void DataLog::AppendStringArray(int entry, std::span arr, size += 4 + str.size(); } std::scoped_lock lock{m_mutex}; - if (m_state != kActive) { + if (m_paused) { [[unlikely]] return; } uint8_t* buf = StartRecord(entry, timestamp, size, 4); @@ -1039,7 +585,7 @@ void DataLog::AppendStringArray(int entry, size += 4 + str.size(); } std::scoped_lock lock{m_mutex}; - if (m_state != kActive) { + if (m_paused) { [[unlikely]] return; } uint8_t* buf = StartRecord(entry, timestamp, size, 4); @@ -1062,7 +608,7 @@ void DataLog::AppendStringArray(int entry, size += 4 + str.len; } std::scoped_lock lock{m_mutex}; - if (m_state != kActive) { + if (m_paused) { [[unlikely]] return; } uint8_t* buf = StartRecord(entry, timestamp, size, 4); @@ -1074,29 +620,10 @@ void DataLog::AppendStringArray(int entry, extern "C" { -struct WPI_DataLog* WPI_DataLog_Create(const char* dir, const char* filename, - double period, const char* extraHeader) { - return reinterpret_cast( - new DataLog{dir, filename, period, extraHeader}); -} - -struct WPI_DataLog* WPI_DataLog_Create_Func( - void (*write)(void* ptr, const uint8_t* data, size_t len), void* ptr, - double period, const char* extraHeader) { - return reinterpret_cast( - new DataLog{[=](auto data) { write(ptr, data.data(), data.size()); }, - period, extraHeader}); -} - void WPI_DataLog_Release(struct WPI_DataLog* datalog) { delete reinterpret_cast(datalog); } -void WPI_DataLog_SetFilename(struct WPI_DataLog* datalog, - const char* filename) { - reinterpret_cast(datalog)->SetFilename(filename); -} - void WPI_DataLog_Flush(struct WPI_DataLog* datalog) { reinterpret_cast(datalog)->Flush(); } diff --git a/wpiutil/src/main/native/cpp/DataLogBackgroundWriter.cpp b/wpiutil/src/main/native/cpp/DataLogBackgroundWriter.cpp new file mode 100644 index 0000000000..4ef7ef7cf9 --- /dev/null +++ b/wpiutil/src/main/native/cpp/DataLogBackgroundWriter.cpp @@ -0,0 +1,483 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#include "wpi/DataLogBackgroundWriter.h" + +#ifndef _WIN32 +#include +#endif + +#ifdef _WIN32 +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif + +#include // NOLINT(build/include_order) + +#endif + +#include + +#include + +#include "wpi/Logger.h" +#include "wpi/fs.h" + +using namespace wpi::log; + +static constexpr uintmax_t kMinFreeSpace = 5 * 1024 * 1024; + +static std::string FormatBytesSize(uintmax_t value) { + static constexpr uintmax_t kKiB = 1024; + static constexpr uintmax_t kMiB = kKiB * 1024; + static constexpr uintmax_t kGiB = kMiB * 1024; + if (value >= kGiB) { + return fmt::format("{:.1f} GiB", static_cast(value) / kGiB); + } else if (value >= kMiB) { + return fmt::format("{:.1f} MiB", static_cast(value) / kMiB); + } else if (value >= kKiB) { + return fmt::format("{:.1f} KiB", static_cast(value) / kKiB); + } else { + return fmt::format("{} B", value); + } +} + +DataLogBackgroundWriter::DataLogBackgroundWriter(std::string_view dir, + std::string_view filename, + double period, + std::string_view extraHeader) + : DataLogBackgroundWriter{s_defaultMessageLog, dir, filename, period, + extraHeader} {} + +DataLogBackgroundWriter::DataLogBackgroundWriter(wpi::Logger& msglog, + std::string_view dir, + std::string_view filename, + double period, + std::string_view extraHeader) + : DataLog{msglog, extraHeader}, + m_period{period}, + m_newFilename{filename}, + m_thread{[this, dir = std::string{dir}] { WriterThreadMain(dir); }} {} + +DataLogBackgroundWriter::DataLogBackgroundWriter( + std::function data)> write, double period, + std::string_view extraHeader) + : DataLogBackgroundWriter{s_defaultMessageLog, std::move(write), period, + extraHeader} {} + +DataLogBackgroundWriter::DataLogBackgroundWriter( + wpi::Logger& msglog, + std::function data)> write, double period, + std::string_view extraHeader) + : DataLog{msglog, extraHeader}, + m_period{period}, + m_thread{[this, write = std::move(write)] { + WriterThreadMain(std::move(write)); + }} {} + +DataLogBackgroundWriter::~DataLogBackgroundWriter() { + { + std::scoped_lock lock{m_mutex}; + m_shutdown = true; + m_doFlush = true; + } + m_cond.notify_all(); + m_thread.join(); +} + +void DataLogBackgroundWriter::SetFilename(std::string_view filename) { + { + std::scoped_lock lock{m_mutex}; + m_newFilename = filename; + } + m_cond.notify_all(); +} + +void DataLogBackgroundWriter::Flush() { + { + std::scoped_lock lock{m_mutex}; + m_doFlush = true; + } + m_cond.notify_all(); +} + +void DataLogBackgroundWriter::Pause() { + DataLog::Pause(); + std::scoped_lock lock{m_mutex}; + m_state = kPaused; +} + +void DataLogBackgroundWriter::Resume() { + DataLog::Resume(); + std::scoped_lock lock{m_mutex}; + if (m_state == kPaused) { + m_state = kActive; + } else if (m_state == kStopped) { + m_state = kStart; + } +} + +void DataLogBackgroundWriter::Stop() { + DataLog::Stop(); + { + std::scoped_lock lock{m_mutex}; + m_state = kStopped; + m_newFilename.clear(); + } + m_cond.notify_all(); +} + +static void WriteToFile(fs::file_t f, std::span data, + std::string_view filename, wpi::Logger& msglog) { + do { +#ifdef _WIN32 + DWORD ret; + if (!WriteFile(f, data.data(), data.size(), &ret, nullptr)) { + WPI_ERROR(msglog, "Error writing to log file '{}': {}", filename, + GetLastError()); + break; + } +#else + ssize_t ret = ::write(f, data.data(), data.size()); + if (ret < 0) { + // If it's a recoverable error, swallow it and retry the write + if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { + continue; + } + + // Otherwise it's a non-recoverable error; quit trying + WPI_ERROR(msglog, "Error writing to log file '{}': {}", filename, + std::strerror(errno)); + break; + } +#endif + + // The write may have written some or all of the data + data = data.subspan(ret); + } while (data.size() > 0); +} + +static std::string MakeRandomFilename() { + // build random filename + static std::random_device dev; + static std::mt19937 rng(dev()); + std::uniform_int_distribution dist(0, 15); + const char* v = "0123456789abcdef"; + std::string filename = "wpilog_"; + for (int i = 0; i < 16; i++) { + filename += v[dist(rng)]; + } + filename += ".wpilog"; + return filename; +} + +struct DataLogBackgroundWriter::WriterThreadState { + explicit WriterThreadState(std::string_view dir) : dirPath{dir} {} + WriterThreadState(const WriterThreadState&) = delete; + WriterThreadState& operator=(const WriterThreadState&) = delete; + ~WriterThreadState() { Close(); } + + void Close() { + if (f != fs::kInvalidFile) { + fs::CloseFile(f); + f = fs::kInvalidFile; + } + } + + void SetFilename(std::string_view fn) { + baseFilename = fn; + filename = fn; + path = dirPath / filename; + segmentCount = 1; + } + + void IncrementFilename() { + fs::path basePath{baseFilename}; + filename = fmt::format("{}.{}{}", basePath.stem().string(), ++segmentCount, + basePath.extension().string()); + path = dirPath / filename; + } + + fs::path dirPath; + std::string baseFilename; + std::string filename; + fs::path path; + fs::file_t f = fs::kInvalidFile; + uintmax_t freeSpace = UINTMAX_MAX; + int segmentCount = 1; +}; + +void DataLogBackgroundWriter::BufferHalfFull() { + Flush(); +} + +bool DataLogBackgroundWriter::BufferFull() { + WPI_ERROR(m_msglog, + "outgoing buffers exceeded threshold, pausing logging--" + "consider flushing to disk more frequently (smaller period)"); + return true; +} + +void DataLogBackgroundWriter::StartLogFile(WriterThreadState& state) { + std::error_code ec; + + if (state.filename.empty()) { + state.SetFilename(MakeRandomFilename()); + } + + // get free space + auto freeSpaceInfo = fs::space(state.dirPath, ec); + if (!ec) { + state.freeSpace = freeSpaceInfo.available; + } else { + state.freeSpace = UINTMAX_MAX; + } + if (state.freeSpace < kMinFreeSpace) { + WPI_ERROR(m_msglog, + "Insufficient free space ({} available), no log being saved", + FormatBytesSize(state.freeSpace)); + } else { + // try preferred filename, or randomize it a few times, before giving up + for (int i = 0; i < 5; ++i) { + // open file for append +#ifdef _WIN32 + // WIN32 doesn't allow combination of CreateNew and Append + state.f = + fs::OpenFileForWrite(state.path, ec, fs::CD_CreateNew, fs::OF_None); +#else + state.f = + fs::OpenFileForWrite(state.path, ec, fs::CD_CreateNew, fs::OF_Append); +#endif + if (ec) { + WPI_ERROR(m_msglog, "Could not open log file '{}': {}", + state.path.string(), ec.message()); + // try again with random filename + state.SetFilename(MakeRandomFilename()); + } else { + break; + } + } + + if (state.f == fs::kInvalidFile) { + WPI_ERROR(m_msglog, "Could not open log file, no log being saved"); + } else { + WPI_INFO(m_msglog, "Logging to '{}' ({} free space)", state.path.string(), + FormatBytesSize(state.freeSpace)); + } + } + + // start file + if (state.f != fs::kInvalidFile) { + StartFile(); + } +} + +void DataLogBackgroundWriter::WriterThreadMain(std::string_view dir) { + std::chrono::duration periodTime{m_period}; + + WriterThreadState state{dir}; + { + std::scoped_lock lock{m_mutex}; + state.SetFilename(m_newFilename); + m_newFilename.clear(); + } + StartLogFile(state); + + std::error_code ec; + std::vector toWrite; + int freeSpaceCount = 0; + int checkExistCount = 0; + bool blocked = false; + uintmax_t written = 0; + + std::unique_lock lock{m_mutex}; + do { + bool doFlush = false; + auto timeoutTime = std::chrono::steady_clock::now() + periodTime; + if (m_cond.wait_until(lock, timeoutTime) == std::cv_status::timeout) { + doFlush = true; + } + + if (m_state == kStopped) { + state.Close(); + continue; + } + + bool doStart = false; + + // if file was deleted, recreate it with the same name + if (++checkExistCount >= 10) { + checkExistCount = 0; + lock.unlock(); + bool exists = fs::exists(state.path, ec); + lock.lock(); + if (!ec && !exists) { + state.Close(); + state.IncrementFilename(); + WPI_INFO(m_msglog, "Log file deleted, recreating as fresh log '{}'", + state.filename); + doStart = true; + } + } + + // start new file if file exceeds 1.8 GB + if (written > 1800000000ull) { + state.Close(); + state.IncrementFilename(); + WPI_INFO(m_msglog, "Log file reached 1.8 GB, starting new file '{}'", + state.filename); + doStart = true; + } + + if (m_state == kStart || doStart) { + lock.unlock(); + DataLog::Stop(); + StartLogFile(state); + lock.lock(); + if (m_state == kStopped) { + continue; + } + m_state = kActive; + written = 0; + } + + if (!m_newFilename.empty() && state.f != fs::kInvalidFile) { + auto newFilename = std::move(m_newFilename); + m_newFilename.clear(); + // rename + if (state.filename != newFilename) { + lock.unlock(); + fs::rename(state.path, state.dirPath / newFilename, ec); + lock.lock(); + } + if (ec) { + WPI_ERROR(m_msglog, "Could not rename log file from '{}' to '{}': {}", + state.filename, newFilename, ec.message()); + } else { + WPI_INFO(m_msglog, "Renamed log file from '{}' to '{}'", state.filename, + newFilename); + } + state.SetFilename(newFilename); + } + + if (doFlush || m_doFlush) { + // flush to file + m_doFlush = false; + DataLog::FlushBufs(&toWrite); + if (toWrite.empty()) { + continue; + } + + if (state.f != fs::kInvalidFile && !blocked) { + lock.unlock(); + + // update free space every 10 flushes (in case other things are writing) + if (++freeSpaceCount >= 10) { + freeSpaceCount = 0; + auto freeSpaceInfo = fs::space(state.dirPath, ec); + if (!ec) { + state.freeSpace = freeSpaceInfo.available; + } else { + state.freeSpace = UINTMAX_MAX; + } + } + + // write buffers to file + for (auto&& buf : toWrite) { + // stop writing when we go below the minimum free space + state.freeSpace -= buf.GetData().size(); + written += buf.GetData().size(); + if (state.freeSpace < kMinFreeSpace) { + [[unlikely]] WPI_ERROR( + m_msglog, + "Stopped logging due to low free space ({} available)", + FormatBytesSize(state.freeSpace)); + blocked = true; + break; + } + WriteToFile(state.f, buf.GetData(), state.filename, m_msglog); + } + + // sync to storage +#if defined(__linux__) + ::fdatasync(state.f); +#elif defined(__APPLE__) + ::fsync(state.f); +#endif + lock.lock(); + if (blocked) { + [[unlikely]] m_state = kPaused; + } + } + + // release buffers back to free list + ReleaseBufs(&toWrite); + } + } while (!m_shutdown); +} + +void DataLogBackgroundWriter::WriterThreadMain( + std::function data)> write) { + std::chrono::duration periodTime{m_period}; + + StartFile(); + + std::vector toWrite; + + std::unique_lock lock{m_mutex}; + do { + bool doFlush = false; + auto timeoutTime = std::chrono::steady_clock::now() + periodTime; + if (m_cond.wait_until(lock, timeoutTime) == std::cv_status::timeout) { + doFlush = true; + } + + if (doFlush || m_doFlush) { + // flush to file + m_doFlush = false; + DataLog::FlushBufs(&toWrite); + if (toWrite.empty()) { + continue; + } + + lock.unlock(); + // write buffers + for (auto&& buf : toWrite) { + if (!buf.GetData().empty()) { + write(buf.GetData()); + } + } + lock.lock(); + + // release buffers back to free list + ReleaseBufs(&toWrite); + } + } while (!m_shutdown); + + write({}); // indicate EOF +} + +extern "C" { + +struct WPI_DataLog* WPI_DataLog_CreateBackgroundWriter( + const char* dir, const char* filename, double period, + const char* extraHeader) { + return reinterpret_cast( + new DataLogBackgroundWriter{dir, filename, period, extraHeader}); +} + +struct WPI_DataLog* WPI_DataLog_CreateBackgroundWriter_Func( + void (*write)(void* ptr, const uint8_t* data, size_t len), void* ptr, + double period, const char* extraHeader) { + return reinterpret_cast(new DataLogBackgroundWriter{ + [=](auto data) { write(ptr, data.data(), data.size()); }, period, + extraHeader}); +} + +void WPI_DataLog_SetBackgroundWriterFilename(struct WPI_DataLog* datalog, + const char* filename) { + reinterpret_cast(datalog)->SetFilename(filename); +} + +} // extern "C" diff --git a/wpiutil/src/main/native/cpp/DataLogWriter.cpp b/wpiutil/src/main/native/cpp/DataLogWriter.cpp new file mode 100644 index 0000000000..ca4645187f --- /dev/null +++ b/wpiutil/src/main/native/cpp/DataLogWriter.cpp @@ -0,0 +1,76 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#include "wpi/DataLogWriter.h" + +#include "wpi/raw_ostream.h" + +using namespace wpi::log; + +DataLogWriter::DataLogWriter(std::string_view filename, std::error_code& ec, + std::string_view extraHeader) + : DataLogWriter{s_defaultMessageLog, filename, ec, extraHeader} {} + +DataLogWriter::DataLogWriter(wpi::Logger& msglog, std::string_view filename, + std::error_code& ec, std::string_view extraHeader) + : DataLogWriter{msglog, std::make_unique(filename, ec), + extraHeader} { + if (ec) { + Stop(); + } +} + +DataLogWriter::DataLogWriter(std::unique_ptr os, + std::string_view extraHeader) + : DataLogWriter{s_defaultMessageLog, std::move(os), extraHeader} {} + +DataLogWriter::DataLogWriter(wpi::Logger& msglog, + std::unique_ptr os, + std::string_view extraHeader) + : DataLog{msglog, extraHeader}, m_os{std::move(os)} { + StartFile(); +} + +DataLogWriter::~DataLogWriter() { + if (m_os) { + Flush(); + } +} + +void DataLogWriter::Flush() { + if (!m_os) { + return; + } + std::vector writeBufs; + FlushBufs(&writeBufs); + for (auto&& buf : writeBufs) { + (*m_os) << buf.GetData(); + } + ReleaseBufs(&writeBufs); +} + +void DataLogWriter::Stop() { + DataLog::Stop(); + Flush(); + m_os.reset(); +} + +bool DataLogWriter::BufferFull() { + Flush(); + return false; +} + +extern "C" { + +struct WPI_DataLog* WPI_DataLog_CreateWriter(const char* filename, + int* errorCode, + const char* extraHeader) { + std::error_code ec; + auto rv = reinterpret_cast( + new DataLogWriter{filename, ec, extraHeader}); + *errorCode = ec.value(); + return rv; +} + +} // extern "C" diff --git a/wpiutil/src/main/native/cpp/jni/DataLogJNI.cpp b/wpiutil/src/main/native/cpp/jni/DataLogJNI.cpp index c78c891ea5..9ac6b3b58a 100644 --- a/wpiutil/src/main/native/cpp/jni/DataLogJNI.cpp +++ b/wpiutil/src/main/native/cpp/jni/DataLogJNI.cpp @@ -4,11 +4,15 @@ #include +#include + #include #include "WPIUtilJNI.h" #include "edu_wpi_first_util_datalog_DataLogJNI.h" #include "wpi/DataLog.h" +#include "wpi/DataLogBackgroundWriter.h" +#include "wpi/DataLogWriter.h" #include "wpi/jni_util.h" using namespace wpi::java; @@ -18,11 +22,11 @@ extern "C" { /* * Class: edu_wpi_first_util_datalog_DataLogJNI - * Method: create + * Method: bgCreate * Signature: (Ljava/lang/String;Ljava/lang/String;DLjava/lang/String;)J */ JNIEXPORT jlong JNICALL -Java_edu_wpi_first_util_datalog_DataLogJNI_create +Java_edu_wpi_first_util_datalog_DataLogJNI_bgCreate (JNIEnv* env, jclass, jstring dir, jstring filename, jdouble period, jstring extraHeader) { @@ -38,18 +42,18 @@ Java_edu_wpi_first_util_datalog_DataLogJNI_create wpi::ThrowNullPointerException(env, "extraHeader is null"); return 0; } - return reinterpret_cast(new DataLog{JStringRef{env, dir}, - JStringRef{env, filename}, period, - JStringRef{env, extraHeader}}); + return reinterpret_cast(new DataLogBackgroundWriter{ + JStringRef{env, dir}, JStringRef{env, filename}, period, + JStringRef{env, extraHeader}}); } /* * Class: edu_wpi_first_util_datalog_DataLogJNI - * Method: setFilename + * Method: bgSetFilename * Signature: (JLjava/lang/String;)V */ JNIEXPORT void JNICALL -Java_edu_wpi_first_util_datalog_DataLogJNI_setFilename +Java_edu_wpi_first_util_datalog_DataLogJNI_bgSetFilename (JNIEnv* env, jclass, jlong impl, jstring filename) { if (impl == 0) { @@ -60,7 +64,36 @@ Java_edu_wpi_first_util_datalog_DataLogJNI_setFilename wpi::ThrowNullPointerException(env, "filename is null"); return; } - reinterpret_cast(impl)->SetFilename(JStringRef{env, filename}); + reinterpret_cast(impl)->SetFilename( + JStringRef{env, filename}); +} + +/* + * Class: edu_wpi_first_util_datalog_DataLogJNI + * Method: fgCreate + * Signature: (Ljava/lang/String;Ljava/lang/String;)J + */ +JNIEXPORT jlong JNICALL +Java_edu_wpi_first_util_datalog_DataLogJNI_fgCreate + (JNIEnv* env, jclass, jstring filename, jstring extraHeader) +{ + if (!filename) { + wpi::ThrowNullPointerException(env, "filename is null"); + return 0; + } + if (!extraHeader) { + wpi::ThrowNullPointerException(env, "extraHeader is null"); + return 0; + } + std::error_code ec; + auto writer = new DataLogWriter{JStringRef{env, filename}, ec, + JStringRef{env, extraHeader}}; + if (ec) { + wpi::ThrowIOException(env, ec.message()); + delete writer; + return 0; + } + return reinterpret_cast(writer); } /* @@ -76,7 +109,7 @@ Java_edu_wpi_first_util_datalog_DataLogJNI_flush wpi::ThrowNullPointerException(env, "impl is null"); return; } - reinterpret_cast(impl)->Flush(); + reinterpret_cast(impl)->Flush(); } /* @@ -92,7 +125,7 @@ Java_edu_wpi_first_util_datalog_DataLogJNI_pause wpi::ThrowNullPointerException(env, "impl is null"); return; } - reinterpret_cast(impl)->Pause(); + reinterpret_cast(impl)->Pause(); } /* @@ -108,7 +141,7 @@ Java_edu_wpi_first_util_datalog_DataLogJNI_resume wpi::ThrowNullPointerException(env, "impl is null"); return; } - reinterpret_cast(impl)->Resume(); + reinterpret_cast(impl)->Resume(); } /* @@ -124,7 +157,7 @@ Java_edu_wpi_first_util_datalog_DataLogJNI_stop wpi::ThrowNullPointerException(env, "impl is null"); return; } - reinterpret_cast(impl)->Stop(); + reinterpret_cast(impl)->Stop(); } /* diff --git a/wpiutil/src/main/native/cpp/jni/WPIUtilJNI.cpp b/wpiutil/src/main/native/cpp/jni/WPIUtilJNI.cpp index c159f7e46b..54b6da2671 100644 --- a/wpiutil/src/main/native/cpp/jni/WPIUtilJNI.cpp +++ b/wpiutil/src/main/native/cpp/jni/WPIUtilJNI.cpp @@ -21,12 +21,14 @@ static uint64_t mockNow = 0; static JException illegalArgEx; static JException indexOobEx; static JException interruptedEx; +static JException ioEx; static JException nullPointerEx; static const JExceptionInit exceptions[] = { {"java/lang/IllegalArgumentException", &illegalArgEx}, {"java/lang/IndexOutOfBoundsException", &indexOobEx}, {"java/lang/InterruptedException", &interruptedEx}, + {"java/io/IOException", &ioEx}, {"java/lang/NullPointerException", &nullPointerEx}}; void wpi::ThrowIllegalArgumentException(JNIEnv* env, std::string_view msg) { @@ -37,6 +39,10 @@ void wpi::ThrowIndexOobException(JNIEnv* env, std::string_view msg) { indexOobEx.Throw(env, msg); } +void wpi::ThrowIOException(JNIEnv* env, std::string_view msg) { + ioEx.Throw(env, msg); +} + void wpi::ThrowNullPointerException(JNIEnv* env, std::string_view msg) { nullPointerEx.Throw(env, msg); } diff --git a/wpiutil/src/main/native/cpp/jni/WPIUtilJNI.h b/wpiutil/src/main/native/cpp/jni/WPIUtilJNI.h index 541064f2b3..4d08fdbf8b 100644 --- a/wpiutil/src/main/native/cpp/jni/WPIUtilJNI.h +++ b/wpiutil/src/main/native/cpp/jni/WPIUtilJNI.h @@ -12,6 +12,7 @@ namespace wpi { void ThrowIllegalArgumentException(JNIEnv* env, std::string_view msg); void ThrowIndexOobException(JNIEnv* env, std::string_view msg); +void ThrowIOException(JNIEnv* env, std::string_view msg); void ThrowNullPointerException(JNIEnv* env, std::string_view msg); } // namespace wpi diff --git a/wpiutil/src/main/native/include/wpi/DataLog.h b/wpiutil/src/main/native/include/wpi/DataLog.h index f6e4b3c7a1..c7665ff1dc 100644 --- a/wpiutil/src/main/native/include/wpi/DataLog.h +++ b/wpiutil/src/main/native/include/wpi/DataLog.h @@ -6,43 +6,25 @@ #include -#ifdef __cplusplus #include -#include #include -#include #include #include #include #include -#include #include #include #include #include +#include "wpi/DataLog_c.h" #include "wpi/DenseMap.h" #include "wpi/SmallVector.h" #include "wpi/StringMap.h" -#include "wpi/condition_variable.h" #include "wpi/mutex.h" #include "wpi/protobuf/Protobuf.h" #include "wpi/struct/Struct.h" #include "wpi/timestamp.h" -#endif // __cplusplus - -/** - * A datalog string (for use with string array). - */ -struct WPI_DataLog_String { - /** Contents. */ - const char* str; - - /** Length. */ - size_t len; -}; - -#ifdef __cplusplus namespace wpi { class Logger; @@ -61,16 +43,11 @@ enum ControlRecordType { } // namespace impl /** - * A data log. The log file is created immediately upon construction with a - * temporary filename. The file may be renamed at any time using the - * SetFilename() function. + * A data log for high-speed writing of data values. * * The lifetime of the data log object must be longer than any data log entry * objects that refer to it. * - * The data log is periodically flushed to disk. It can also be explicitly - * flushed to disk by using the Flush() function. - * * Finish() is needed only to indicate in the log that a particular entry is * no longer being used (it releases the name to ID mapping). Finish() is not * required to be called for data to be flushed to disk; entries in the log @@ -87,105 +64,36 @@ enum ControlRecordType { * arbitrary values), records in the log are not guaranteed to be sorted by * timestamp. */ -class DataLog final { +class DataLog { public: - /** - * Construct a new Data Log. The log will be initially created with a - * temporary filename. - * - * @param dir directory to store the log - * @param filename filename to use; if none provided, a random filename is - * generated of the form "wpilog_{}.wpilog" - * @param period time between automatic flushes to disk, in seconds; - * this is a time/storage tradeoff - * @param extraHeader extra header data - */ - explicit DataLog(std::string_view dir = "", std::string_view filename = "", - double period = 0.25, std::string_view extraHeader = ""); + virtual ~DataLog() = default; - /** - * Construct a new Data Log. The log will be initially created with a - * temporary filename. - * - * @param msglog message logger (will be called from separate thread) - * @param dir directory to store the log - * @param filename filename to use; if none provided, a random filename is - * generated of the form "wpilog_{}.wpilog" - * @param period time between automatic flushes to disk, in seconds; - * this is a time/storage tradeoff - * @param extraHeader extra header data - */ - explicit DataLog(wpi::Logger& msglog, std::string_view dir = "", - std::string_view filename = "", double period = 0.25, - std::string_view extraHeader = ""); - - /** - * Construct a new Data Log that passes its output to the provided function - * rather than a file. The write function will be called on a separate - * background thread and may block. The write function is called with an - * empty data array when the thread is terminating. - * - * @param write write function - * @param period time between automatic calls to write, in seconds; - * this is a time/storage tradeoff - * @param extraHeader extra header data - */ - explicit DataLog(std::function data)> write, - double period = 0.25, std::string_view extraHeader = ""); - - /** - * Construct a new Data Log that passes its output to the provided function - * rather than a file. The write function will be called on a separate - * background thread and may block. The write function is called with an - * empty data array when the thread is terminating. - * - * @param msglog message logger (will be called from separate thread) - * @param write write function - * @param period time between automatic calls to write, in seconds; - * this is a time/storage tradeoff - * @param extraHeader extra header data - */ - explicit DataLog(wpi::Logger& msglog, - std::function data)> write, - double period = 0.25, std::string_view extraHeader = ""); - - ~DataLog(); DataLog(const DataLog&) = delete; DataLog& operator=(const DataLog&) = delete; DataLog(DataLog&&) = delete; DataLog& operator=(const DataLog&&) = delete; - /** - * Change log filename. - * - * @param filename filename - */ - void SetFilename(std::string_view filename); - /** * Explicitly flushes the log data to disk. */ - void Flush(); + virtual void Flush() = 0; /** * Pauses appending of data records to the log. While paused, no data records * are saved (e.g. AppendX is a no-op). Has no effect on entry starts / * finishes / metadata changes. */ - void Pause(); + virtual void Pause(); /** - * Resumes appending of data records to the log. If called after Stop(), - * opens a new file (with random name if SetFilename was not called after - * Stop()) and appends Start records and schema data values for all previously - * started entries and schemas. + * Resumes appending of data records to the log. */ - void Resume(); + virtual void Resume(); /** - * Stops appending all records to the log, and closes the log file. + * Stops appending start/metadata/schema records to the log. */ - void Stop(); + virtual void Stop(); /** * Returns whether there is a data schema already registered with the given @@ -468,13 +376,116 @@ class DataLog final { void AppendStringArray(int entry, std::span arr, int64_t timestamp); - private: - struct WriterThreadState; + protected: + static constexpr size_t kBlockSize = 16 * 1024; + static wpi::Logger s_defaultMessageLog; - void StartLogFile(WriterThreadState& state); - void WriterThreadMain(std::string_view dir); - void WriterThreadMain( - std::function data)> write); + class Buffer { + public: + explicit Buffer(size_t alloc = kBlockSize) + : m_buf{new uint8_t[alloc]}, m_maxLen{alloc} {} + ~Buffer() { delete[] m_buf; } + + Buffer(const Buffer&) = delete; + Buffer& operator=(const Buffer&) = delete; + + Buffer(Buffer&& oth) + : m_buf{oth.m_buf}, m_len{oth.m_len}, m_maxLen{oth.m_maxLen} { + oth.m_buf = nullptr; + oth.m_len = 0; + oth.m_maxLen = 0; + } + + Buffer& operator=(Buffer&& oth) { + if (m_buf) { + delete[] m_buf; + } + m_buf = oth.m_buf; + m_len = oth.m_len; + m_maxLen = oth.m_maxLen; + oth.m_buf = nullptr; + oth.m_len = 0; + oth.m_maxLen = 0; + return *this; + } + + uint8_t* Reserve(size_t size) { + assert(size <= GetRemaining()); + uint8_t* rv = m_buf + m_len; + m_len += size; + return rv; + } + + void Unreserve(size_t size) { m_len -= size; } + + void Clear() { m_len = 0; } + + size_t GetRemaining() const { return m_maxLen - m_len; } + + std::span GetData() { return {m_buf, m_len}; } + std::span GetData() const { return {m_buf, m_len}; } + + private: + uint8_t* m_buf; + size_t m_len = 0; + size_t m_maxLen; + }; + + /** + * Constructs the log. StartFile() must be called to actually start the + * file output. + * + * @param msglog message logger (will be called from separate thread) + * @param extraHeader extra header metadata + */ + explicit DataLog(wpi::Logger& msglog, std::string_view extraHeader = "") + : m_msglog{msglog}, m_extraHeader{extraHeader} {} + + /** + * Starts the log. Appends file header and Start records and schema data + * values for all previously started entries and schemas. No effect unless + * the data log is currently stopped. + */ + void StartFile(); + + /** + * Provides complete set of all buffers that need to be written. + * + * Any existing contents of writeBufs will be released as if ReleaseBufs() + * was called prior to this call. + * + * Returned buffers should provided back via ReleaseBufs() after the write is + * complete. + * + * @param writeBufs buffers to be written (output) + */ + void FlushBufs(std::vector* writeBufs); + + /** + * Releases memory for a set of buffers back to the internal buffer pool. + * + * @param bufs buffers; empty on return + */ + void ReleaseBufs(std::vector* bufs); + + /** + * Called when internal buffers are half the maximum count. Called with + * internal mutex held; do not call any other DataLog functions from this + * function. + */ + virtual void BufferHalfFull(); + + /** + * Called when internal buffers reach the maximum count. Called with internal + * mutex held; do not call any other DataLog functions from this function. + * + * @return true if log should be paused (don't call PauseLog) + */ + virtual bool BufferFull() = 0; + + private: + static constexpr size_t kMaxBufferCount = 1024 * 1024 / kBlockSize; + static constexpr size_t kMaxFreeCount = 256 * 1024 / kBlockSize; // must be called with m_mutex held int StartImpl(std::string_view name, std::string_view type, @@ -486,22 +497,16 @@ class DataLog final { void AppendStringImpl(std::string_view str); void AppendStartRecord(int id, std::string_view name, std::string_view type, std::string_view metadata, int64_t timestamp); + void DoReleaseBufs(std::vector* bufs); + protected: wpi::Logger& m_msglog; + + private: mutable wpi::mutex m_mutex; - wpi::condition_variable m_cond; - bool m_doFlush{false}; - bool m_shutdown{false}; - enum State { - kStart, - kActive, - kPaused, - kStopped, - } m_state = kActive; - double m_period; + bool m_active = false; + bool m_paused = false; std::string m_extraHeader; - std::string m_newFilename; - class Buffer; std::vector m_free; std::vector m_outgoing; struct EntryInfo { @@ -516,7 +521,6 @@ class DataLog final { }; wpi::DenseMap m_entryIds; int m_lastId = 0; - std::thread m_thread; }; /** @@ -1103,286 +1107,3 @@ class ProtobufLogEntry : public DataLogEntry { }; } // namespace wpi::log - -extern "C" { -#endif // __cplusplus - -/** C-compatible data log (opaque struct). */ -struct WPI_DataLog; - -/** - * Construct a new Data Log. The log will be initially created with a - * temporary filename. - * - * @param dir directory to store the log - * @param filename filename to use; if none provided, a random filename is - * generated of the form "wpilog_{}.wpilog" - * @param period time between automatic flushes to disk, in seconds; - * this is a time/storage tradeoff - * @param extraHeader extra header data - */ -struct WPI_DataLog* WPI_DataLog_Create(const char* dir, const char* filename, - double period, const char* extraHeader); - -/** - * Construct a new Data Log that passes its output to the provided function - * rather than a file. The write function will be called on a separate - * background thread and may block. The write function is called with an - * empty data array (data=NULL, len=0) when the thread is terminating. - * - * @param write write function - * @param ptr pointer to pass to write function ptr parameter - * @param period time between automatic calls to write, in seconds; - * this is a time/storage tradeoff - * @param extraHeader extra header data - */ -struct WPI_DataLog* WPI_DataLog_Create_Func( - void (*write)(void* ptr, const uint8_t* data, size_t len), void* ptr, - double period, const char* extraHeader); - -/** - * Releases a data log object. Closes the file and returns resources to the - * system. - * - * @param datalog data log - */ -void WPI_DataLog_Release(struct WPI_DataLog* datalog); - -/** - * Change log filename. - * - * @param datalog data log - * @param filename filename - */ -void WPI_DataLog_SetFilename(struct WPI_DataLog* datalog, const char* filename); - -/** - * Explicitly flushes the log data to disk. - * - * @param datalog data log - */ -void WPI_DataLog_Flush(struct WPI_DataLog* datalog); - -/** - * Pauses appending of data records to the log. While paused, no data records - * are saved (e.g. AppendX is a no-op). Has no effect on entry starts / - * finishes / metadata changes. - * - * @param datalog data log - */ -void WPI_DataLog_Pause(struct WPI_DataLog* datalog); - -/** - * Resumes appending of data records to the log. If called after Stop(), - * opens a new file (with random name if SetFilename was not called after - * Stop()) and appends Start records and schema data values for all previously - * started entries and schemas. - * - * @param datalog data log - */ -void WPI_DataLog_Resume(struct WPI_DataLog* datalog); - -/** - * Stops appending all records to the log, and closes the log file. - * - * @param datalog data log - */ -void WPI_DataLog_Stop(struct WPI_DataLog* datalog); - -/** - * Start an entry. Duplicate names are allowed (with the same type), and - * result in the same index being returned (Start/Finish are reference - * counted). A duplicate name with a different type will result in an error - * message being printed to the console and 0 being returned (which will be - * ignored by the Append functions). - * - * @param datalog data log - * @param name Name - * @param type Data type - * @param metadata Initial metadata (e.g. data properties) - * @param timestamp Time stamp (may be 0 to indicate now) - * - * @return Entry index - */ -int WPI_DataLog_Start(struct WPI_DataLog* datalog, const char* name, - const char* type, const char* metadata, - int64_t timestamp); - -/** - * Finish an entry. - * - * @param datalog data log - * @param entry Entry index - * @param timestamp Time stamp (may be 0 to indicate now) - */ -void WPI_DataLog_Finish(struct WPI_DataLog* datalog, int entry, - int64_t timestamp); - -/** - * Updates the metadata for an entry. - * - * @param datalog data log - * @param entry Entry index - * @param metadata New metadata for the entry - * @param timestamp Time stamp (may be 0 to indicate now) - */ -void WPI_DataLog_SetMetadata(struct WPI_DataLog* datalog, int entry, - const char* metadata, int64_t timestamp); - -/** - * Appends a raw record to the log. - * - * @param datalog data log - * @param entry Entry index, as returned by WPI_DataLog_Start() - * @param data Byte array to record - * @param len Length of byte array - * @param timestamp Time stamp (may be 0 to indicate now) - */ -void WPI_DataLog_AppendRaw(struct WPI_DataLog* datalog, int entry, - const uint8_t* data, size_t len, int64_t timestamp); - -/** - * Appends a boolean record to the log. - * - * @param datalog data log - * @param entry Entry index, as returned by WPI_DataLog_Start() - * @param value Boolean value to record - * @param timestamp Time stamp (may be 0 to indicate now) - */ -void WPI_DataLog_AppendBoolean(struct WPI_DataLog* datalog, int entry, - int value, int64_t timestamp); - -/** - * Appends an integer record to the log. - * - * @param datalog data log - * @param entry Entry index, as returned by WPI_DataLog_Start() - * @param value Integer value to record - * @param timestamp Time stamp (may be 0 to indicate now) - */ -void WPI_DataLog_AppendInteger(struct WPI_DataLog* datalog, int entry, - int64_t value, int64_t timestamp); - -/** - * Appends a float record to the log. - * - * @param datalog data log - * @param entry Entry index, as returned by WPI_DataLog_Start() - * @param value Float value to record - * @param timestamp Time stamp (may be 0 to indicate now) - */ -void WPI_DataLog_AppendFloat(struct WPI_DataLog* datalog, int entry, - float value, int64_t timestamp); - -/** - * Appends a double record to the log. - * - * @param datalog data log - * @param entry Entry index, as returned by WPI_DataLog_Start() - * @param value Double value to record - * @param timestamp Time stamp (may be 0 to indicate now) - */ -void WPI_DataLog_AppendDouble(struct WPI_DataLog* datalog, int entry, - double value, int64_t timestamp); - -/** - * Appends a string record to the log. - * - * @param datalog data log - * @param entry Entry index, as returned by WPI_DataLog_Start() - * @param value String value to record - * @param len Length of string - * @param timestamp Time stamp (may be 0 to indicate now) - */ -void WPI_DataLog_AppendString(struct WPI_DataLog* datalog, int entry, - const char* value, size_t len, int64_t timestamp); - -/** - * Appends a boolean array record to the log. - * - * @param datalog data log - * @param entry Entry index, as returned by WPI_DataLog_Start() - * @param arr Boolean array to record - * @param len Number of elements in array - * @param timestamp Time stamp (may be 0 to indicate now) - */ -void WPI_DataLog_AppendBooleanArray(struct WPI_DataLog* datalog, int entry, - const int* arr, size_t len, - int64_t timestamp); - -/** - * Appends a boolean array record to the log. - * - * @param datalog data log - * @param entry Entry index, as returned by WPI_DataLog_Start() - * @param arr Boolean array to record - * @param len Number of elements in array - * @param timestamp Time stamp (may be 0 to indicate now) - */ -void WPI_DataLog_AppendBooleanArrayByte(struct WPI_DataLog* datalog, int entry, - const uint8_t* arr, size_t len, - int64_t timestamp); - -/** - * Appends an integer array record to the log. - * - * @param datalog data log - * @param entry Entry index, as returned by WPI_DataLog_Start() - * @param arr Integer array to record - * @param len Number of elements in array - * @param timestamp Time stamp (may be 0 to indicate now) - */ -void WPI_DataLog_AppendIntegerArray(struct WPI_DataLog* datalog, int entry, - const int64_t* arr, size_t len, - int64_t timestamp); - -/** - * Appends a float array record to the log. - * - * @param datalog data log - * @param entry Entry index, as returned by WPI_DataLog_Start() - * @param arr Float array to record - * @param len Number of elements in array - * @param timestamp Time stamp (may be 0 to indicate now) - */ -void WPI_DataLog_AppendFloatArray(struct WPI_DataLog* datalog, int entry, - const float* arr, size_t len, - int64_t timestamp); - -/** - * Appends a double array record to the log. - * - * @param datalog data log - * @param entry Entry index, as returned by WPI_DataLog_Start() - * @param arr Double array to record - * @param len Number of elements in array - * @param timestamp Time stamp (may be 0 to indicate now) - */ -void WPI_DataLog_AppendDoubleArray(struct WPI_DataLog* datalog, int entry, - const double* arr, size_t len, - int64_t timestamp); - -/** - * Appends a string array record to the log. - * - * @param datalog data log - * @param entry Entry index, as returned by WPI_DataLog_Start() - * @param arr String array to record - * @param len Number of elements in array - * @param timestamp Time stamp (may be 0 to indicate now) - */ -void WPI_DataLog_AppendStringArray(struct WPI_DataLog* datalog, int entry, - const WPI_DataLog_String* arr, size_t len, - int64_t timestamp); - -void WPI_DataLog_AddSchemaString(struct WPI_DataLog* datalog, const char* name, - const char* type, const char* schema, - int64_t timestamp); - -void WPI_DataLog_AddSchema(struct WPI_DataLog* datalog, const char* name, - const char* type, const uint8_t* schema, - size_t schema_len, int64_t timestamp); - -#ifdef __cplusplus -} // extern "C" -#endif // __cplusplus diff --git a/wpiutil/src/main/native/include/wpi/DataLogBackgroundWriter.h b/wpiutil/src/main/native/include/wpi/DataLogBackgroundWriter.h new file mode 100644 index 0000000000..7926299ca1 --- /dev/null +++ b/wpiutil/src/main/native/include/wpi/DataLogBackgroundWriter.h @@ -0,0 +1,170 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include + +#include +#include +#include +#include +#include + +#include "wpi/DataLog.h" +#include "wpi/condition_variable.h" +#include "wpi/mutex.h" + +namespace wpi { +class Logger; +} // namespace wpi + +namespace wpi::log { + +/** + * A data log background writer that periodically flushes the data log on a + * background thread. The data log file is created immediately upon + * construction with a temporary filename. The file may be renamed at any time + * using the SetFilename() function. + * + * The lifetime of this object must be longer than any data log entry objects + * that refer to it. + * + * The data log is periodically flushed to disk. It can also be explicitly + * flushed to disk by using the Flush() function. This operation is, however, + * non-blocking. + */ +class DataLogBackgroundWriter final : public DataLog { + public: + /** + * Construct a new Data Log. The log will be initially created with a + * temporary filename. + * + * @param dir directory to store the log + * @param filename filename to use; if none provided, a random filename is + * generated of the form "wpilog_{}.wpilog" + * @param period time between automatic flushes to disk, in seconds; + * this is a time/storage tradeoff + * @param extraHeader extra header data + */ + explicit DataLogBackgroundWriter(std::string_view dir = "", + std::string_view filename = "", + double period = 0.25, + std::string_view extraHeader = ""); + + /** + * Construct a new Data Log. The log will be initially created with a + * temporary filename. + * + * @param msglog message logger (will be called from separate thread) + * @param dir directory to store the log + * @param filename filename to use; if none provided, a random filename is + * generated of the form "wpilog_{}.wpilog" + * @param period time between automatic flushes to disk, in seconds; + * this is a time/storage tradeoff + * @param extraHeader extra header data + */ + explicit DataLogBackgroundWriter(wpi::Logger& msglog, + std::string_view dir = "", + std::string_view filename = "", + double period = 0.25, + std::string_view extraHeader = ""); + + /** + * Construct a new Data Log that passes its output to the provided function + * rather than a file. The write function will be called on a separate + * background thread and may block. The write function is called with an + * empty data array when the thread is terminating. + * + * @param write write function + * @param period time between automatic calls to write, in seconds; + * this is a time/storage tradeoff + * @param extraHeader extra header data + */ + explicit DataLogBackgroundWriter( + std::function data)> write, + double period = 0.25, std::string_view extraHeader = ""); + + /** + * Construct a new Data Log that passes its output to the provided function + * rather than a file. The write function will be called on a separate + * background thread and may block. The write function is called with an + * empty data array when the thread is terminating. + * + * @param msglog message logger (will be called from separate thread) + * @param write write function + * @param period time between automatic calls to write, in seconds; + * this is a time/storage tradeoff + * @param extraHeader extra header data + */ + explicit DataLogBackgroundWriter( + wpi::Logger& msglog, + std::function data)> write, + double period = 0.25, std::string_view extraHeader = ""); + + ~DataLogBackgroundWriter() final; + DataLogBackgroundWriter(const DataLogBackgroundWriter&) = delete; + DataLogBackgroundWriter& operator=(const DataLogBackgroundWriter&) = delete; + DataLogBackgroundWriter(DataLogBackgroundWriter&&) = delete; + DataLogBackgroundWriter& operator=(const DataLogBackgroundWriter&&) = delete; + + /** + * Change log filename. + * + * @param filename filename + */ + void SetFilename(std::string_view filename); + + /** + * Explicitly flushes the log data to disk. + */ + void Flush() final; + + /** + * Pauses appending of data records to the log. While paused, no data records + * are saved (e.g. AppendX is a no-op). Has no effect on entry starts / + * finishes / metadata changes. + */ + void Pause() final; + + /** + * Resumes appending of data records to the log. If called after Stop(), + * opens a new file (with random name if SetFilename was not called after + * Stop()) and appends Start records and schema data values for all previously + * started entries and schemas. + */ + void Resume() final; + + /** + * Stops appending all records to the log, and closes the log file. + */ + void Stop() final; + + private: + struct WriterThreadState; + + void BufferHalfFull() final; + bool BufferFull() final; + + void StartLogFile(WriterThreadState& state); + void WriterThreadMain(std::string_view dir); + void WriterThreadMain( + std::function data)> write); + + mutable wpi::mutex m_mutex; + wpi::condition_variable m_cond; + bool m_doFlush{false}; + bool m_shutdown{false}; + enum State { + kStart, + kActive, + kPaused, + kStopped, + } m_state = kActive; + double m_period; + std::string m_newFilename; + std::thread m_thread; +}; + +} // namespace wpi::log diff --git a/wpiutil/src/main/native/include/wpi/DataLogWriter.h b/wpiutil/src/main/native/include/wpi/DataLogWriter.h new file mode 100644 index 0000000000..15e106e94f --- /dev/null +++ b/wpiutil/src/main/native/include/wpi/DataLogWriter.h @@ -0,0 +1,90 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include +#include +#include + +#include "wpi/DataLog.h" + +namespace wpi { +class raw_ostream; +class Logger; +} // namespace wpi + +namespace wpi::log { + +/** + * A data log writer that flushes the data log to a file when Flush() is called. + * + * The lifetime of this object must be longer than any data log entry objects + * that refer to it. + */ +class DataLogWriter final : public DataLog { + public: + /** + * Constructs with a filename. + * + * @param filename filename to use + * @param ec error code if failed to open file (output) + * @param extraHeader extra header data + */ + explicit DataLogWriter(std::string_view filename, std::error_code& ec, + std::string_view extraHeader = ""); + + /** + * Construct with a filename. + * + * @param msglog message logger + * @param filename filename to use + * @param ec error code if failed to open file (output) + * @param extraHeader extra header data + */ + DataLogWriter(wpi::Logger& msglog, std::string_view filename, + std::error_code& ec, std::string_view extraHeader = ""); + + /** + * Constructs with an output stream. + * + * @param os output stream + * @param extraHeader extra header data + */ + explicit DataLogWriter(std::unique_ptr os, + std::string_view extraHeader = ""); + + /** + * Constructs with an output stream. + * + * @param msglog message logger + * @param os output stream + * @param extraHeader extra header data + */ + DataLogWriter(wpi::Logger& msglog, std::unique_ptr os, + std::string_view extraHeader = ""); + + ~DataLogWriter() final; + DataLogWriter(const DataLogWriter&) = delete; + DataLogWriter& operator=(const DataLogWriter&) = delete; + DataLogWriter(DataLogWriter&&) = delete; + DataLogWriter& operator=(const DataLogWriter&&) = delete; + + /** + * Flushes the log data to disk. + */ + void Flush() final; + + /** + * Stops appending all records to the log, and closes the log file. + */ + void Stop() final; + + private: + bool BufferFull() final; + + std::unique_ptr m_os; +}; + +} // namespace wpi::log diff --git a/wpiutil/src/main/native/include/wpi/DataLog_c.h b/wpiutil/src/main/native/include/wpi/DataLog_c.h new file mode 100644 index 0000000000..c4f48e3a38 --- /dev/null +++ b/wpiutil/src/main/native/include/wpi/DataLog_c.h @@ -0,0 +1,318 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include // NOLINT + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * A datalog string (for use with string array). + */ +struct WPI_DataLog_String { + /** Contents. */ + const char* str; + + /** Length. */ + size_t len; +}; + +/** C-compatible data log (opaque struct). */ +struct WPI_DataLog; + +/** + * Construct a new Data Log. + * + * @param filename filename to use + * @param errorCode error if file failed to open (output) + * @param extraHeader extra header data + */ +struct WPI_DataLog* WPI_DataLog_CreateWriter(const char* filename, + int* errorCode, + const char* extraHeader); + +/** + * Construct a new Data Log background writer. The log will be initially + * created with a temporary filename. + * + * @param dir directory to store the log + * @param filename filename to use; if none provided, a random filename is + * generated of the form "wpilog_{}.wpilog" + * @param period time between automatic flushes to disk, in seconds; + * this is a time/storage tradeoff + * @param extraHeader extra header data + */ +struct WPI_DataLog* WPI_DataLog_CreateBackgroundWriter(const char* dir, + const char* filename, + double period, + const char* extraHeader); + +/** + * Construct a new Data Log background writer that passes its output to the + * provided function rather than a file. The write function will be called on a + * separate background thread and may block. The write function is called with + * an empty data array (data=NULL, len=0) when the thread is terminating. + * + * @param write write function + * @param ptr pointer to pass to write function ptr parameter + * @param period time between automatic calls to write, in seconds; + * this is a time/storage tradeoff + * @param extraHeader extra header data + */ +struct WPI_DataLog* WPI_DataLog_CreateBackgroundWriter_Func( + void (*write)(void* ptr, const uint8_t* data, size_t len), void* ptr, + double period, const char* extraHeader); + +/** + * Change log filename. Can only be used on background writer data logs. + * + * @param datalog data log + * @param filename filename + */ +void WPI_DataLog_SetBackgroundWriterFilename(struct WPI_DataLog* datalog, + const char* filename); + +/** + * Releases a data log object. Closes the file and returns resources to the + * system. + * + * @param datalog data log + */ +void WPI_DataLog_Release(struct WPI_DataLog* datalog); + +/** + * Explicitly flushes the log data to disk. + * + * @param datalog data log + */ +void WPI_DataLog_Flush(struct WPI_DataLog* datalog); + +/** + * Pauses appending of data records to the log. While paused, no data records + * are saved (e.g. AppendX is a no-op). Has no effect on entry starts / + * finishes / metadata changes. + * + * @param datalog data log + */ +void WPI_DataLog_Pause(struct WPI_DataLog* datalog); + +/** + * Resumes appending of data records to the log. If called after Stop(), + * opens a new file (with random name if SetFilename was not called after + * Stop()) and appends Start records and schema data values for all previously + * started entries and schemas. + * + * @param datalog data log + */ +void WPI_DataLog_Resume(struct WPI_DataLog* datalog); + +/** + * Stops appending all records to the log, and closes the log file. + * + * @param datalog data log + */ +void WPI_DataLog_Stop(struct WPI_DataLog* datalog); + +/** + * Start an entry. Duplicate names are allowed (with the same type), and + * result in the same index being returned (Start/Finish are reference + * counted). A duplicate name with a different type will result in an error + * message being printed to the console and 0 being returned (which will be + * ignored by the Append functions). + * + * @param datalog data log + * @param name Name + * @param type Data type + * @param metadata Initial metadata (e.g. data properties) + * @param timestamp Time stamp (may be 0 to indicate now) + * + * @return Entry index + */ +int WPI_DataLog_Start(struct WPI_DataLog* datalog, const char* name, + const char* type, const char* metadata, + int64_t timestamp); + +/** + * Finish an entry. + * + * @param datalog data log + * @param entry Entry index + * @param timestamp Time stamp (may be 0 to indicate now) + */ +void WPI_DataLog_Finish(struct WPI_DataLog* datalog, int entry, + int64_t timestamp); + +/** + * Updates the metadata for an entry. + * + * @param datalog data log + * @param entry Entry index + * @param metadata New metadata for the entry + * @param timestamp Time stamp (may be 0 to indicate now) + */ +void WPI_DataLog_SetMetadata(struct WPI_DataLog* datalog, int entry, + const char* metadata, int64_t timestamp); + +/** + * Appends a raw record to the log. + * + * @param datalog data log + * @param entry Entry index, as returned by WPI_DataLog_Start() + * @param data Byte array to record + * @param len Length of byte array + * @param timestamp Time stamp (may be 0 to indicate now) + */ +void WPI_DataLog_AppendRaw(struct WPI_DataLog* datalog, int entry, + const uint8_t* data, size_t len, int64_t timestamp); + +/** + * Appends a boolean record to the log. + * + * @param datalog data log + * @param entry Entry index, as returned by WPI_DataLog_Start() + * @param value Boolean value to record + * @param timestamp Time stamp (may be 0 to indicate now) + */ +void WPI_DataLog_AppendBoolean(struct WPI_DataLog* datalog, int entry, + int value, int64_t timestamp); + +/** + * Appends an integer record to the log. + * + * @param datalog data log + * @param entry Entry index, as returned by WPI_DataLog_Start() + * @param value Integer value to record + * @param timestamp Time stamp (may be 0 to indicate now) + */ +void WPI_DataLog_AppendInteger(struct WPI_DataLog* datalog, int entry, + int64_t value, int64_t timestamp); + +/** + * Appends a float record to the log. + * + * @param datalog data log + * @param entry Entry index, as returned by WPI_DataLog_Start() + * @param value Float value to record + * @param timestamp Time stamp (may be 0 to indicate now) + */ +void WPI_DataLog_AppendFloat(struct WPI_DataLog* datalog, int entry, + float value, int64_t timestamp); + +/** + * Appends a double record to the log. + * + * @param datalog data log + * @param entry Entry index, as returned by WPI_DataLog_Start() + * @param value Double value to record + * @param timestamp Time stamp (may be 0 to indicate now) + */ +void WPI_DataLog_AppendDouble(struct WPI_DataLog* datalog, int entry, + double value, int64_t timestamp); + +/** + * Appends a string record to the log. + * + * @param datalog data log + * @param entry Entry index, as returned by WPI_DataLog_Start() + * @param value String value to record + * @param len Length of string + * @param timestamp Time stamp (may be 0 to indicate now) + */ +void WPI_DataLog_AppendString(struct WPI_DataLog* datalog, int entry, + const char* value, size_t len, int64_t timestamp); + +/** + * Appends a boolean array record to the log. + * + * @param datalog data log + * @param entry Entry index, as returned by WPI_DataLog_Start() + * @param arr Boolean array to record + * @param len Number of elements in array + * @param timestamp Time stamp (may be 0 to indicate now) + */ +void WPI_DataLog_AppendBooleanArray(struct WPI_DataLog* datalog, int entry, + const int* arr, size_t len, + int64_t timestamp); + +/** + * Appends a boolean array record to the log. + * + * @param datalog data log + * @param entry Entry index, as returned by WPI_DataLog_Start() + * @param arr Boolean array to record + * @param len Number of elements in array + * @param timestamp Time stamp (may be 0 to indicate now) + */ +void WPI_DataLog_AppendBooleanArrayByte(struct WPI_DataLog* datalog, int entry, + const uint8_t* arr, size_t len, + int64_t timestamp); + +/** + * Appends an integer array record to the log. + * + * @param datalog data log + * @param entry Entry index, as returned by WPI_DataLog_Start() + * @param arr Integer array to record + * @param len Number of elements in array + * @param timestamp Time stamp (may be 0 to indicate now) + */ +void WPI_DataLog_AppendIntegerArray(struct WPI_DataLog* datalog, int entry, + const int64_t* arr, size_t len, + int64_t timestamp); + +/** + * Appends a float array record to the log. + * + * @param datalog data log + * @param entry Entry index, as returned by WPI_DataLog_Start() + * @param arr Float array to record + * @param len Number of elements in array + * @param timestamp Time stamp (may be 0 to indicate now) + */ +void WPI_DataLog_AppendFloatArray(struct WPI_DataLog* datalog, int entry, + const float* arr, size_t len, + int64_t timestamp); + +/** + * Appends a double array record to the log. + * + * @param datalog data log + * @param entry Entry index, as returned by WPI_DataLog_Start() + * @param arr Double array to record + * @param len Number of elements in array + * @param timestamp Time stamp (may be 0 to indicate now) + */ +void WPI_DataLog_AppendDoubleArray(struct WPI_DataLog* datalog, int entry, + const double* arr, size_t len, + int64_t timestamp); + +/** + * Appends a string array record to the log. + * + * @param datalog data log + * @param entry Entry index, as returned by WPI_DataLog_Start() + * @param arr String array to record + * @param len Number of elements in array + * @param timestamp Time stamp (may be 0 to indicate now) + */ +void WPI_DataLog_AppendStringArray(struct WPI_DataLog* datalog, int entry, + const WPI_DataLog_String* arr, size_t len, + int64_t timestamp); + +void WPI_DataLog_AddSchemaString(struct WPI_DataLog* datalog, const char* name, + const char* type, const char* schema, + int64_t timestamp); + +void WPI_DataLog_AddSchema(struct WPI_DataLog* datalog, const char* name, + const char* type, const uint8_t* schema, + size_t schema_len, int64_t timestamp); + +#ifdef __cplusplus +} // extern "C" +#endif diff --git a/wpiutil/src/test/native/cpp/DataLogTest.cpp b/wpiutil/src/test/native/cpp/DataLogTest.cpp index dd0f3d12d7..f4389abcb0 100644 --- a/wpiutil/src/test/native/cpp/DataLogTest.cpp +++ b/wpiutil/src/test/native/cpp/DataLogTest.cpp @@ -3,10 +3,13 @@ // the WPILib BSD license file in the root directory of this project. #include +#include #include -#include "wpi/DataLog.h" +#include "wpi/DataLogWriter.h" +#include "wpi/Logger.h" +#include "wpi/raw_ostream.h" namespace { struct ThingA { @@ -118,19 +121,22 @@ static_assert(wpi::StructSerializable); static_assert(wpi::StructSerializable); static_assert(wpi::StructSerializable); -TEST(DataLogTest, SimpleInt) { +class DataLogTest : public ::testing::Test { + public: + wpi::Logger msglog; std::vector data; - { - wpi::log::DataLog log{ - [&](auto out) { data.insert(data.end(), out.begin(), out.end()); }}; - int entry = log.Start("test", "int64"); - log.AppendInteger(entry, 1, 0); - } + wpi::log::DataLogWriter log{msglog, + std::make_unique(data)}; +}; + +TEST_F(DataLogTest, SimpleInt) { + int entry = log.Start("test", "int64"); + log.AppendInteger(entry, 1, 0); + log.Flush(); ASSERT_EQ(data.size(), 66u); } -TEST(DataLogTest, StructA) { - wpi::log::DataLog log{[](auto) {}}; +TEST_F(DataLogTest, StructA) { [[maybe_unused]] wpi::log::StructLogEntry entry0; wpi::log::StructLogEntry entry{log, "a", 5}; @@ -138,8 +144,7 @@ TEST(DataLogTest, StructA) { entry.Append(ThingA{}, 7); } -TEST(DataLogTest, StructArrayA) { - wpi::log::DataLog log{[](auto) {}}; +TEST_F(DataLogTest, StructArrayA) { [[maybe_unused]] wpi::log::StructArrayLogEntry entry0; wpi::log::StructArrayLogEntry entry{log, "a", 5}; @@ -147,8 +152,7 @@ TEST(DataLogTest, StructArrayA) { entry.Append({{ThingA{}, ThingA{}}}, 7); } -TEST(DataLogTest, StructFixedArrayA) { - wpi::log::DataLog log{[](auto) {}}; +TEST_F(DataLogTest, StructFixedArrayA) { [[maybe_unused]] wpi::log::StructArrayLogEntry> entry0; wpi::log::StructLogEntry> entry{log, "a", 5}; @@ -157,8 +161,7 @@ TEST(DataLogTest, StructFixedArrayA) { entry.Append(arr, 7); } -TEST(DataLogTest, StructB) { - wpi::log::DataLog log{[](auto) {}}; +TEST_F(DataLogTest, StructB) { Info1 info; [[maybe_unused]] wpi::log::StructLogEntry entry0; @@ -167,8 +170,7 @@ TEST(DataLogTest, StructB) { entry.Append(ThingB{}, 7); } -TEST(DataLogTest, StructArrayB) { - wpi::log::DataLog log{[](auto) {}}; +TEST_F(DataLogTest, StructArrayB) { Info1 info; [[maybe_unused]] wpi::log::StructArrayLogEntry entry0; @@ -177,8 +179,7 @@ TEST(DataLogTest, StructArrayB) { entry.Append({{ThingB{}, ThingB{}}}, 7); } -TEST(DataLogTest, StructFixedArrayB) { - wpi::log::DataLog log{[](auto) {}}; +TEST_F(DataLogTest, StructFixedArrayB) { Info1 info; wpi::log::StructLogEntry, Info1> entry{log, "a", info, 5}; @@ -187,8 +188,7 @@ TEST(DataLogTest, StructFixedArrayB) { entry.Append(arr, 7); } -TEST(DataLogTest, StructC) { - wpi::log::DataLog log{[](auto) {}}; +TEST_F(DataLogTest, StructC) { { wpi::log::StructLogEntry entry{log, "c", 5}; entry.Append(ThingC{}); @@ -208,8 +208,7 @@ TEST(DataLogTest, StructC) { } } -TEST(DataLogTest, StructArrayC) { - wpi::log::DataLog log{[](auto) {}}; +TEST_F(DataLogTest, StructArrayC) { { wpi::log::StructArrayLogEntry entry{log, "c", 5}; entry.Append({{ThingC{}, ThingC{}}}); @@ -229,8 +228,7 @@ TEST(DataLogTest, StructArrayC) { } } -TEST(DataLogTest, StructFixedArrayC) { - wpi::log::DataLog log{[](auto) {}}; +TEST_F(DataLogTest, StructFixedArrayC) { std::array arr; { wpi::log::StructLogEntry> entry{log, "c", 5};