diff --git a/wpilibc/src/main/native/cpp/DataLogManager.cpp b/wpilibc/src/main/native/cpp/DataLogManager.cpp new file mode 100644 index 0000000000..c5546c42ba --- /dev/null +++ b/wpilibc/src/main/native/cpp/DataLogManager.cpp @@ -0,0 +1,311 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#include "frc/DataLogManager.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "frc/DriverStation.h" +#include "frc/Filesystem.h" + +using namespace frc; + +namespace { + +struct Thread final : public wpi::SafeThread { + Thread(std::string_view dir, std::string_view filename, double period); + + void Main() final; + + void StartNTLog(); + void StopNTLog(); + + std::string m_logDir; + bool m_filenameOverride; + wpi::log::DataLog m_log; + bool m_ntLoggerEnabled = false; + NT_DataLogger m_ntEntryLogger = 0; + NT_ConnectionDataLogger m_ntConnLogger = 0; + wpi::log::StringLogEntry m_messageLog; +}; + +struct Instance { + Instance(std::string_view dir, std::string_view filename, double period); + wpi::SafeThreadOwner owner; +}; + +} // namespace + +// if less than this much free space, delete log files until there is this much +// free space OR there are this many files remaining. +static constexpr uintmax_t kFreeSpaceThreshold = 50000000; +static constexpr int kFileCountThreshold = 10; + +static std::string MakeLogDir(std::string_view dir) { + if (!dir.empty()) { + return std::string{dir}; + } +#ifdef __FRC_ROBORIO__ + // prefer a mounted USB drive if one is accessible + constexpr std::string_view usbDir{"/u"}; + std::error_code ec; + auto s = fs::status(usbDir, ec); + if (!ec && fs::is_directory(s) && + (s.permissions() & fs::perms::others_write) != fs::perms::none) { + return std::string{usbDir}; + } +#endif + return frc::filesystem::GetOperatingDirectory(); +} + +static std::string MakeLogFilename(std::string_view filenameOverride) { + if (!filenameOverride.empty()) { + return std::string{filenameOverride}; + } + static std::random_device dev; + static std::mt19937 rng(dev()); + std::uniform_int_distribution dist(0, 15); + const char* v = "0123456789abcdef"; + std::string filename = "FRC_TBD_"; + for (int i = 0; i < 16; i++) { + filename += v[dist(rng)]; + } + filename += ".wpilog"; + return filename; +} + +Thread::Thread(std::string_view dir, std::string_view filename, double period) + : m_logDir{dir}, + m_filenameOverride{!filename.empty()}, + m_log{dir, MakeLogFilename(filename), period}, + m_messageLog{m_log, "messages"} { + StartNTLog(); +} + +void Thread::Main() { + // based on free disk space, scan for "old" FRC_*.wpilog files and remove + { + uintmax_t freeSpace = fs::space(m_logDir).free; + if (freeSpace < kFreeSpaceThreshold) { + // Delete oldest FRC_*.wpilog files (ignore FRC_TBD_*.wpilog as we just + // created one) + std::vector entries; + std::error_code ec; + for (auto&& entry : fs::directory_iterator{m_logDir, ec}) { + auto stem = entry.path().stem().string(); + if (wpi::starts_with(stem, "FRC_") && + entry.path().extension() == ".wpilog" && + !wpi::starts_with(stem, "FRC_TBD_")) { + entries.emplace_back(entry); + } + } + std::sort(entries.begin(), entries.end(), + [](const auto& a, const auto& b) { + return a.last_write_time() < b.last_write_time(); + }); + + int count = entries.size(); + for (auto&& entry : entries) { + --count; + if (count < kFileCountThreshold) { + break; + } + auto size = entry.file_size(); + if (fs::remove(entry.path(), ec)) { + freeSpace += size; + if (freeSpace >= kFreeSpaceThreshold) { + break; + } + } else { + fmt::print(stderr, "DataLogManager: could not delete {}\n", + entry.path().string()); + } + } + } + } + + int timeoutCount = 0; + bool paused = false; + int dsAttachCount = 0; + int fmsAttachCount = 0; + bool dsRenamed = m_filenameOverride; + bool fmsRenamed = m_filenameOverride; + int sysTimeCount = 0; + wpi::log::IntegerLogEntry sysTimeEntry{ + m_log, "systemTime", + "{\"source\":\"DataLogManager\",\"format\":\"time_t_us\"}"}; + + for (;;) { + bool newData = DriverStation::WaitForData(0.25_s); + if (!m_active) { + break; + } + if (!newData) { + ++timeoutCount; + // pause logging after being disconnected for 10 seconds + if (timeoutCount > 40 && !paused) { + timeoutCount = 0; + paused = true; + m_log.Pause(); + } + continue; + } + // when we connect to the DS, resume logging + timeoutCount = 0; + if (paused) { + paused = false; + m_log.Resume(); + } + + if (!dsRenamed) { + // track DS attach + if (DriverStation::IsDSAttached()) { + ++dsAttachCount; + } else { + dsAttachCount = 0; + } + if (dsAttachCount > 50) { // 1 second + std::time_t now = std::time(nullptr); + auto tm = std::gmtime(&now); + if (tm->tm_year > 100) { + // assume local clock is now synchronized to DS, so rename based on + // local time + m_log.SetFilename(fmt::format("FRC_{:%Y%m%d_%H%M%S}.wpilog", *tm)); + dsRenamed = true; + } else { + dsAttachCount = 0; // wait a bit and try again + } + } + } + + if (!fmsRenamed) { + // track FMS attach + if (DriverStation::IsFMSAttached()) { + ++fmsAttachCount; + } else { + fmsAttachCount = 0; + } + if (fmsAttachCount > 100) { // 2 seconds + // match info comes through TCP, so we need to double-check we've + // actually received it + auto matchType = DriverStation::GetMatchType(); + if (matchType != DriverStation::kNone) { + // rename per match info + char matchTypeChar; + switch (matchType) { + case DriverStation::kPractice: + matchTypeChar = 'P'; + break; + case DriverStation::kQualification: + matchTypeChar = 'Q'; + break; + case DriverStation::kElimination: + matchTypeChar = 'E'; + break; + default: + matchTypeChar = '_'; + break; + } + std::time_t now = std::time(nullptr); + m_log.SetFilename( + fmt::format("FRC_{:%Y%m%d_%H%M%S}_{}_{}{}.wpilog", + *std::gmtime(&now), DriverStation::GetEventName(), + matchTypeChar, DriverStation::GetMatchNumber())); + fmsRenamed = true; + dsRenamed = true; // don't override FMS rename + } + } + } + + // Write system time every ~5 seconds + ++sysTimeCount; + if (sysTimeCount >= 250) { + sysTimeCount = 0; + sysTimeEntry.Append(wpi::GetSystemTime(), wpi::Now()); + } + } +} + +void Thread::StartNTLog() { + if (!m_ntLoggerEnabled) { + m_ntLoggerEnabled = true; + auto inst = nt::NetworkTableInstance::GetDefault(); + m_ntEntryLogger = inst.StartEntryDataLog(m_log, "", "NT:"); + m_ntConnLogger = inst.StartConnectionDataLog(m_log, "NTConnection"); + } +} + +void Thread::StopNTLog() { + if (m_ntLoggerEnabled) { + m_ntLoggerEnabled = false; + nt::NetworkTableInstance::StopEntryDataLog(m_ntEntryLogger); + nt::NetworkTableInstance::StopConnectionDataLog(m_ntConnLogger); + } +} + +Instance::Instance(std::string_view dir, std::string_view filename, + double period) { + // Delete all previously existing FRC_TBD_*.wpilog files. These only exist + // when the robot never connects to the DS, so they are very unlikely to + // have useful data and just clutter the filesystem. + auto logDir = MakeLogDir(dir); + std::error_code ec; + for (auto&& entry : fs::directory_iterator{logDir, ec}) { + if (wpi::starts_with(entry.path().stem().string(), "FRC_TBD_") && + entry.path().extension() == ".wpilog") { + if (!fs::remove(entry, ec)) { + fmt::print(stderr, "DataLogManager: could not delete {}\n", + entry.path().string()); + } + } + } + + owner.Start(logDir, filename, period); +} + +static Instance& GetInstance(std::string_view dir = "", + std::string_view filename = "", + double period = 0.25) { + static Instance instance(dir, filename, period); + return instance; +} + +void DataLogManager::Start(std::string_view dir, std::string_view filename, + double period) { + GetInstance(dir, filename, period); +} + +void DataLogManager::Log(std::string_view message) { + GetInstance().owner.GetThread()->m_messageLog.Append(message); + fmt::print("{}\n", message); +} + +wpi::log::DataLog& DataLogManager::GetLog() { + return GetInstance().owner.GetThread()->m_log; +} + +std::string DataLogManager::GetLogDir() { + return GetInstance().owner.GetThread()->m_logDir; +} + +void DataLogManager::LogNetworkTables(bool enabled) { + if (auto thr = GetInstance().owner.GetThread()) { + if (enabled) { + thr->StartNTLog(); + } else if (!enabled) { + thr->StopNTLog(); + } + } +} diff --git a/wpilibc/src/main/native/include/frc/DataLogManager.h b/wpilibc/src/main/native/include/frc/DataLogManager.h new file mode 100644 index 0000000000..fa7abbaec5 --- /dev/null +++ b/wpilibc/src/main/native/include/frc/DataLogManager.h @@ -0,0 +1,85 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include +#include + +namespace wpi::log { +class DataLog; +} // namespace wpi::log + +namespace frc { + +/** + * Centralized data log that provides automatic data log file management. It + * automatically cleans up old files when disk space is low and renames the file + * based either on current date/time or (if available) competition match number. + * The deta file will be saved to a USB flash drive if one is attached, or to + * /home/lvuser otherwise. + * + * Log files are initially named "FRC_TBD_{random}.wpilog" until the DS + * connects. After the DS connects, the log file is renamed to + * "FRC_yyyyMMdd_HHmmss.wpilog" (where the date/time is UTC). If the FMS is + * connected and provides a match number, the log file is renamed to + * "FRC_yyyyMMdd_HHmmss_{event}_{match}.wpilog". + * + * On startup, all existing FRC_TBD log files are deleted. If there is less than + * 50 MB of free space on the target storage, FRC_ log files are deleted (oldest + * to newest) until there is 50 MB free OR there are 10 files remaining. + * + * By default, all NetworkTables value changes are stored to the data log. + */ +class DataLogManager final { + public: + DataLogManager() = delete; + + /** + * Start data log manager. The parameters have no effect if the data log + * manager was already started (e.g. by calling another static function). + * + * @param dir if not empty, directory to use for data log storage + * @param filename filename to use; if none provided, the filename is + * automatically generated + * @param period time between automatic flushes to disk, in seconds; + * this is a time/storage tradeoff + */ + static void Start(std::string_view dir = "", std::string_view filename = "", + double period = 0.25); + + /** + * Log a message to the "messages" entry. The message is also printed to + * standard output (followed by a newline). + * + * @param message message + */ + static void Log(std::string_view message); + + /** + * Get the managed data log (for custom logging). Starts the data log manager + * if not already started. + * + * @return data log + */ + static wpi::log::DataLog& GetLog(); + + /** + * Get the log directory. + * + * @return log directory + */ + static std::string GetLogDir(); + + /** + * Enable or disable logging of NetworkTables data. Note that unlike the + * network interface for NetworkTables, this will capture every value change. + * Defaults to enabled. + * + * @param enabled true to enable, false to disable + */ + static void LogNetworkTables(bool enabled); +}; + +} // namespace frc diff --git a/wpilibj/src/main/java/edu/wpi/first/wpilibj/DataLogManager.java b/wpilibj/src/main/java/edu/wpi/first/wpilibj/DataLogManager.java new file mode 100644 index 0000000000..d1d3db03e6 --- /dev/null +++ b/wpilibj/src/main/java/edu/wpi/first/wpilibj/DataLogManager.java @@ -0,0 +1,376 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +package edu.wpi.first.wpilibj; + +import edu.wpi.first.networktables.NetworkTableInstance; +import edu.wpi.first.util.WPIUtilJNI; +import edu.wpi.first.util.datalog.DataLog; +import edu.wpi.first.util.datalog.IntegerLogEntry; +import edu.wpi.first.util.datalog.StringLogEntry; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.Arrays; +import java.util.Comparator; +import java.util.Random; + +/** + * Centralized data log that provides automatic data log file management. It automatically cleans up + * old files when disk space is low and renames the file based either on current date/time or (if + * available) competition match number. The deta file will be saved to a USB flash drive if one is + * attached, or to /home/lvuser otherwise. + * + *

Log files are initially named "FRC_TBD_{random}.wpilog" until the DS connects. After the DS + * connects, the log file is renamed to "FRC_yyyyMMdd_HHmmss.wpilog" (where the date/time is UTC). + * If the FMS is connected and provides a match number, the log file is renamed to + * "FRC_yyyyMMdd_HHmmss_{event}_{match}.wpilog". + * + *

On startup, all existing FRC_TBD log files are deleted. If there is less than 50 MB of free + * space on the target storage, FRC_ log files are deleted (oldest to newest) until there is 50 MB + * free OR there are 10 files remaining. + * + *

By default, all NetworkTables value changes are stored to the data log. + */ +public final class DataLogManager { + private static DataLog m_log; + private static String m_logDir; + private static boolean m_filenameOverride; + private static final Thread m_thread; + private static final ZoneId m_utc = ZoneId.of("UTC"); + private static final DateTimeFormatter m_timeFormatter = + DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss").withZone(m_utc); + private static boolean m_ntLoggerEnabled = true; + private static int m_ntEntryLogger; + private static int m_ntConnLogger; + private static StringLogEntry m_messageLog; + + // if less than this much free space, delete log files until there is this much free space + // OR there are this many files remaining. + private static final long kFreeSpaceThreshold = 50000000L; + private static final int kFileCountThreshold = 10; + + private DataLogManager() {} + + static { + m_thread = new Thread(DataLogManager::logMain, "DataLogDS"); + m_thread.setDaemon(true); + } + + /** Start data log manager with default directory location. */ + public static synchronized void start() { + start("", "", 0.25); + } + + /** + * Start data log manager. The parameters have no effect if the data log manager was already + * started (e.g. by calling another static function). + * + * @param dir if not empty, directory to use for data log storage + */ + public static synchronized void start(String dir) { + start(dir, "", 0.25); + } + + /** + * Start data log manager. The parameters have no effect if the data log manager was already + * started (e.g. by calling another static function). + * + * @param dir if not empty, directory to use for data log storage + * @param filename filename to use; if none provided, the filename is automatically generated + */ + public static synchronized void start(String dir, String filename) { + start(dir, filename, 0.25); + } + + /** + * Start data log manager. The parameters have no effect if the data log manager was already + * started (e.g. by calling another static function). + * + * @param dir if not empty, directory to use for data log storage + * @param filename filename to use; if none provided, the filename is automatically generated + * @param period time between automatic flushes to disk, in seconds; this is a time/storage + * tradeoff + */ + public static synchronized void start(String dir, String filename, double period) { + if (m_log != null) { + return; + } + m_logDir = makeLogDir(dir); + m_filenameOverride = !filename.isEmpty(); + + // Delete all previously existing FRC_TBD_*.wpilog files. These only exist when the robot + // never connects to the DS, so they are very unlikely to have useful data and just clutter + // the filesystem. + File[] files = + new File(m_logDir) + .listFiles((d, name) -> name.startsWith("FRC_TBD_") && name.endsWith(".wpilog")); + if (files != null) { + for (File file : files) { + if (!file.delete()) { + System.err.println("DataLogManager: could not delete " + file); + } + } + } + + m_log = new DataLog(m_logDir, makeLogFilename(filename), period); + m_messageLog = new StringLogEntry(m_log, "messages"); + m_thread.start(); + + // Log all NT entries and connections + if (m_ntLoggerEnabled) { + startNtLog(); + } + } + + /** + * Log a message to the "messages" entry. The message is also printed to standard output (followed + * by a newline). + * + * @param message message + */ + public static synchronized void log(String message) { + if (m_messageLog == null) { + start(); + } + m_messageLog.append(message); + System.out.println(message); + } + + /** + * Get the managed data log (for custom logging). Starts the data log manager if not already + * started. + * + * @return data log + */ + public static synchronized DataLog getLog() { + if (m_log == null) { + start(); + } + return m_log; + } + + /** + * Get the log directory. + * + * @return log directory, or empty string if logging not yet started + */ + public static synchronized String getLogDir() { + if (m_logDir == null) { + return ""; + } + return m_logDir; + } + + /** + * Enable or disable logging of NetworkTables data. Note that unlike the network interface for + * NetworkTables, this will capture every value change. Defaults to enabled. + * + * @param enabled true to enable, false to disable + */ + public static synchronized void logNetworkTables(boolean enabled) { + boolean wasEnabled = m_ntLoggerEnabled; + m_ntLoggerEnabled = enabled; + if (m_log == null) { + start(); + return; + } + if (enabled && !wasEnabled) { + startNtLog(); + } else if (!enabled && wasEnabled) { + stopNtLog(); + } + } + + @SuppressWarnings("PMD.EmptyCatchBlock") + private static String makeLogDir(String dir) { + if (!dir.isEmpty()) { + return dir; + } + + if (RobotBase.isReal()) { + try { + // prefer a mounted USB drive if one is accessible + Path usbDir = Paths.get("/u").toRealPath(); + if (Files.isWritable(usbDir)) { + return usbDir.toString(); + } + } catch (IOException ex) { + // ignored + } + } + + return Filesystem.getOperatingDirectory().getAbsolutePath(); + } + + private static String makeLogFilename(String filenameOverride) { + if (!filenameOverride.isEmpty()) { + return filenameOverride; + } + Random rnd = new Random(); + StringBuilder filename = new StringBuilder(); + filename.append("FRC_TBD_"); + for (int i = 0; i < 4; i++) { + filename.append(String.format("%04x", rnd.nextInt(0x10000))); + } + filename.append(".wpilog"); + return filename.toString(); + } + + private static void startNtLog() { + NetworkTableInstance inst = NetworkTableInstance.getDefault(); + m_ntEntryLogger = inst.startEntryDataLog(m_log, "", "NT:"); + m_ntConnLogger = inst.startConnectionDataLog(m_log, "NTConnection"); + } + + private static void stopNtLog() { + NetworkTableInstance.stopEntryDataLog(m_ntEntryLogger); + NetworkTableInstance.stopConnectionDataLog(m_ntConnLogger); + } + + private static void logMain() { + // based on free disk space, scan for "old" FRC_*.wpilog files and remove + { + File logDir = new File(m_logDir); + long freeSpace = logDir.getFreeSpace(); + if (freeSpace < kFreeSpaceThreshold) { + // Delete oldest FRC_*.wpilog files (ignore FRC_TBD_*.wpilog as we just created one) + File[] files = + logDir.listFiles( + (dir, name) -> + name.startsWith("FRC_") + && name.endsWith(".wpilog") + && !name.startsWith("FRC_TBD_")); + if (files != null) { + Arrays.sort(files, Comparator.comparingLong(File::lastModified)); + int count = files.length; + for (File file : files) { + --count; + if (count < kFileCountThreshold) { + break; + } + long length = file.length(); + if (file.delete()) { + freeSpace += length; + if (freeSpace >= kFreeSpaceThreshold) { + break; + } + } else { + System.err.println("DataLogManager: could not delete " + file); + } + } + } + } + } + + int timeoutCount = 0; + boolean paused = false; + int dsAttachCount = 0; + int fmsAttachCount = 0; + boolean dsRenamed = m_filenameOverride; + boolean fmsRenamed = m_filenameOverride; + int sysTimeCount = 0; + IntegerLogEntry sysTimeEntry = + new IntegerLogEntry( + m_log, "systemTime", "{\"source\":\"DataLogManager\",\"format\":\"time_t_us\"}"); + + while (!Thread.interrupted()) { + boolean newData = DriverStation.waitForData(0.25); + if (Thread.interrupted()) { + break; + } + if (!newData) { + timeoutCount++; + // pause logging after being disconnected for 10 seconds + if (timeoutCount > 40 && !paused) { + timeoutCount = 0; + paused = true; + m_log.pause(); + } + continue; + } + // when we connect to the DS, resume logging + timeoutCount = 0; + if (paused) { + paused = false; + m_log.resume(); + } + + if (!dsRenamed) { + // track DS attach + if (DriverStation.isDSAttached()) { + dsAttachCount++; + } else { + dsAttachCount = 0; + } + if (dsAttachCount > 50) { // 1 second + LocalDateTime now = LocalDateTime.now(m_utc); + if (now.getYear() > 2000) { + // assume local clock is now synchronized to DS, so rename based on + // local time + m_log.setFilename("FRC_" + m_timeFormatter.format(now) + ".wpilog"); + dsRenamed = true; + } else { + dsAttachCount = 0; // wait a bit and try again + } + } + } + + if (!fmsRenamed) { + // track FMS attach + if (DriverStation.isFMSAttached()) { + fmsAttachCount++; + } else { + fmsAttachCount = 0; + } + if (fmsAttachCount > 100) { // 2 seconds + // match info comes through TCP, so we need to double-check we've + // actually received it + DriverStation.MatchType matchType = DriverStation.getMatchType(); + if (matchType != DriverStation.MatchType.None) { + // rename per match info + char matchTypeChar; + switch (matchType) { + case Practice: + matchTypeChar = 'P'; + break; + case Qualification: + matchTypeChar = 'Q'; + break; + case Elimination: + matchTypeChar = 'E'; + break; + default: + matchTypeChar = '_'; + break; + } + m_log.setFilename( + "FRC_" + + m_timeFormatter.format(LocalDateTime.now(m_utc)) + + "_" + + DriverStation.getEventName() + + "_" + + matchTypeChar + + DriverStation.getMatchNumber() + + ".wpilog"); + fmsRenamed = true; + dsRenamed = true; // don't override FMS rename + } + } + } + + // Write system time every ~5 seconds + sysTimeCount++; + if (sysTimeCount >= 250) { + sysTimeCount = 0; + sysTimeEntry.append(WPIUtilJNI.getSystemTime(), WPIUtilJNI.now()); + } + } + } +}