[wpiutil, wpilib] Add FileLogger and log console output (#6977)

ntcoreffi/src/main/native/cpp/DataLogManager.cpp

@@ -14,6 +14,7 @@
 #include <fmt/format.h>
 #include <networktables/NetworkTableInstance.h>
 #include <wpi/DataLogBackgroundWriter.h>
+#include <wpi/FileLogger.h>
 #include <wpi/SafeThread.h>
 #include <wpi/StringExtras.h>
 #include <wpi/fs.h>
@@ -191,6 +192,8 @@ struct Thread final : public wpi::SafeThread {
 
   void StartNTLog();
   void StopNTLog();
+  void StartConsoleLog();
+  void StopConsoleLog();
 
   std::string m_logDir;
   bool m_filenameOverride;
@@ -198,6 +201,8 @@ struct Thread final : public wpi::SafeThread {
   bool m_ntLoggerEnabled = false;
   NT_DataLogger m_ntEntryLogger = 0;
   NT_ConnectionDataLogger m_ntConnLogger = 0;
+  bool m_consoleLoggerEnabled = false;
+  wpi::FileLogger m_consoleLogger;
   wpi::log::StringLogEntry m_messageLog;
 };
 
@@ -452,6 +457,20 @@ void Thread::StopNTLog() {
   }
 }
 
+void Thread::StartConsoleLog() {
+  if (!m_consoleLoggerEnabled) {
+    m_consoleLoggerEnabled = true;
+    m_consoleLogger = {"/home/lvuser/FRC_UserProgram.log", m_log, "output"};
+  }
+}
+
+void Thread::StopConsoleLog() {
+  if (m_consoleLoggerEnabled) {
+    m_consoleLoggerEnabled = false;
+    m_consoleLogger = {};
+  }
+}
+
 Instance::Instance(std::string_view dir, std::string_view filename,
                    double period) {
   // Delete all previously existing FRC_TBD_*.wpilog files. These only exist
@@ -516,6 +535,16 @@ void DataLogManager::LogNetworkTables(bool enabled) {
   }
 }
 
+void DataLogManager::LogConsoleOutput(bool enabled) {
+  if (auto thr = GetInstance().owner.GetThread()) {
+    if (enabled) {
+      thr->StartConsoleLog();
+    } else if (!enabled) {
+      thr->StopConsoleLog();
+    }
+  }
+}
+
 void DataLogManager::SignalNewDSDataOccur() {
   wpi::SetSignalObject(DriverStation::gNewDataEvent);
 }
@@ -546,6 +575,10 @@ void DLM_LogNetworkTables(int enabled) {
   DataLogManager::LogNetworkTables(enabled);
 }
 
+void DLM_LogConsoleOutput(int enabled) {
+  DataLogManager::LogConsoleOutput(enabled);
+}
+
 void DLM_SignalNewDSDataOccur(void) {
   DataLogManager::SignalNewDSDataOccur();
 }

ntcoreffi/src/main/native/include/DataLogManager.h

@@ -89,6 +89,11 @@ class DataLogManager final {
    */
   static void LogNetworkTables(bool enabled);
 
+  /**
+   * Enable or disable logging of the console output. Defaults to enabled.
+   * @param enabled  true to enable, false to disable
+   */
+  static void LogConsoleOutput(bool enabled);
   /**
    * Signal new DS data is available.
    */
@@ -152,6 +157,13 @@ const char* DLM_GetLogDir(void);
  */
 void DLM_LogNetworkTables(int enabled);
 
+
+  /**
+   * Enable or disable logging of the console output. Defaults to enabled.
+   * @param enabled  true to enable, false to disable
+   */
+void DLM_LogConsoleOutput(int enabled);
+
 /**
  * Signal new DS data is available.
  */

ntcoreffi/src/main/native/symbols.txt

@@ -1,6 +1,7 @@
 DLM_GetLog
 DLM_GetLogDir
 DLM_Log
+DLM_LogConsoleOutput
 DLM_LogNetworkTables
 DLM_SignalNewDSDataOccur
 DLM_Start

wpilibc/src/main/native/cpp/DataLogManager.cpp

@@ -14,6 +14,7 @@
 #include <networktables/NetworkTableInstance.h>
 #include <wpi/DataLog.h>
 #include <wpi/DataLogBackgroundWriter.h>
+#include <wpi/FileLogger.h>
 #include <wpi/SafeThread.h>
 #include <wpi/StringExtras.h>
 #include <wpi/fs.h>
@@ -37,6 +38,8 @@ struct Thread final : public wpi::SafeThread {
 
   void StartNTLog();
   void StopNTLog();
+  void StartConsoleLog();
+  void StopConsoleLog();
 
   std::string m_logDir;
   bool m_filenameOverride;
@@ -44,6 +47,8 @@ struct Thread final : public wpi::SafeThread {
   bool m_ntLoggerEnabled = false;
   NT_DataLogger m_ntEntryLogger = 0;
   NT_ConnectionDataLogger m_ntConnLogger = 0;
+  bool m_consoleLoggerEnabled = false;
+  wpi::FileLogger m_consoleLogger;
   wpi::log::StringLogEntry m_messageLog;
 };
 
@@ -109,10 +114,12 @@ Thread::Thread(std::string_view dir, std::string_view filename, double period)
       m_log{dir, MakeLogFilename(filename), period},
       m_messageLog{m_log, "messages"} {
   StartNTLog();
+  StartConsoleLog();
 }
 
 Thread::~Thread() {
   StopNTLog();
+  StopConsoleLog();
 }
 
 void Thread::Main() {
@@ -297,6 +304,20 @@ void Thread::StopNTLog() {
   }
 }
 
+void Thread::StartConsoleLog() {
+  if (!m_consoleLoggerEnabled && RobotBase::IsReal()) {
+    m_consoleLoggerEnabled = true;
+    m_consoleLogger = {"/home/lvuser/FRC_UserProgram.log", m_log, "output"};
+  }
+}
+
+void Thread::StopConsoleLog() {
+  if (m_consoleLoggerEnabled && RobotBase::IsReal()) {
+    m_consoleLoggerEnabled = false;
+    m_consoleLogger = {};
+  }
+}
+
 Instance::Instance(std::string_view dir, std::string_view filename,
                    double period) {
   // Delete all previously existing FRC_TBD_*.wpilog files. These only exist
@@ -360,3 +381,13 @@ void DataLogManager::LogNetworkTables(bool enabled) {
     }
   }
 }
+
+void DataLogManager::LogConsoleOutput(bool enabled) {
+  if (auto thr = GetInstance().owner.GetThread()) {
+    if (enabled) {
+      thr->StartConsoleLog();
+    } else if (!enabled) {
+      thr->StopConsoleLog();
+    }
+  }
+}

wpilibc/src/main/native/include/frc/DataLogManager.h

@@ -85,6 +85,12 @@ class DataLogManager final {
    * @param enabled true to enable, false to disable
    */
   static void LogNetworkTables(bool enabled);
+
+  /**
+   * Enable or disable logging of the console output. Defaults to enabled.
+   * @param enabled  true to enable, false to disable
+   */
+  static void LogConsoleOutput(bool enabled);
 };
 
 }  // namespace frc

wpilibj/src/main/java/edu/wpi/first/wpilibj/DataLogManager.java

@@ -5,6 +5,7 @@
 package edu.wpi.first.wpilibj;
 
 import edu.wpi.first.networktables.NetworkTableInstance;
+import edu.wpi.first.util.FileLogger;
 import edu.wpi.first.util.WPIUtilJNI;
 import edu.wpi.first.util.concurrent.Event;
 import edu.wpi.first.util.datalog.DataLog;
@@ -52,6 +53,8 @@ public final class DataLogManager {
   private static boolean m_ntLoggerEnabled = true;
   private static int m_ntEntryLogger;
   private static int m_ntConnLogger;
+  private static boolean m_consoleLoggerEnabled = true;
+  private static FileLogger m_consoleLogger;
   private static StringLogEntry m_messageLog;
 
   // if less than this much free space, delete log files until there is this much free space
@@ -121,6 +124,10 @@ public final class DataLogManager {
       if (m_ntLoggerEnabled) {
         startNtLog();
       }
+      // Log console output
+      if (m_consoleLoggerEnabled) {
+        startConsoleLog();
+      }
     } else if (m_stopped) {
       m_log.setFilename(makeLogFilename(filename));
       m_log.resume();
@@ -205,6 +212,25 @@ public final class DataLogManager {
     }
   }
 
+  /**
+   * Enable or disable logging of the console output. Defaults to enabled.
+   *
+   * @param enabled true to enable, false to disable
+   */
+  public static synchronized void logConsoleOutput(boolean enabled) {
+    boolean wasEnabled = m_consoleLoggerEnabled;
+    m_consoleLoggerEnabled = enabled;
+    if (m_log == null) {
+      start();
+      return;
+    }
+    if (enabled && !wasEnabled) {
+      startConsoleLog();
+    } else if (!enabled && wasEnabled) {
+      stopConsoleLog();
+    }
+  }
+
   private static String makeLogDir(String dir) {
     if (!dir.isEmpty()) {
       return dir;
@@ -266,6 +292,18 @@ public final class DataLogManager {
     NetworkTableInstance.stopConnectionDataLog(m_ntConnLogger);
   }
 
+  private static void startConsoleLog() {
+    if (RobotBase.isReal()) {
+      m_consoleLogger = new FileLogger("/home/lvuser/FRC_UserProgram.log", m_log, "console");
+    }
+  }
+
+  private static void stopConsoleLog() {
+    if (RobotBase.isReal()) {
+      m_consoleLogger.close();
+    }
+  }
+
   private static void logMain() {
     // based on free disk space, scan for "old" FRC_*.wpilog files and remove
     {

wpiutil/src/main/java/edu/wpi/first/util/FileLogger.java

@@ -0,0 +1,32 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+package edu.wpi.first.util;
+
+import edu.wpi.first.util.datalog.DataLog;
+
+/**
+ * A class version of `tail -f`, otherwise known as `tail -f` at home. Watches a file and puts the
+ * data into a data log. Only works on Linux-based platforms.
+ */
+public class FileLogger implements AutoCloseable {
+  private final long m_impl;
+
+  /**
+   * Construct a FileLogger. When the specified file is modified, appended data will be appended to
+   * the specified data log.
+   *
+   * @param file The path to the file.
+   * @param log A data log.
+   * @param key The log key to append data to.
+   */
+  public FileLogger(String file, DataLog log, String key) {
+    m_impl = WPIUtilJNI.createFileLogger(file, log.getImpl(), key);
+  }
+
+  @Override
+  public void close() {
+    WPIUtilJNI.freeFileLogger(m_impl);
+  }
+}

wpiutil/src/main/java/edu/wpi/first/util/WPIUtilJNI.java

@@ -217,6 +217,24 @@ public class WPIUtilJNI {
   public static native int[] waitForObjectsTimeout(int[] handles, double timeout)
       throws InterruptedException;
 
+  /**
+   * Create a native FileLogger. When the specified file is modified, appended data will be appended
+   * to the specified data log.
+   *
+   * @param file path to the file
+   * @param log data log implementation handle
+   * @param key log key to append data to
+   * @return The FileLogger handle.
+   */
+  public static native long createFileLogger(String file, long log, String key);
+
+  /**
+   * Free a native FileLogger. This causes the FileLogger to stop appending data to the log.
+   *
+   * @param fileTail The FileLogger handle.
+   */
+  public static native void freeFileLogger(long fileTail);
+
   /** Utility class. */
   protected WPIUtilJNI() {}
 }

wpiutil/src/main/native/cpp/FileLogger.cpp

@@ -0,0 +1,105 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#include "wpi/FileLogger.h"
+
+#ifdef __linux__
+#include <fcntl.h>
+#include <sys/inotify.h>
+#include <unistd.h>
+#endif
+
+#include <string>
+#include <string_view>
+#include <tuple>
+#include <utility>
+
+#include "wpi/StringExtras.h"
+
+namespace wpi {
+FileLogger::FileLogger(std::string_view file,
+                       std::function<void(std::string_view)> callback)
+#ifdef __linux__
+    : m_fileHandle{open(file.data(), O_RDONLY)},
+      m_inotifyHandle{inotify_init()},
+      m_inotifyWatchHandle{
+          inotify_add_watch(m_inotifyHandle, file.data(), IN_MODIFY)},
+      m_thread{[=, this] {
+        char buf[4000];
+        struct inotify_event ev;
+        int len = 0;
+        lseek(m_fileHandle, 0, SEEK_END);
+        while ((len = read(m_inotifyHandle, &ev, sizeof(ev))) > 0) {
+          int bufLen = 0;
+          if ((bufLen = read(m_fileHandle, buf, sizeof(buf)) > 0)) {
+            callback(std::string_view{buf, static_cast<size_t>(bufLen)});
+          }
+        }
+      }}
+#endif
+{
+}
+FileLogger::FileLogger(std::string_view file, log::DataLog& log,
+                       std::string_view key)
+    : FileLogger(file, LineBuffer([entry = log.Start(key, "string"),
+                                   &log](std::string_view line) {
+                   log.AppendString(entry, line, 0);
+                 })) {}
+FileLogger::FileLogger(FileLogger&& other)
+#ifdef __linux__
+    : m_fileHandle{std::exchange(other.m_fileHandle, -1)},
+      m_inotifyHandle{std::exchange(other.m_inotifyHandle, -1)},
+      m_inotifyWatchHandle{std::exchange(other.m_inotifyWatchHandle, -1)},
+      m_thread{std::move(other.m_thread)}
+#endif
+{
+}
+FileLogger& FileLogger::operator=(FileLogger&& rhs) {
+#ifdef __linux__
+  std::swap(m_fileHandle, rhs.m_fileHandle);
+  std::swap(m_inotifyHandle, rhs.m_inotifyHandle);
+  std::swap(m_inotifyWatchHandle, rhs.m_inotifyWatchHandle);
+  m_thread = std::move(rhs.m_thread);
+#endif
+  return *this;
+}
+FileLogger::~FileLogger() {
+#ifdef __linux__
+  if (m_inotifyWatchHandle != -1) {
+    inotify_rm_watch(m_inotifyHandle, m_inotifyWatchHandle);
+  }
+  if (m_inotifyHandle != -1) {
+    close(m_inotifyHandle);
+  }
+  if (m_fileHandle != -1) {
+    close(m_fileHandle);
+  }
+  if (m_thread.joinable()) {
+    m_thread.join();
+  }
+#endif
+}
+std::function<void(std::string_view)> FileLogger::LineBuffer(
+    std::function<void(std::string_view)> callback) {
+  return [callback,
+          buf = wpi::SmallVector<char, 32>{}](std::string_view data) mutable {
+    if (!wpi::contains(data, "\n")) {
+      buf.append(data.begin(), data.end());
+      return;
+    }
+    std::string_view line;
+    std::string_view remainingData;
+    std::tie(line, remainingData) = wpi::split(data, "\n");
+    buf.append(line.begin(), line.end());
+    callback(std::string_view{buf.data(), buf.size()});
+
+    while (wpi::contains(remainingData, "\n")) {
+      std::tie(line, remainingData) = wpi::split(remainingData, "\n");
+      callback(line);
+    }
+    buf.clear();
+    buf.append(remainingData.begin(), remainingData.end());
+  };
+}
+}  // namespace wpi

wpiutil/src/main/native/cpp/jni/WPIUtilJNI.cpp

@@ -7,6 +7,8 @@
 #include <jni.h>
 
 #include "edu_wpi_first_util_WPIUtilJNI.h"
+#include "wpi/DataLog.h"
+#include "wpi/FileLogger.h"
 #include "wpi/RawFrame.h"
 #include "wpi/Synchronization.h"
 #include "wpi/jni_util.h"
@@ -414,4 +416,41 @@ Java_edu_wpi_first_util_WPIUtilJNI_setRawFrameInfo
   f->pixelFormat = pixelFormat;
 }
 
+/*
+ * Class:     edu_wpi_first_util_WPIUtilJNI
+ * Method:    createFileLogger
+ * Signature: (Ljava/lang/String;JLjava/lang/String;)J
+ */
+JNIEXPORT jlong JNICALL
+Java_edu_wpi_first_util_WPIUtilJNI_createFileLogger
+  (JNIEnv* env, jclass, jstring file, jlong log, jstring key)
+{
+  if (!file) {
+    wpi::ThrowNullPointerException(env, "file is null");
+    return 0;
+  }
+  auto* f = reinterpret_cast<wpi::log::DataLog*>(log);
+  if (!f) {
+    wpi::ThrowNullPointerException(env, "log is null");
+    return 0;
+  }
+  if (!key) {
+    wpi::ThrowNullPointerException(env, "key is null");
+    return 0;
+  }
+  return reinterpret_cast<jlong>(
+      new wpi::FileLogger{JStringRef{env, file}, *f, JStringRef{env, key}});
+}
+
+/*
+ * Class:     edu_wpi_first_util_WPIUtilJNI
+ * Method:    freeFileLogger
+ * Signature: (J)V
+ */
+JNIEXPORT void JNICALL
+Java_edu_wpi_first_util_WPIUtilJNI_freeFileLogger
+  (JNIEnv* env, jclass, jlong fileTail)
+{
+  delete reinterpret_cast<wpi::FileLogger*>(fileTail);
+}
 }  // extern "C"

wpiutil/src/main/native/include/wpi/FileLogger.h

@@ -0,0 +1,61 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#pragma once
+
+#include <functional>
+#include <string_view>
+#include <thread>
+
+#include "wpi/DataLog.h"
+
+namespace wpi {
+/**
+ * A class version of `tail -f`, otherwise known as `tail -f` at home.  Watches
+ * a file and puts the data somewhere else. Only works on Linux-based platforms.
+ */
+class FileLogger {
+ public:
+  FileLogger() = default;
+  /**
+   * Construct a FileLogger. When the specified file is modified, the callback
+   * will be called with the appended changes.
+   *
+   * @param file The path to the file.
+   * @param callback A callback that accepts the appended file data.
+   */
+  FileLogger(std::string_view file,
+             std::function<void(std::string_view)> callback);
+
+  /**
+   * Construct a FileLogger. When the specified file is modified, appended data
+   * will be appended to the specified data log.
+   *
+   * @param file The path to the file.
+   * @param log A data log.
+   * @param key The log key to append data to.
+   */
+  FileLogger(std::string_view file, log::DataLog& log, std::string_view key);
+  FileLogger(FileLogger&& other);
+  FileLogger& operator=(FileLogger&& rhs);
+  ~FileLogger();
+  /**
+   * Creates a function that chunks incoming data into lines before calling the
+   * callback with the individual line.
+   *
+   * @param callback The callback that logs lines.
+   * @return The function.
+   */
+  static std::function<void(std::string_view)> LineBuffer(
+      std::function<void(std::string_view)> callback);
+
+ private:
+#ifdef __linux__
+  int m_fileHandle = -1;
+  int m_inotifyHandle = -1;
+  int m_inotifyWatchHandle = -1;
+  std::thread m_thread;
+#endif
+};
+}  // namespace wpi

wpiutil/src/test/native/cpp/FileLoggerTest.cpp

@@ -0,0 +1,52 @@
+// Copyright (c) FIRST and other WPILib contributors.
+// Open Source Software; you can modify and/or share it under the terms of
+// the WPILib BSD license file in the root directory of this project.
+
+#include <string_view>
+#include <vector>
+
+#include <gtest/gtest.h>
+
+#include "wpi/FileLogger.h"
+
+TEST(FileLoggerTest, LineBufferSingleLine) {
+  std::vector<std::string> buf;
+  auto func = wpi::FileLogger::LineBuffer(
+      [&buf](std::string_view line) { buf.emplace_back(line); });
+  func("qwertyuiop\n");
+  EXPECT_EQ(buf.front(), "qwertyuiop");
+  buf.clear();
+}
+
+TEST(FileLoggerTest, LineBufferMultiLine) {
+  std::vector<std::string> buf;
+  auto func = wpi::FileLogger::LineBuffer(
+      [&buf](std::string_view line) { buf.emplace_back(line); });
+  func("line 1\nline 2\nline 3\n");
+  EXPECT_EQ("line 1", buf[0]);
+  EXPECT_EQ("line 2", buf[1]);
+  EXPECT_EQ("line 3", buf[2]);
+}
+
+TEST(FileLoggerTest, LineBufferPartials) {
+  std::vector<std::string> buf;
+  auto func = wpi::FileLogger::LineBuffer(
+      [&buf](std::string_view line) { buf.emplace_back(line); });
+  func("part 1");
+  func("part 2\npart 3");
+  EXPECT_EQ("part 1part 2", buf.front());
+  buf.clear();
+  func("\n");
+  EXPECT_EQ("part 3", buf.front());
+}
+
+TEST(FileLoggerTest, LineBufferMultiplePartials) {
+  std::vector<std::string> buf;
+  auto func = wpi::FileLogger::LineBuffer(
+      [&buf](std::string_view line) { buf.emplace_back(line); });
+  func("part 1");
+  func("part 2");
+  func("part 3");
+  func("part 4\n");
+  EXPECT_EQ("part 1part 2part 3part 4", buf.front());
+}