mirror of
https://github.com/wpilibsuite/allwpilib
synced 2026-06-19 00:41:43 +00:00
[datalogtool] Add datalogtool
This is a support tool for datalog file conversion (and eventually download/remote datalog file management).
This commit is contained in:
Peter Johnson
393
datalogtool/src/main/native/cpp/Downloader.cpp
Normal file
393
datalogtool/src/main/native/cpp/Downloader.cpp
Normal file
@@ -0,0 +1,393 @@
|
||||
// Copyright (c) FIRST and other WPILib contributors.
|
||||
// Open Source Software; you can modify and/or share it under the terms of
|
||||
// the WPILib BSD license file in the root directory of this project.
|
||||
|
||||
#include "Downloader.h"
|
||||
|
||||
#include <libssh/sftp.h>
|
||||
|
||||
#ifdef _WIN32
|
||||
#include <fcntl.h>
|
||||
#include <io.h>
|
||||
#else
|
||||
#include <sys/fcntl.h>
|
||||
#endif
|
||||
|
||||
#include <algorithm>
|
||||
#include <filesystem>
|
||||
|
||||
#include <fmt/format.h>
|
||||
#include <glass/Storage.h>
|
||||
#include <imgui.h>
|
||||
#include <imgui_stdlib.h>
|
||||
#include <portable-file-dialogs.h>
|
||||
#include <wpi/StringExtras.h>
|
||||
#include <wpi/fs.h>
|
||||
|
||||
#include "Sftp.h"
|
||||
|
||||
Downloader::Downloader(glass::Storage& storage)
|
||||
: m_serverTeam{storage.GetString("serverTeam")},
|
||||
m_remoteDir{storage.GetString("remoteDir", "/home/lvuser")},
|
||||
m_username{storage.GetString("username", "lvuser")},
|
||||
m_localDir{storage.GetString("localDir")},
|
||||
m_deleteAfter{storage.GetBool("deleteAfter", true)},
|
||||
m_thread{[this] { ThreadMain(); }} {}
|
||||
|
||||
Downloader::~Downloader() {
|
||||
{
|
||||
std::scoped_lock lock{m_mutex};
|
||||
m_state = kExit;
|
||||
}
|
||||
m_cv.notify_all();
|
||||
m_thread.join();
|
||||
}
|
||||
|
||||
void Downloader::DisplayConnect() {
|
||||
// IP or Team Number text box
|
||||
ImGui::SetNextItemWidth(ImGui::GetFontSize() * 12);
|
||||
ImGui::InputText("Team Number / Address", &m_serverTeam);
|
||||
|
||||
// Username/password
|
||||
ImGui::SetNextItemWidth(ImGui::GetFontSize() * 12);
|
||||
ImGui::InputText("Username", &m_username);
|
||||
ImGui::SetNextItemWidth(ImGui::GetFontSize() * 12);
|
||||
ImGui::InputText("Password", &m_password, ImGuiInputTextFlags_Password);
|
||||
|
||||
// Connect button
|
||||
if (ImGui::Button("Connect")) {
|
||||
m_state = kConnecting;
|
||||
m_cv.notify_all();
|
||||
}
|
||||
}
|
||||
|
||||
void Downloader::DisplayDisconnectButton() {
|
||||
if (ImGui::Button("Disconnect")) {
|
||||
m_state = kDisconnecting;
|
||||
m_cv.notify_all();
|
||||
}
|
||||
}
|
||||
|
||||
void Downloader::DisplayRemoteDirSelector() {
|
||||
ImGui::SameLine();
|
||||
if (ImGui::Button("Refresh")) {
|
||||
m_state = kGetFiles;
|
||||
m_cv.notify_all();
|
||||
}
|
||||
|
||||
// Remote directory text box
|
||||
ImGui::SetNextItemWidth(ImGui::GetFontSize() * 20);
|
||||
if (ImGui::InputText("Remote Dir", &m_remoteDir,
|
||||
ImGuiInputTextFlags_EnterReturnsTrue)) {
|
||||
m_state = kGetFiles;
|
||||
m_cv.notify_all();
|
||||
}
|
||||
|
||||
// List directories
|
||||
for (auto&& dir : m_dirList) {
|
||||
if (ImGui::Selectable(dir.c_str())) {
|
||||
if (dir == "..") {
|
||||
if (wpi::ends_with(m_remoteDir, '/')) {
|
||||
m_remoteDir.resize(m_remoteDir.size() - 1);
|
||||
}
|
||||
m_remoteDir = wpi::rsplit(m_remoteDir, '/').first;
|
||||
if (m_remoteDir.empty()) {
|
||||
m_remoteDir = "/";
|
||||
}
|
||||
} else {
|
||||
if (!wpi::ends_with(m_remoteDir, '/')) {
|
||||
m_remoteDir += '/';
|
||||
}
|
||||
m_remoteDir += dir;
|
||||
}
|
||||
m_state = kGetFiles;
|
||||
m_cv.notify_all();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Downloader::DisplayLocalDirSelector() {
|
||||
// Local directory text / select button
|
||||
if (ImGui::Button("Select Download Folder...")) {
|
||||
m_localDirSelector =
|
||||
std::make_unique<pfd::select_folder>("Select Download Folder");
|
||||
}
|
||||
ImGui::TextUnformatted(m_localDir.c_str());
|
||||
|
||||
// Delete after download (checkbox)
|
||||
ImGui::Checkbox("Delete after download", &m_deleteAfter);
|
||||
|
||||
// Download button
|
||||
if (!m_localDir.empty()) {
|
||||
if (ImGui::Button("Download")) {
|
||||
m_state = kDownload;
|
||||
m_cv.notify_all();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
size_t Downloader::DisplayFiles() {
|
||||
// List of files (multi-select) (changes to progress bar for downloading)
|
||||
size_t fileCount = 0;
|
||||
if (ImGui::BeginTable(
|
||||
"files", 3,
|
||||
ImGuiTableFlags_Borders | ImGuiTableFlags_SizingStretchProp)) {
|
||||
ImGui::TableSetupColumn("File");
|
||||
ImGui::TableSetupColumn("Size");
|
||||
ImGui::TableSetupColumn("Download");
|
||||
ImGui::TableHeadersRow();
|
||||
for (auto&& download : m_downloadList) {
|
||||
if ((m_state == kDownload || m_state == kDownloadDone) &&
|
||||
!download.enabled) {
|
||||
continue;
|
||||
}
|
||||
|
||||
++fileCount;
|
||||
|
||||
ImGui::TableNextRow();
|
||||
ImGui::TableNextColumn();
|
||||
ImGui::TextUnformatted(download.name.c_str());
|
||||
ImGui::TableNextColumn();
|
||||
auto sizeText = fmt::format("{}", download.size);
|
||||
ImGui::TextUnformatted(sizeText.c_str());
|
||||
ImGui::TableNextColumn();
|
||||
if (m_state == kDownload || m_state == kDownloadDone) {
|
||||
if (!download.status.empty()) {
|
||||
ImGui::TextUnformatted(download.status.c_str());
|
||||
} else {
|
||||
ImGui::ProgressBar(download.complete);
|
||||
}
|
||||
} else {
|
||||
auto checkboxLabel = fmt::format("##{}", download.name);
|
||||
ImGui::Checkbox(checkboxLabel.c_str(), &download.enabled);
|
||||
}
|
||||
}
|
||||
ImGui::EndTable();
|
||||
}
|
||||
|
||||
return fileCount;
|
||||
}
|
||||
|
||||
void Downloader::Display() {
|
||||
if (m_localDirSelector && m_localDirSelector->ready(0)) {
|
||||
m_localDir = m_localDirSelector->result();
|
||||
m_localDirSelector.reset();
|
||||
}
|
||||
|
||||
std::scoped_lock lock{m_mutex};
|
||||
|
||||
if (!m_error.empty()) {
|
||||
ImGui::TextUnformatted(m_error.c_str());
|
||||
}
|
||||
|
||||
switch (m_state) {
|
||||
case kDisconnected:
|
||||
DisplayConnect();
|
||||
break;
|
||||
case kConnecting:
|
||||
DisplayDisconnectButton();
|
||||
ImGui::Text("Connecting to %s...", m_serverTeam.c_str());
|
||||
break;
|
||||
case kDisconnecting:
|
||||
ImGui::TextUnformatted("Disconnecting...");
|
||||
break;
|
||||
case kConnected:
|
||||
case kGetFiles:
|
||||
DisplayDisconnectButton();
|
||||
DisplayRemoteDirSelector();
|
||||
if (DisplayFiles() > 0) {
|
||||
DisplayLocalDirSelector();
|
||||
}
|
||||
break;
|
||||
case kDownload:
|
||||
case kDownloadDone:
|
||||
DisplayDisconnectButton();
|
||||
DisplayFiles();
|
||||
if (m_state == kDownloadDone) {
|
||||
if (ImGui::Button("Download complete!")) {
|
||||
m_state = kGetFiles;
|
||||
m_cv.notify_all();
|
||||
}
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
void Downloader::ThreadMain() {
|
||||
std::unique_ptr<sftp::Session> session;
|
||||
|
||||
static constexpr size_t kBufSize = 32 * 1024;
|
||||
std::unique_ptr<uint8_t[]> copyBuf = std::make_unique<uint8_t[]>(kBufSize);
|
||||
|
||||
std::unique_lock lock{m_mutex};
|
||||
while (m_state != kExit) {
|
||||
State prev = m_state;
|
||||
m_cv.wait(lock, [&] { return m_state != prev; });
|
||||
m_error.clear();
|
||||
try {
|
||||
switch (m_state) {
|
||||
case kConnecting:
|
||||
if (auto team = wpi::parse_integer<unsigned int>(m_serverTeam, 10)) {
|
||||
// team number
|
||||
session = std::make_unique<sftp::Session>(
|
||||
fmt::format("roborio-{}-frc.local", team.value()), 22,
|
||||
m_username, m_password);
|
||||
} else {
|
||||
session = std::make_unique<sftp::Session>(m_serverTeam, 22,
|
||||
m_username, m_password);
|
||||
}
|
||||
lock.unlock();
|
||||
try {
|
||||
session->Connect();
|
||||
} catch (...) {
|
||||
lock.lock();
|
||||
throw;
|
||||
}
|
||||
lock.lock();
|
||||
// FALLTHROUGH
|
||||
case kGetFiles: {
|
||||
std::string dir = m_remoteDir;
|
||||
std::vector<sftp::Attributes> fileList;
|
||||
lock.unlock();
|
||||
try {
|
||||
fileList = session->ReadDir(dir);
|
||||
} catch (sftp::Exception& ex) {
|
||||
lock.lock();
|
||||
if (ex.err == SSH_FX_OK || ex.err == SSH_FX_CONNECTION_LOST) {
|
||||
throw;
|
||||
}
|
||||
m_error = ex.what();
|
||||
m_dirList.clear();
|
||||
m_downloadList.clear();
|
||||
m_state = kConnected;
|
||||
break;
|
||||
}
|
||||
std::sort(
|
||||
fileList.begin(), fileList.end(),
|
||||
[](const auto& l, const auto& r) { return l.name < r.name; });
|
||||
lock.lock();
|
||||
|
||||
m_dirList.clear();
|
||||
m_downloadList.clear();
|
||||
for (auto&& attr : fileList) {
|
||||
if (attr.type == SSH_FILEXFER_TYPE_DIRECTORY) {
|
||||
if (attr.name != ".") {
|
||||
m_dirList.emplace_back(attr.name);
|
||||
}
|
||||
} else if (attr.type == SSH_FILEXFER_TYPE_REGULAR &&
|
||||
(attr.flags & SSH_FILEXFER_ATTR_SIZE) != 0 &&
|
||||
wpi::ends_with(attr.name, ".wpilog")) {
|
||||
m_downloadList.emplace_back(attr.name, attr.size);
|
||||
}
|
||||
}
|
||||
|
||||
m_state = kConnected;
|
||||
break;
|
||||
}
|
||||
case kDisconnecting:
|
||||
session.reset();
|
||||
m_state = kDisconnected;
|
||||
break;
|
||||
case kDownload: {
|
||||
for (auto&& download : m_downloadList) {
|
||||
if (m_state != kDownload) {
|
||||
// user aborted
|
||||
break;
|
||||
}
|
||||
if (!download.enabled) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto remoteFilename = fmt::format(
|
||||
"{}{}{}", m_remoteDir,
|
||||
wpi::ends_with(m_remoteDir, '/') ? "" : "/", download.name);
|
||||
auto localFilename = fs::path{m_localDir} / download.name;
|
||||
uint64_t fileSize = download.size;
|
||||
|
||||
lock.unlock();
|
||||
|
||||
// open local file
|
||||
std::error_code ec;
|
||||
fs::file_t of = fs::OpenFileForWrite(localFilename, ec,
|
||||
fs::CD_CreateNew, fs::OF_None);
|
||||
if (ec) {
|
||||
// failed to open
|
||||
lock.lock();
|
||||
download.status = ec.message();
|
||||
continue;
|
||||
}
|
||||
int ofd = fs::FileToFd(of, ec, fs::OF_None);
|
||||
if (ofd == -1 || ec) {
|
||||
// failed to convert to fd
|
||||
lock.lock();
|
||||
download.status = ec.message();
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
// open remote file
|
||||
sftp::File f = session->Open(remoteFilename, O_RDONLY, 0);
|
||||
|
||||
// copy in chunks
|
||||
uint64_t total = 0;
|
||||
while (total < fileSize) {
|
||||
uint64_t toCopy = (std::min)(fileSize - total,
|
||||
static_cast<uint64_t>(kBufSize));
|
||||
auto copied = f.Read(copyBuf.get(), toCopy);
|
||||
if (write(ofd, copyBuf.get(), copied) !=
|
||||
static_cast<int64_t>(copied)) {
|
||||
// error writing
|
||||
close(ofd);
|
||||
fs::remove(localFilename, ec);
|
||||
lock.lock();
|
||||
download.status = "error writing local file";
|
||||
goto err;
|
||||
}
|
||||
total += copied;
|
||||
lock.lock();
|
||||
download.complete = static_cast<float>(total) / fileSize;
|
||||
lock.unlock();
|
||||
}
|
||||
|
||||
// close local file
|
||||
close(ofd);
|
||||
ofd = -1;
|
||||
|
||||
// delete remote file (if enabled)
|
||||
if (m_deleteAfter) {
|
||||
f = sftp::File{};
|
||||
session->Unlink(remoteFilename);
|
||||
}
|
||||
} catch (sftp::Exception& ex) {
|
||||
if (ofd != -1) {
|
||||
// close local file and delete it (due to failure)
|
||||
close(ofd);
|
||||
fs::remove(localFilename, ec);
|
||||
}
|
||||
lock.lock();
|
||||
download.status = ex.what();
|
||||
if (ex.err == SSH_FX_OK || ex.err == SSH_FX_CONNECTION_LOST) {
|
||||
throw;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
lock.lock();
|
||||
err : {}
|
||||
}
|
||||
if (m_state == kDownload) {
|
||||
m_state = kDownloadDone;
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
} catch (sftp::Exception& ex) {
|
||||
m_error = ex.what();
|
||||
session.reset();
|
||||
m_state = kDisconnected;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user