diff --git a/src/main/native/cpp/Storage.cpp b/src/main/native/cpp/Storage.cpp index 82c0aa4f91..8a8f27df77 100644 --- a/src/main/native/cpp/Storage.cpp +++ b/src/main/native/cpp/Storage.cpp @@ -7,12 +7,6 @@ #include "Storage.h" -#include -#include -#include - -#include "llvm/StringExtras.h" -#include "support/Base64.h" #include "support/timestamp.h" #include "Handle.h" @@ -717,7 +711,7 @@ void Storage::DeleteAllEntries() { dispatcher->QueueOutgoing(Message::ClearEntries(), nullptr, nullptr); } -inline Storage::Entry* Storage::GetOrNew(StringRef name, bool* is_new) { +Storage::Entry* Storage::GetOrNew(StringRef name, bool* is_new) { auto& entry = m_entries[name]; if (!entry) { if (is_new) *is_new = true; @@ -813,38 +807,6 @@ std::vector Storage::GetEntryInfo(int inst, StringRef prefix, return infos; } -/* Escapes and writes a string, including start and end double quotes */ -static void WriteString(std::ostream& os, llvm::StringRef str) { - os << '"'; - for (auto c : str) { - switch (c) { - case '\\': - os << "\\\\"; - break; - case '\t': - os << "\\t"; - break; - case '\n': - os << "\\n"; - break; - case '"': - os << "\\\""; - break; - default: - if (std::isprint(c) && c != '=') { - os << c; - break; - } - - // Write out the escaped representation. - os << "\\x"; - os << llvm::hexdigit((c >> 4) & 0xF); - os << llvm::hexdigit((c >> 0) & 0xF); - } - } - os << '"'; -} - bool Storage::GetPersistentEntries( bool periodic, std::vector>>* entries) @@ -873,508 +835,6 @@ bool Storage::GetPersistentEntries( return true; } -static void SavePersistentImpl( - std::ostream& os, - llvm::ArrayRef>> entries) { - std::string base64_encoded; - - // header - os << "[NetworkTables Storage 3.0]\n"; - - for (auto& i : entries) { - // type - auto v = i.second; - if (!v) continue; - switch (v->type()) { - case NT_BOOLEAN: - os << "boolean "; - break; - case NT_DOUBLE: - os << "double "; - break; - case NT_STRING: - os << "string "; - break; - case NT_RAW: - os << "raw "; - break; - case NT_BOOLEAN_ARRAY: - os << "array boolean "; - break; - case NT_DOUBLE_ARRAY: - os << "array double "; - break; - case NT_STRING_ARRAY: - os << "array string "; - break; - default: - continue; - } - - // name - WriteString(os, i.first); - - // = - os << '='; - - // value - switch (v->type()) { - case NT_BOOLEAN: - os << (v->GetBoolean() ? "true" : "false"); - break; - case NT_DOUBLE: - os << v->GetDouble(); - break; - case NT_STRING: - WriteString(os, v->GetString()); - break; - case NT_RAW: - wpi::Base64Encode(v->GetRaw(), &base64_encoded); - os << base64_encoded; - break; - case NT_BOOLEAN_ARRAY: { - bool first = true; - for (auto elem : v->GetBooleanArray()) { - if (!first) os << ','; - first = false; - os << (elem ? "true" : "false"); - } - break; - } - case NT_DOUBLE_ARRAY: { - bool first = true; - for (auto elem : v->GetDoubleArray()) { - if (!first) os << ','; - first = false; - os << elem; - } - break; - } - case NT_STRING_ARRAY: { - bool first = true; - for (auto& elem : v->GetStringArray()) { - if (!first) os << ','; - first = false; - WriteString(os, elem); - } - break; - } - default: - break; - } - - // eol - os << '\n'; - } -} - -void Storage::SavePersistent(std::ostream& os, bool periodic) const { - std::vector>> entries; - if (!GetPersistentEntries(periodic, &entries)) return; - SavePersistentImpl(os, entries); -} - -const char* Storage::SavePersistent(StringRef filename, bool periodic) const { - std::string fn = filename; - std::string tmp = filename; - tmp += ".tmp"; - std::string bak = filename; - bak += ".bak"; - - // Get entries before creating file - std::vector>> entries; - if (!GetPersistentEntries(periodic, &entries)) return nullptr; - - const char* err = nullptr; - - // start by writing to temporary file - std::ofstream os(tmp); - if (!os) { - err = "could not open file"; - goto done; - } - DEBUG("saving persistent file '" << filename << "'"); - SavePersistentImpl(os, entries); - os.flush(); - if (!os) { - os.close(); - std::remove(tmp.c_str()); - err = "error saving file"; - goto done; - } - os.close(); - - // Safely move to real file. We ignore any failures related to the backup. - std::remove(bak.c_str()); - std::rename(fn.c_str(), bak.c_str()); - if (std::rename(tmp.c_str(), fn.c_str()) != 0) { - std::rename(bak.c_str(), fn.c_str()); // attempt to restore backup - err = "could not rename temp file to real file"; - goto done; - } - -done: - // try again if there was an error - if (err && periodic) m_persistent_dirty = true; - return err; -} - -/* Extracts an escaped string token. Does not unescape the string. - * If a string cannot be matched, an empty string is returned. - * If the string is unterminated, an empty tail string is returned. - * The returned token includes the starting and trailing quotes (unless the - * string is unterminated). - * Returns a pair containing the extracted token (if any) and the remaining - * tail string. - */ -static std::pair ReadStringToken( - llvm::StringRef source) { - // Match opening quote - if (source.empty() || source.front() != '"') - return std::make_pair(llvm::StringRef(), source); - - // Scan for ending double quote, checking for escaped as we go. - std::size_t size = source.size(); - std::size_t pos; - for (pos = 1; pos < size; ++pos) { - if (source[pos] == '"' && source[pos - 1] != '\\') { - ++pos; // we want to include the trailing quote in the result - break; - } - } - return std::make_pair(source.slice(0, pos), source.substr(pos)); -} - -static int fromxdigit(char ch) { - if (ch >= 'a' && ch <= 'f') - return (ch - 'a' + 10); - else if (ch >= 'A' && ch <= 'F') - return (ch - 'A' + 10); - else - return ch - '0'; -} - -static void UnescapeString(llvm::StringRef source, std::string* dest) { - assert(source.size() >= 2 && source.front() == '"' && source.back() == '"'); - dest->clear(); - dest->reserve(source.size() - 2); - for (auto s = source.begin() + 1, end = source.end() - 1; s != end; ++s) { - if (*s != '\\') { - dest->push_back(*s); - continue; - } - switch (*++s) { - case '\\': - case '"': - dest->push_back(s[-1]); - break; - case 't': - dest->push_back('\t'); - break; - case 'n': - dest->push_back('\n'); - break; - case 'x': { - if (!isxdigit(*(s + 1))) { - dest->push_back('x'); // treat it like a unknown escape - break; - } - int ch = fromxdigit(*++s); - if (isxdigit(*(s + 1))) { - ch <<= 4; - ch |= fromxdigit(*++s); - } - dest->push_back(static_cast(ch)); - break; - } - default: - dest->push_back(s[-1]); - break; - } - } -} - -bool Storage::LoadPersistent( - std::istream& is, - std::function warn) { - std::string line_str; - std::size_t line_num = 1; - - // entries to add - std::vector>> entries; - - // declare these outside the loop to reduce reallocs - std::string name, str; - std::vector boolean_array; - std::vector double_array; - std::vector string_array; - - // ignore blank lines and lines that start with ; or # (comments) - while (std::getline(is, line_str)) { - llvm::StringRef line = llvm::StringRef(line_str).trim(); - if (!line.empty() && line.front() != ';' && line.front() != '#') break; - } - - // header - if (line_str != "[NetworkTables Storage 3.0]") { - if (warn) warn(line_num, "header line mismatch, ignoring rest of file"); - return false; - } - - while (std::getline(is, line_str)) { - llvm::StringRef line = llvm::StringRef(line_str).trim(); - ++line_num; - - // ignore blank lines and lines that start with ; or # (comments) - if (line.empty() || line.front() == ';' || line.front() == '#') continue; - - // type - llvm::StringRef type_tok; - std::tie(type_tok, line) = line.split(' '); - NT_Type type = NT_UNASSIGNED; - if (type_tok == "boolean") - type = NT_BOOLEAN; - else if (type_tok == "double") - type = NT_DOUBLE; - else if (type_tok == "string") - type = NT_STRING; - else if (type_tok == "raw") - type = NT_RAW; - else if (type_tok == "array") { - llvm::StringRef array_tok; - std::tie(array_tok, line) = line.split(' '); - if (array_tok == "boolean") - type = NT_BOOLEAN_ARRAY; - else if (array_tok == "double") - type = NT_DOUBLE_ARRAY; - else if (array_tok == "string") - type = NT_STRING_ARRAY; - } - if (type == NT_UNASSIGNED) { - if (warn) warn(line_num, "unrecognized type"); - continue; - } - - // name - llvm::StringRef name_tok; - std::tie(name_tok, line) = ReadStringToken(line); - if (name_tok.empty()) { - if (warn) warn(line_num, "missing name"); - continue; - } - if (name_tok.back() != '"') { - if (warn) warn(line_num, "unterminated name string"); - continue; - } - UnescapeString(name_tok, &name); - - // = - line = line.ltrim(" \t"); - if (line.empty() || line.front() != '=') { - if (warn) warn(line_num, "expected = after name"); - continue; - } - line = line.drop_front().ltrim(" \t"); - - // value - std::shared_ptr value; - switch (type) { - case NT_BOOLEAN: - // only true or false is accepted - if (line == "true") - value = Value::MakeBoolean(true); - else if (line == "false") - value = Value::MakeBoolean(false); - else { - if (warn) - warn(line_num, "unrecognized boolean value, not 'true' or 'false'"); - goto next_line; - } - break; - case NT_DOUBLE: { - // need to convert to null-terminated string for strtod() - str.clear(); - str += line; - char* end; - double v = std::strtod(str.c_str(), &end); - if (*end != '\0') { - if (warn) warn(line_num, "invalid double value"); - goto next_line; - } - value = Value::MakeDouble(v); - break; - } - case NT_STRING: { - llvm::StringRef str_tok; - std::tie(str_tok, line) = ReadStringToken(line); - if (str_tok.empty()) { - if (warn) warn(line_num, "missing string value"); - goto next_line; - } - if (str_tok.back() != '"') { - if (warn) warn(line_num, "unterminated string value"); - goto next_line; - } - UnescapeString(str_tok, &str); - value = Value::MakeString(std::move(str)); - break; - } - case NT_RAW: - wpi::Base64Decode(line, &str); - value = Value::MakeRaw(std::move(str)); - break; - case NT_BOOLEAN_ARRAY: { - llvm::StringRef elem_tok; - boolean_array.clear(); - while (!line.empty()) { - std::tie(elem_tok, line) = line.split(','); - elem_tok = elem_tok.trim(" \t"); - if (elem_tok == "true") - boolean_array.push_back(1); - else if (elem_tok == "false") - boolean_array.push_back(0); - else { - if (warn) - warn(line_num, - "unrecognized boolean value, not 'true' or 'false'"); - goto next_line; - } - } - - value = Value::MakeBooleanArray(std::move(boolean_array)); - break; - } - case NT_DOUBLE_ARRAY: { - llvm::StringRef elem_tok; - double_array.clear(); - while (!line.empty()) { - std::tie(elem_tok, line) = line.split(','); - elem_tok = elem_tok.trim(" \t"); - // need to convert to null-terminated string for strtod() - str.clear(); - str += elem_tok; - char* end; - double v = std::strtod(str.c_str(), &end); - if (*end != '\0') { - if (warn) warn(line_num, "invalid double value"); - goto next_line; - } - double_array.push_back(v); - } - - value = Value::MakeDoubleArray(std::move(double_array)); - break; - } - case NT_STRING_ARRAY: { - llvm::StringRef elem_tok; - string_array.clear(); - while (!line.empty()) { - std::tie(elem_tok, line) = ReadStringToken(line); - if (elem_tok.empty()) { - if (warn) warn(line_num, "missing string value"); - goto next_line; - } - if (elem_tok.back() != '"') { - if (warn) warn(line_num, "unterminated string value"); - goto next_line; - } - - UnescapeString(elem_tok, &str); - string_array.push_back(std::move(str)); - - line = line.ltrim(" \t"); - if (line.empty()) break; - if (line.front() != ',') { - if (warn) warn(line_num, "expected comma between strings"); - goto next_line; - } - line = line.drop_front().ltrim(" \t"); - } - - value = Value::MakeStringArray(std::move(string_array)); - break; - } - default: - break; - } - if (!name.empty() && value) - entries.push_back(std::make_pair(std::move(name), std::move(value))); - next_line: - continue; - } - - // copy values into storage as quickly as possible so lock isn't held - { - std::vector> msgs; - std::unique_lock lock(m_mutex); - for (auto& i : entries) { - Entry* entry = GetOrNew(i.first); - auto old_value = entry->value; - entry->value = i.second; - bool was_persist = entry->IsPersistent(); - if (!was_persist) entry->flags |= NT_PERSISTENT; - - // if we're the server, assign an id if it doesn't have one - if (m_server && entry->id == 0xffff) { - unsigned int id = m_idmap.size(); - entry->id = id; - m_idmap.push_back(entry); - } - - // notify (for local listeners) - if (m_notifier.local_notifiers()) { - if (!old_value) { - m_notifier.NotifyEntry(entry->local_id, i.first, i.second, - NT_NOTIFY_NEW | NT_NOTIFY_LOCAL); - } else if (*old_value != *i.second) { - unsigned int notify_flags = NT_NOTIFY_UPDATE | NT_NOTIFY_LOCAL; - if (!was_persist) notify_flags |= NT_NOTIFY_FLAGS; - m_notifier.NotifyEntry(entry->local_id, i.first, i.second, - notify_flags); - } else if (!was_persist) { - m_notifier.NotifyEntry(entry->local_id, i.first, i.second, - NT_NOTIFY_FLAGS | NT_NOTIFY_LOCAL); - } - } - - if (!m_dispatcher) continue; // shortcut - ++entry->seq_num; - - // put on update queue - if (!old_value || old_value->type() != i.second->type()) - msgs.emplace_back(Message::EntryAssign(i.first, entry->id, - entry->seq_num.value(), i.second, - entry->flags)); - else if (entry->id != 0xffff) { - // don't send an update if we don't have an assigned id yet - if (*old_value != *i.second) - msgs.emplace_back(Message::EntryUpdate( - entry->id, entry->seq_num.value(), i.second)); - if (!was_persist) - msgs.emplace_back(Message::FlagsUpdate(entry->id, entry->flags)); - } - } - - if (m_dispatcher) { - auto dispatcher = m_dispatcher; - lock.unlock(); - for (auto& msg : msgs) - dispatcher->QueueOutgoing(std::move(msg), nullptr, nullptr); - } - } - - return true; -} - -const char* Storage::LoadPersistent( - StringRef filename, - std::function warn) { - std::ifstream is(filename); - if (!is) return "could not open file"; - if (!LoadPersistent(is, warn)) return "error reading file"; - return nullptr; -} - void Storage::CreateRpc(unsigned int local_id, StringRef def, unsigned int rpc_uid) { std::unique_lock lock(m_mutex); diff --git a/src/main/native/cpp/Storage_load.cpp b/src/main/native/cpp/Storage_load.cpp new file mode 100644 index 0000000000..2cf1eb35fb --- /dev/null +++ b/src/main/native/cpp/Storage_load.cpp @@ -0,0 +1,441 @@ +/*----------------------------------------------------------------------------*/ +/* Copyright (c) FIRST 2015. All Rights Reserved. */ +/* Open Source Software - may be modified and shared by FRC teams. The code */ +/* must be accompanied by the FIRST BSD license file in the root directory of */ +/* the project. */ +/*----------------------------------------------------------------------------*/ + +#include "Storage.h" + +#include +#include + +#include "llvm/StringExtras.h" +#include "support/Base64.h" + +#include "IDispatcher.h" +#include "IEntryNotifier.h" + +using namespace nt; + +namespace { + +class LoadPersistentImpl { + public: + typedef std::pair> Entry; + typedef std::function WarnFunc; + + LoadPersistentImpl(std::istream& is, WarnFunc warn) + : m_is(is), m_warn(warn) {} + + bool Load(std::vector* entries); + + private: + bool ReadLine(); + bool ReadHeader(); + NT_Type ReadType(); + bool ReadName(std::string* name); + std::shared_ptr ReadValue(NT_Type type); + std::shared_ptr ReadBooleanValue(); + std::shared_ptr ReadDoubleValue(); + std::shared_ptr ReadStringValue(); + std::shared_ptr ReadRawValue(); + std::shared_ptr ReadBooleanArrayValue(); + std::shared_ptr ReadDoubleArrayValue(); + std::shared_ptr ReadStringArrayValue(); + + void Warn(const char* msg) { + if (m_warn) m_warn(m_line_num, msg); + } + + std::istream& m_is; + WarnFunc m_warn; + + llvm::StringRef m_line; + std::string m_line_buf; + std::size_t m_line_num = 0; + + std::string m_buf_str; + std::vector m_buf_boolean_array; + std::vector m_buf_double_array; + std::vector m_buf_string_array; +}; + +} // anonymous namespace + +/* Extracts an escaped string token. Does not unescape the string. + * If a string cannot be matched, an empty string is returned. + * If the string is unterminated, an empty tail string is returned. + * The returned token includes the starting and trailing quotes (unless the + * string is unterminated). + * Returns a pair containing the extracted token (if any) and the remaining + * tail string. + */ +static std::pair ReadStringToken( + llvm::StringRef source) { + // Match opening quote + if (source.empty() || source.front() != '"') + return std::make_pair(llvm::StringRef(), source); + + // Scan for ending double quote, checking for escaped as we go. + std::size_t size = source.size(); + std::size_t pos; + for (pos = 1; pos < size; ++pos) { + if (source[pos] == '"' && source[pos - 1] != '\\') { + ++pos; // we want to include the trailing quote in the result + break; + } + } + return std::make_pair(source.slice(0, pos), source.substr(pos)); +} + +static int fromxdigit(char ch) { + if (ch >= 'a' && ch <= 'f') + return (ch - 'a' + 10); + else if (ch >= 'A' && ch <= 'F') + return (ch - 'A' + 10); + else + return ch - '0'; +} + +static void UnescapeString(llvm::StringRef source, std::string* dest) { + assert(source.size() >= 2 && source.front() == '"' && source.back() == '"'); + dest->clear(); + dest->reserve(source.size() - 2); + for (auto s = source.begin() + 1, end = source.end() - 1; s != end; ++s) { + if (*s != '\\') { + dest->push_back(*s); + continue; + } + switch (*++s) { + case '\\': + case '"': + dest->push_back(s[-1]); + break; + case 't': + dest->push_back('\t'); + break; + case 'n': + dest->push_back('\n'); + break; + case 'x': { + if (!isxdigit(*(s + 1))) { + dest->push_back('x'); // treat it like a unknown escape + break; + } + int ch = fromxdigit(*++s); + if (isxdigit(*(s + 1))) { + ch <<= 4; + ch |= fromxdigit(*++s); + } + dest->push_back(static_cast(ch)); + break; + } + default: + dest->push_back(s[-1]); + break; + } + } +} + +bool LoadPersistentImpl::Load(std::vector* entries) { + if (!ReadHeader()) return false; // header + + // declare this outside the loop to reduce reallocs + std::string name; + + while (ReadLine()) { + // type + NT_Type type = ReadType(); + if (type == NT_UNASSIGNED) { + Warn("unrecognized type"); + continue; + } + + // name + if (!ReadName(&name)) continue; + + // = + m_line = m_line.ltrim(" \t"); + if (m_line.empty() || m_line.front() != '=') { + Warn("expected = after name"); + continue; + } + m_line = m_line.drop_front().ltrim(" \t"); + + // value + auto value = ReadValue(type); + + // move to entries + if (!name.empty() && value) + entries->emplace_back(std::move(name), std::move(value)); + } + return true; +} + +bool LoadPersistentImpl::ReadLine() { + // ignore blank lines and lines that start with ; or # (comments) + while (std::getline(m_is, m_line_buf)) { + ++m_line_num; + m_line = llvm::StringRef(m_line_buf).trim(); + if (!m_line.empty() && m_line.front() != ';' && m_line.front() != '#') + return true; + } + return false; +} + +bool LoadPersistentImpl::ReadHeader() { + // header + if (!ReadLine() || m_line != "[NetworkTables Storage 3.0]") { + Warn("header line mismatch, ignoring rest of file"); + return false; + } + return true; +} + +NT_Type LoadPersistentImpl::ReadType() { + llvm::StringRef tok; + std::tie(tok, m_line) = m_line.split(' '); + if (tok == "boolean") + return NT_BOOLEAN; + else if (tok == "double") + return NT_DOUBLE; + else if (tok == "string") + return NT_STRING; + else if (tok == "raw") + return NT_RAW; + else if (tok == "array") { + llvm::StringRef array_tok; + std::tie(array_tok, m_line) = m_line.split(' '); + if (array_tok == "boolean") + return NT_BOOLEAN_ARRAY; + else if (array_tok == "double") + return NT_DOUBLE_ARRAY; + else if (array_tok == "string") + return NT_STRING_ARRAY; + } + return NT_UNASSIGNED; +} + +bool LoadPersistentImpl::ReadName(std::string* name) { + llvm::StringRef tok; + std::tie(tok, m_line) = ReadStringToken(m_line); + if (tok.empty()) { + Warn("missing name"); + return false; + } + if (tok.back() != '"') { + Warn("unterminated name string"); + return false; + } + UnescapeString(tok, name); + return true; +} + +std::shared_ptr LoadPersistentImpl::ReadValue(NT_Type type) { + switch (type) { + case NT_BOOLEAN: + return ReadBooleanValue(); + case NT_DOUBLE: + return ReadDoubleValue(); + case NT_STRING: + return ReadStringValue(); + case NT_RAW: + return ReadRawValue(); + case NT_BOOLEAN_ARRAY: + return ReadBooleanArrayValue(); + case NT_DOUBLE_ARRAY: + return ReadDoubleArrayValue(); + case NT_STRING_ARRAY: + return ReadStringArrayValue(); + default: + return nullptr; + } +} + +std::shared_ptr LoadPersistentImpl::ReadBooleanValue() { + // only true or false is accepted + if (m_line == "true") return Value::MakeBoolean(true); + if (m_line == "false") return Value::MakeBoolean(false); + Warn("unrecognized boolean value, not 'true' or 'false'"); + return nullptr; +} + +std::shared_ptr LoadPersistentImpl::ReadDoubleValue() { + // need to convert to null-terminated string for strtod() + m_buf_str.clear(); + m_buf_str += m_line; + char* end; + double v = std::strtod(m_buf_str.c_str(), &end); + if (*end != '\0') { + Warn("invalid double value"); + return nullptr; + } + return Value::MakeDouble(v); +} + +std::shared_ptr LoadPersistentImpl::ReadStringValue() { + llvm::StringRef tok; + std::tie(tok, m_line) = ReadStringToken(m_line); + if (tok.empty()) { + Warn("missing string value"); + return nullptr; + } + if (tok.back() != '"') { + Warn("unterminated string value"); + return nullptr; + } + UnescapeString(tok, &m_buf_str); + return Value::MakeString(std::move(m_buf_str)); +} + +std::shared_ptr LoadPersistentImpl::ReadRawValue() { + wpi::Base64Decode(m_line, &m_buf_str); + return Value::MakeRaw(std::move(m_buf_str)); +} + +std::shared_ptr LoadPersistentImpl::ReadBooleanArrayValue() { + m_buf_boolean_array.clear(); + while (!m_line.empty()) { + llvm::StringRef tok; + std::tie(tok, m_line) = m_line.split(','); + tok = tok.trim(" \t"); + if (tok == "true") + m_buf_boolean_array.push_back(1); + else if (tok == "false") + m_buf_boolean_array.push_back(0); + else { + Warn("unrecognized boolean value, not 'true' or 'false'"); + return nullptr; + } + } + return Value::MakeBooleanArray(std::move(m_buf_boolean_array)); +} + +std::shared_ptr LoadPersistentImpl::ReadDoubleArrayValue() { + m_buf_double_array.clear(); + while (!m_line.empty()) { + llvm::StringRef tok; + std::tie(tok, m_line) = m_line.split(','); + tok = tok.trim(" \t"); + // need to convert to null-terminated string for strtod() + m_buf_str.clear(); + m_buf_str += tok; + char* end; + double v = std::strtod(m_buf_str.c_str(), &end); + if (*end != '\0') { + Warn("invalid double value"); + return nullptr; + } + m_buf_double_array.push_back(v); + } + + return Value::MakeDoubleArray(std::move(m_buf_double_array)); +} + +std::shared_ptr LoadPersistentImpl::ReadStringArrayValue() { + m_buf_string_array.clear(); + while (!m_line.empty()) { + llvm::StringRef tok; + std::tie(tok, m_line) = ReadStringToken(m_line); + if (tok.empty()) { + Warn("missing string value"); + return nullptr; + } + if (tok.back() != '"') { + Warn("unterminated string value"); + return nullptr; + } + + UnescapeString(tok, &m_buf_str); + m_buf_string_array.push_back(std::move(m_buf_str)); + + m_line = m_line.ltrim(" \t"); + if (m_line.empty()) break; + if (m_line.front() != ',') { + Warn("expected comma between strings"); + return nullptr; + } + m_line = m_line.drop_front().ltrim(" \t"); + } + + return Value::MakeStringArray(std::move(m_buf_string_array)); +} + +bool Storage::LoadPersistent( + std::istream& is, + std::function warn) { + // entries to add + std::vector entries; + + // load file + if (!LoadPersistentImpl(is, warn).Load(&entries)) return false; + + // copy values into storage as quickly as possible so lock isn't held + std::vector> msgs; + std::unique_lock lock(m_mutex); + for (auto& i : entries) { + Entry* entry = GetOrNew(i.first); + auto old_value = entry->value; + entry->value = i.second; + bool was_persist = entry->IsPersistent(); + if (!was_persist) entry->flags |= NT_PERSISTENT; + + // if we're the server, assign an id if it doesn't have one + if (m_server && entry->id == 0xffff) { + unsigned int id = m_idmap.size(); + entry->id = id; + m_idmap.push_back(entry); + } + + // notify (for local listeners) + if (m_notifier.local_notifiers()) { + if (!old_value) { + m_notifier.NotifyEntry(entry->local_id, i.first, i.second, + NT_NOTIFY_NEW | NT_NOTIFY_LOCAL); + } else if (*old_value != *i.second) { + unsigned int notify_flags = NT_NOTIFY_UPDATE | NT_NOTIFY_LOCAL; + if (!was_persist) notify_flags |= NT_NOTIFY_FLAGS; + m_notifier.NotifyEntry(entry->local_id, i.first, i.second, + notify_flags); + } else if (!was_persist) { + m_notifier.NotifyEntry(entry->local_id, i.first, i.second, + NT_NOTIFY_FLAGS | NT_NOTIFY_LOCAL); + } + } + + if (!m_dispatcher) continue; // shortcut + ++entry->seq_num; + + // put on update queue + if (!old_value || old_value->type() != i.second->type()) + msgs.emplace_back(Message::EntryAssign( + i.first, entry->id, entry->seq_num.value(), i.second, entry->flags)); + else if (entry->id != 0xffff) { + // don't send an update if we don't have an assigned id yet + if (*old_value != *i.second) + msgs.emplace_back( + Message::EntryUpdate(entry->id, entry->seq_num.value(), i.second)); + if (!was_persist) + msgs.emplace_back(Message::FlagsUpdate(entry->id, entry->flags)); + } + } + + if (m_dispatcher) { + auto dispatcher = m_dispatcher; + lock.unlock(); + for (auto& msg : msgs) + dispatcher->QueueOutgoing(std::move(msg), nullptr, nullptr); + } + + return true; +} + +const char* Storage::LoadPersistent( + StringRef filename, + std::function warn) { + std::ifstream is(filename); + if (!is) return "could not open file"; + if (!LoadPersistent(is, warn)) return "error reading file"; + return nullptr; +} diff --git a/src/main/native/cpp/Storage_save.cpp b/src/main/native/cpp/Storage_save.cpp new file mode 100644 index 0000000000..c9efafd9b3 --- /dev/null +++ b/src/main/native/cpp/Storage_save.cpp @@ -0,0 +1,226 @@ +/*----------------------------------------------------------------------------*/ +/* Copyright (c) FIRST 2015. All Rights Reserved. */ +/* Open Source Software - may be modified and shared by FRC teams. The code */ +/* must be accompanied by the FIRST BSD license file in the root directory of */ +/* the project. */ +/*----------------------------------------------------------------------------*/ + +#include "Storage.h" + +#include +#include + +#include "llvm/StringExtras.h" +#include "support/Base64.h" + +#include "Log.h" + +using namespace nt; + +namespace { + +class SavePersistentImpl { + public: + typedef std::pair> Entry; + + SavePersistentImpl(std::ostream& os) : m_os(os) {} + + void Save(llvm::ArrayRef entries); + + private: + void WriteString(llvm::StringRef str); + void WriteHeader(); + void WriteEntries(llvm::ArrayRef entries); + void WriteEntry(llvm::StringRef name, const Value& value); + bool WriteType(NT_Type type); + void WriteValue(const Value& value); + + std::ostream& m_os; +}; + +} // anonymous namespace + +/* Escapes and writes a string, including start and end double quotes */ +void SavePersistentImpl::WriteString(llvm::StringRef str) { + m_os << '"'; + for (auto c : str) { + switch (c) { + case '\\': + m_os << "\\\\"; + break; + case '\t': + m_os << "\\t"; + break; + case '\n': + m_os << "\\n"; + break; + case '"': + m_os << "\\\""; + break; + default: + if (std::isprint(c) && c != '=') { + m_os << c; + break; + } + + // Write out the escaped representation. + m_os << "\\x"; + m_os << llvm::hexdigit((c >> 4) & 0xF); + m_os << llvm::hexdigit((c >> 0) & 0xF); + } + } + m_os << '"'; +} + +void SavePersistentImpl::Save(llvm::ArrayRef entries) { + WriteHeader(); + WriteEntries(entries); +} + +void SavePersistentImpl::WriteHeader() { + m_os << "[NetworkTables Storage 3.0]\n"; +} + +void SavePersistentImpl::WriteEntries(llvm::ArrayRef entries) { + for (auto& i : entries) { + if (!i.second) continue; + WriteEntry(i.first, *i.second); + } +} + +void SavePersistentImpl::WriteEntry(llvm::StringRef name, const Value& value) { + if (!WriteType(value.type())) return; // type + WriteString(name); // name + m_os << '='; // '=' + WriteValue(value); // value + m_os << '\n'; // eol +} + +bool SavePersistentImpl::WriteType(NT_Type type) { + switch (type) { + case NT_BOOLEAN: + m_os << "boolean "; + break; + case NT_DOUBLE: + m_os << "double "; + break; + case NT_STRING: + m_os << "string "; + break; + case NT_RAW: + m_os << "raw "; + break; + case NT_BOOLEAN_ARRAY: + m_os << "array boolean "; + break; + case NT_DOUBLE_ARRAY: + m_os << "array double "; + break; + case NT_STRING_ARRAY: + m_os << "array string "; + break; + default: + return false; + } + return true; +} + +void SavePersistentImpl::WriteValue(const Value& value) { + switch (value.type()) { + case NT_BOOLEAN: + m_os << (value.GetBoolean() ? "true" : "false"); + break; + case NT_DOUBLE: + m_os << value.GetDouble(); + break; + case NT_STRING: + WriteString(value.GetString()); + break; + case NT_RAW: { + std::string base64_encoded; + wpi::Base64Encode(value.GetRaw(), &base64_encoded); + m_os << base64_encoded; + break; + } + case NT_BOOLEAN_ARRAY: { + bool first = true; + for (auto elem : value.GetBooleanArray()) { + if (!first) m_os << ','; + first = false; + m_os << (elem ? "true" : "false"); + } + break; + } + case NT_DOUBLE_ARRAY: { + bool first = true; + for (auto elem : value.GetDoubleArray()) { + if (!first) m_os << ','; + first = false; + m_os << elem; + } + break; + } + case NT_STRING_ARRAY: { + bool first = true; + for (auto& elem : value.GetStringArray()) { + if (!first) m_os << ','; + first = false; + WriteString(elem); + } + break; + } + default: + break; + } +} + +void Storage::SavePersistent(std::ostream& os, bool periodic) const { + std::vector entries; + if (!GetPersistentEntries(periodic, &entries)) return; + SavePersistentImpl(os).Save(entries); +} + +const char* Storage::SavePersistent(StringRef filename, bool periodic) const { + std::string fn = filename; + std::string tmp = filename; + tmp += ".tmp"; + std::string bak = filename; + bak += ".bak"; + + // Get entries before creating file + std::vector entries; + if (!GetPersistentEntries(periodic, &entries)) return nullptr; + + const char* err = nullptr; + + // start by writing to temporary file + std::ofstream os(tmp); + if (!os) { + err = "could not open file"; + goto done; + } + DEBUG("saving persistent file '" << filename << "'"); + SavePersistentImpl(os).Save(entries); + os.flush(); + if (!os) { + os.close(); + std::remove(tmp.c_str()); + err = "error saving file"; + goto done; + } + os.close(); + + // Safely move to real file. We ignore any failures related to the backup. + std::remove(bak.c_str()); + std::rename(fn.c_str(), bak.c_str()); + if (std::rename(tmp.c_str(), fn.c_str()) != 0) { + std::rename(bak.c_str(), fn.c_str()); // attempt to restore backup + err = "could not rename temp file to real file"; + goto done; + } + +done: + // try again if there was an error + if (err && periodic) m_persistent_dirty = true; + return err; +} diff --git a/src/test/native/cpp/StorageTest.cpp b/src/test/native/cpp/StorageTest.cpp index 34b00c7068..de00a2527e 100644 --- a/src/test/native/cpp/StorageTest.cpp +++ b/src/test/native/cpp/StorageTest.cpp @@ -385,7 +385,7 @@ TEST_P(StorageTestPopulated, SetDefaultEntryEmptyName) { EXPECT_EQ(0u, idmap().size()); } -TEST_P(StorageTestPopulated, SetDefaultEntryEmptyValue) { +TEST_P(StorageTestPopulated, SetDefaultEntryEmptyValue) { auto value = Value::MakeBoolean(true); auto ret_val = storage.SetDefaultEntryValue("", nullptr); EXPECT_FALSE(ret_val); @@ -616,7 +616,7 @@ TEST_P(StorageTestEmpty, LoadPersistentBadHeader) { std::istringstream iss(""); EXPECT_CALL( warn, - Warn(1, llvm::StringRef("header line mismatch, ignoring rest of file"))); + Warn(0, llvm::StringRef("header line mismatch, ignoring rest of file"))); EXPECT_FALSE(storage.LoadPersistent(iss, warn_func)); std::istringstream iss2("[NetworkTables");