From b1546b8038b9d6c3b91c3e52240e8096319d9106 Mon Sep 17 00:00:00 2001 From: Matt Date: Sat, 17 Jun 2023 21:47:18 -0700 Subject: [PATCH] Add SQL config manager (#818) Serializes settings using a sqlite database instead of just putting them on the filesystem. Ideally since sqlite deals with filesystem robustness stuff this should work a lot better Merging this now so we have lots of time to stabilize pre-beta --- .gitignore | 1 + photon-core/build.gradle | 2 + .../common/configuration/ConfigManager.java | 396 +++++---------- .../common/configuration/ConfigProvider.java | 38 ++ .../common/configuration/HardwareConfig.java | 49 ++ .../configuration/HardwareSettings.java | 5 + .../configuration/LegacyConfigProvider.java | 443 ++++++++++++++++ .../common/configuration/NetworkConfig.java | 26 + .../configuration/PhotonConfiguration.java | 13 + .../configuration/SqlConfigProvider.java | 475 ++++++++++++++++++ .../photonvision/common/logging/LogGroup.java | 3 +- .../photonvision/common/logging/Logger.java | 3 +- .../common/util/file/FileUtils.java | 3 +- .../common/util/file/JacksonUtils.java | 23 + .../common/configuration/ConfigTest.java | 3 +- .../configuration/NetworkConfigTest.java | 8 +- .../common/configuration/SQLConfigTest.java | 68 +++ .../common/util/LogFileManagementTest.java | 10 +- .../src/main/java/org/photonvision/Main.java | 1 + 19 files changed, 1278 insertions(+), 292 deletions(-) create mode 100644 photon-core/src/main/java/org/photonvision/common/configuration/ConfigProvider.java create mode 100644 photon-core/src/main/java/org/photonvision/common/configuration/LegacyConfigProvider.java create mode 100644 photon-core/src/main/java/org/photonvision/common/configuration/SqlConfigProvider.java create mode 100644 photon-core/src/test/java/org/photonvision/common/configuration/SQLConfigTest.java diff --git a/.gitignore b/.gitignore index bf87286f8..4c59f92df 100644 --- a/.gitignore +++ b/.gitignore @@ -157,3 +157,4 @@ photonlib-java-examples/*/vendordeps/* photonlib-cpp-examples/*/vendordeps/* */networktables.json +*.sqlite diff --git a/photon-core/build.gradle b/photon-core/build.gradle index 782e16a21..feaa5b0c7 100644 --- a/photon-core/build.gradle +++ b/photon-core/build.gradle @@ -25,6 +25,8 @@ dependencies { implementation 'org.zeroturnaround:zt-zip:1.14' implementation wpilibTools.deps.wpilibJava("apriltag") + + implementation "org.xerial:sqlite-jdbc:3.41.0.0" } task writeCurrentVersionJava { diff --git a/photon-core/src/main/java/org/photonvision/common/configuration/ConfigManager.java b/photon-core/src/main/java/org/photonvision/common/configuration/ConfigManager.java index 94aab34bf..8fd4dd47f 100644 --- a/photon-core/src/main/java/org/photonvision/common/configuration/ConfigManager.java +++ b/photon-core/src/main/java/org/photonvision/common/configuration/ConfigManager.java @@ -17,11 +17,11 @@ package org.photonvision.common.configuration; -import com.fasterxml.jackson.core.JsonProcessingException; import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -29,13 +29,9 @@ import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.time.temporal.TemporalAccessor; import java.util.*; -import java.util.stream.Collectors; import org.photonvision.common.logging.LogGroup; import org.photonvision.common.logging.Logger; import org.photonvision.common.util.file.FileUtils; -import org.photonvision.common.util.file.JacksonUtils; -import org.photonvision.vision.pipeline.CVPipelineSettings; -import org.photonvision.vision.pipeline.DriverModePipelineSettings; import org.photonvision.vision.processes.VisionSource; import org.zeroturnaround.zip.ZipUtil; @@ -47,290 +43,141 @@ public class ConfigManager { public static final String HW_SET_FNAME = "hardwareSettings.json"; public static final String NET_SET_FNAME = "networkSettings.json"; - private PhotonConfiguration config; - private final File hardwareConfigFile; - private final File hardwareSettingsFile; - private final File networkConfigFile; - private final File camerasFolder; - final File configDirectoryFile; - private long saveRequestTimestamp = -1; + private final ConfigProvider m_provider; + private Thread settingsSaveThread; + private long saveRequestTimestamp = -1; + + enum ConfigSaveStrategy { + SQL, + LEGACY, + ATOMIC_ZIP; + } + + // This logic decides which kind of ConfigManager we load as the default. If we want + // to switch back to the legacy config manager, change this constant + private static final ConfigSaveStrategy m_saveStrat = ConfigSaveStrategy.SQL; public static ConfigManager getInstance() { if (INSTANCE == null) { - INSTANCE = new ConfigManager(getRootFolder()); + switch (m_saveStrat) { + case SQL: + INSTANCE = new ConfigManager(getRootFolder(), new SqlConfigProvider(getRootFolder())); + break; + case LEGACY: + INSTANCE = new ConfigManager(getRootFolder(), new LegacyConfigProvider(getRootFolder())); + break; + case ATOMIC_ZIP: + // not yet done, fall through + default: + break; + } } return INSTANCE; } + private void translateLegacyIfPresent(Path folderPath) { + if (!(m_provider instanceof SqlConfigProvider)) { + // Cannot import into SQL if we aren't in SQL mode rn + return; + } + logger.info("Translating settings zip!"); + + var maybeCams = Path.of(folderPath.toAbsolutePath().toString(), "cameras").toFile(); + var maybeCamsBak = Path.of(folderPath.toAbsolutePath().toString(), "cameras_backup").toFile(); + + if (maybeCams.exists() && maybeCams.isDirectory()) { + var legacy = new LegacyConfigProvider(folderPath); + legacy.load(); + var loadedConfig = legacy.getConfig(); + + // yeet our current cameras directory, not needed anymore + if (maybeCamsBak.exists()) FileUtils.deleteDirectory(maybeCamsBak.toPath()); + if (!maybeCams.canWrite()) { + maybeCams.setWritable(true); + } + + try { + Files.move(maybeCams.toPath(), maybeCamsBak.toPath(), StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + logger.error("Exception moving cameras to cameras_bak!", e); + + // Try to just copy from cams to cams-bak instead of moving? Windows sometimes needs us to + // do that + try { + org.apache.commons.io.FileUtils.copyDirectory(maybeCams, maybeCamsBak); + } catch (IOException e1) { + // So we can't move to cams_bak, and we can't copy and delete either? We just have to give + // up here on preserving the old folder + logger.error("Exception while backup-copying cameras to cameras_bak!", e); + e1.printStackTrace(); + } + + // So we can't save the old config, and we couldn't copy the folder + // But we've loaded the config. So just try to delete the directory so we don't try to load + // form it next time. That does mean we have no backup recourse, tho + if (maybeCams.exists()) FileUtils.deleteDirectory(maybeCams.toPath()); + } + + // Save the same config out using SQL loader + var sql = new SqlConfigProvider(getRootFolder()); + sql.setConfig(loadedConfig); + sql.saveToDisk(); + } + } + public static void saveUploadedSettingsZip(File uploadPath) { + // Unpack to /tmp/something/photonvision var folderPath = Path.of(System.getProperty("java.io.tmpdir"), "photonvision").toFile(); folderPath.mkdirs(); ZipUtil.unpack(uploadPath, folderPath); + + // Nuke the current settings directory FileUtils.deleteDirectory(getRootFolder()); - try { - org.apache.commons.io.FileUtils.copyDirectory(folderPath, getRootFolder().toFile()); - logger.info("Copied settings successfully!"); - } catch (IOException e) { - logger.error("Exception copying uploaded settings!", e); - return; + + // If there's a cameras folder in the upload, we know we need to import from the + // old style + var maybeCams = Path.of(folderPath.getAbsolutePath(), "cameras").toFile(); + if (maybeCams.exists() && maybeCams.isDirectory()) { + var legacy = new LegacyConfigProvider(folderPath.toPath()); + legacy.load(); + var loadedConfig = legacy.getConfig(); + + var sql = new SqlConfigProvider(getRootFolder()); + sql.setConfig(loadedConfig); + sql.saveToDisk(); + } else { + // new structure -- just copy and save like we used to + try { + org.apache.commons.io.FileUtils.copyDirectory(folderPath, getRootFolder().toFile()); + logger.info("Copied settings successfully!"); + } catch (IOException e) { + logger.error("Exception copying uploaded settings!", e); + } } } public PhotonConfiguration getConfig() { - return config; + return m_provider.getConfig(); } private static Path getRootFolder() { return Path.of("photonvision_config"); } - ConfigManager(Path configDirectoryFile) { - this.configDirectoryFile = new File(configDirectoryFile.toUri()); - this.hardwareConfigFile = - new File(Path.of(configDirectoryFile.toString(), HW_CFG_FNAME).toUri()); - this.hardwareSettingsFile = - new File(Path.of(configDirectoryFile.toString(), HW_SET_FNAME).toUri()); - this.networkConfigFile = - new File(Path.of(configDirectoryFile.toString(), NET_SET_FNAME).toUri()); - this.camerasFolder = new File(Path.of(configDirectoryFile.toString(), "cameras").toUri()); + ConfigManager(Path configDirectory, ConfigProvider provider) { + this.configDirectoryFile = new File(configDirectory.toUri()); + m_provider = provider; settingsSaveThread = new Thread(this::saveAndWriteTask); settingsSaveThread.start(); } public void load() { - logger.info("Loading settings..."); - if (!configDirectoryFile.exists()) { - if (configDirectoryFile.mkdirs()) { - logger.debug("Root config folder did not exist. Created!"); - } else { - logger.error("Failed to create root config folder!"); - } - } - if (!configDirectoryFile.canWrite()) { - logger.debug("Making root dir writeable..."); - try { - var success = configDirectoryFile.setWritable(true); - if (success) logger.debug("Set root dir writeable!"); - else logger.error("Could not make root dir writeable!"); - } catch (SecurityException e) { - logger.error("Could not make root dir writeable!", e); - } - } - - HardwareConfig hardwareConfig; - HardwareSettings hardwareSettings; - NetworkConfig networkConfig; - - if (hardwareConfigFile.exists()) { - try { - hardwareConfig = - JacksonUtils.deserialize(hardwareConfigFile.toPath(), HardwareConfig.class); - if (hardwareConfig == null) { - logger.error("Could not deserialize hardware config! Loading defaults"); - hardwareConfig = new HardwareConfig(); - } - } catch (IOException e) { - logger.error("Could not deserialize hardware config! Loading defaults"); - hardwareConfig = new HardwareConfig(); - } - } else { - logger.info("Hardware config does not exist! Loading defaults"); - hardwareConfig = new HardwareConfig(); - } - - if (hardwareSettingsFile.exists()) { - try { - hardwareSettings = - JacksonUtils.deserialize(hardwareSettingsFile.toPath(), HardwareSettings.class); - if (hardwareSettings == null) { - logger.error("Could not deserialize hardware settings! Loading defaults"); - hardwareSettings = new HardwareSettings(); - } - } catch (IOException e) { - logger.error("Could not deserialize hardware settings! Loading defaults"); - hardwareSettings = new HardwareSettings(); - } - } else { - logger.info("Hardware settings does not exist! Loading defaults"); - hardwareSettings = new HardwareSettings(); - } - - if (networkConfigFile.exists()) { - try { - networkConfig = JacksonUtils.deserialize(networkConfigFile.toPath(), NetworkConfig.class); - if (networkConfig == null) { - logger.error("Could not deserialize network config! Loading defaults"); - networkConfig = new NetworkConfig(); - } - } catch (IOException e) { - logger.error("Could not deserialize network config! Loading defaults"); - networkConfig = new NetworkConfig(); - } - } else { - logger.info("Network config file does not exist! Loading defaults"); - networkConfig = new NetworkConfig(); - } - - if (!camerasFolder.exists()) { - if (camerasFolder.mkdirs()) { - logger.debug("Cameras config folder did not exist. Created!"); - } else { - logger.error("Failed to create cameras config folder!"); - } - } - - HashMap cameraConfigurations = loadCameraConfigs(); - - this.config = - new PhotonConfiguration( - hardwareConfig, hardwareSettings, networkConfig, cameraConfigurations); - } - - public void saveToDisk() { - // Delete old configs - FileUtils.deleteDirectory(camerasFolder.toPath()); - - try { - JacksonUtils.serialize(networkConfigFile.toPath(), config.getNetworkConfig()); - } catch (IOException e) { - logger.error("Could not save network config!", e); - } - try { - JacksonUtils.serialize(hardwareSettingsFile.toPath(), config.getHardwareSettings()); - } catch (IOException e) { - logger.error("Could not save hardware config!", e); - } - - // save all of our cameras - var cameraConfigMap = config.getCameraConfigurations(); - for (var subdirName : cameraConfigMap.keySet()) { - var camConfig = cameraConfigMap.get(subdirName); - var subdir = Path.of(camerasFolder.toPath().toString(), subdirName); - - if (!subdir.toFile().exists()) { - // TODO: check for error - subdir.toFile().mkdirs(); - } - - try { - JacksonUtils.serialize(Path.of(subdir.toString(), "config.json"), camConfig); - } catch (IOException e) { - logger.error("Could not save config.json for " + subdir, e); - } - - try { - JacksonUtils.serialize( - Path.of(subdir.toString(), "drivermode.json"), camConfig.driveModeSettings); - } catch (IOException e) { - logger.error("Could not save drivermode.json for " + subdir, e); - } - - for (var pipe : camConfig.pipelineSettings) { - var pipePath = Path.of(subdir.toString(), "pipelines", pipe.pipelineNickname + ".json"); - - if (!pipePath.getParent().toFile().exists()) { - // TODO: check for error - pipePath.getParent().toFile().mkdirs(); - } - - try { - JacksonUtils.serialize(pipePath, pipe); - } catch (IOException e) { - logger.error("Could not save " + pipe.pipelineNickname + ".json!", e); - } - } - } - logger.info("Settings saved!"); - } - - private HashMap loadCameraConfigs() { - HashMap loadedConfigurations = new HashMap<>(); - try { - var subdirectories = - Files.list(camerasFolder.toPath()) - .filter(f -> f.toFile().isDirectory()) - .collect(Collectors.toList()); - - for (var subdir : subdirectories) { - var cameraConfigPath = Path.of(subdir.toString(), "config.json"); - CameraConfiguration loadedConfig = null; - try { - loadedConfig = - JacksonUtils.deserialize( - cameraConfigPath.toAbsolutePath(), CameraConfiguration.class); - } catch (JsonProcessingException e) { - logger.error("Camera config deserialization failed!", e); - e.printStackTrace(); - } - if (loadedConfig == null) { // If the file could not be deserialized - logger.warn("Could not load camera " + subdir + "'s config.json! Loading " + "default"); - continue; // TODO how do we later try to load this camera if it gets reconnected? - } - - // At this point we have only loaded the base stuff - // We still need to deserialize pipelines, as well as - // driver mode settings - var driverModeFile = Path.of(subdir.toString(), "drivermode.json"); - DriverModePipelineSettings driverMode; - try { - driverMode = - JacksonUtils.deserialize( - driverModeFile.toAbsolutePath(), DriverModePipelineSettings.class); - } catch (JsonProcessingException e) { - logger.error("Could not deserialize drivermode.json! Loading defaults"); - logger.debug(Arrays.toString(e.getStackTrace())); - driverMode = new DriverModePipelineSettings(); - } - if (driverMode == null) { - logger.warn( - "Could not load camera " + subdir + "'s drivermode.json! Loading" + " default"); - driverMode = new DriverModePipelineSettings(); - } - - // Load pipelines by mapping the files within the pipelines subdir - // to their deserialized equivalents - var pipelineSubdirectory = Path.of(subdir.toString(), "pipelines"); - List settings = - pipelineSubdirectory.toFile().exists() - ? Files.list(pipelineSubdirectory) - .filter(p -> p.toFile().isFile()) - .map( - p -> { - var relativizedFilePath = - configDirectoryFile - .toPath() - .toAbsolutePath() - .relativize(p) - .toString(); - try { - return JacksonUtils.deserialize(p, CVPipelineSettings.class); - } catch (JsonProcessingException e) { - logger.error("Exception while deserializing " + relativizedFilePath, e); - } catch (IOException e) { - logger.warn( - "Could not load pipeline at " - + relativizedFilePath - + "! Skipping..."); - } - return null; - }) - .filter(Objects::nonNull) - .collect(Collectors.toList()) - : Collections.emptyList(); - - loadedConfig.driveModeSettings = driverMode; - loadedConfig.addPipelineSettings(settings); - - loadedConfigurations.put(subdir.toFile().getName(), loadedConfig); - } - } catch (IOException e) { - logger.error("Error loading camera configs!", e); - } - return loadedConfigurations; + translateLegacyIfPresent(this.configDirectoryFile.toPath()); + m_provider.load(); } public void addCameraConfigurations(List sources) { @@ -394,31 +241,16 @@ public class ConfigManager { return imgFilePath.toPath(); } - public Path getHardwareConfigFile() { - return this.hardwareConfigFile.toPath(); - } - - public Path getHardwareSettingsFile() { - return this.hardwareSettingsFile.toPath(); - } - - public Path getNetworkConfigFile() { - return this.networkConfigFile.toPath(); - } - public void saveUploadedHardwareConfig(Path uploadPath) { - FileUtils.deleteFile(this.getHardwareConfigFile()); - FileUtils.copyFile(uploadPath, this.getHardwareConfigFile()); + m_provider.saveUploadedHardwareConfig(uploadPath); } public void saveUploadedHardwareSettings(Path uploadPath) { - FileUtils.deleteFile(this.getHardwareSettingsFile()); - FileUtils.copyFile(uploadPath, this.getHardwareSettingsFile()); + m_provider.saveUploadedHardwareSettings(uploadPath); } public void saveUploadedNetworkConfig(Path uploadPath) { - FileUtils.deleteFile(this.getNetworkConfigFile()); - FileUtils.copyFile(uploadPath, this.getNetworkConfigFile()); + m_provider.saveUploadedNetworkConfig(uploadPath); } public void requestSave() { @@ -426,6 +258,14 @@ public class ConfigManager { saveRequestTimestamp = System.currentTimeMillis(); } + public void unloadCameraConfigs() { + this.getConfig().getCameraConfigurations().clear(); + } + + public void saveToDisk() { + m_provider.saveToDisk(); + } + private void saveAndWriteTask() { // Only save if 1 second has past since the request was made while (!Thread.currentThread().isInterrupted()) { @@ -442,8 +282,4 @@ public class ConfigManager { } } } - - public void unloadCameraConfigs() { - this.config.getCameraConfigurations().clear(); - } } diff --git a/photon-core/src/main/java/org/photonvision/common/configuration/ConfigProvider.java b/photon-core/src/main/java/org/photonvision/common/configuration/ConfigProvider.java new file mode 100644 index 000000000..f5078d525 --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/common/configuration/ConfigProvider.java @@ -0,0 +1,38 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.common.configuration; + +import java.nio.file.Path; + +public abstract class ConfigProvider { + private PhotonConfiguration config; + + abstract void load(); + + abstract void saveToDisk(); + + PhotonConfiguration getConfig() { + return config; + } + + public abstract void saveUploadedHardwareConfig(Path uploadPath); + + public abstract void saveUploadedHardwareSettings(Path uploadPath); + + public abstract void saveUploadedNetworkConfig(Path uploadPath); +} diff --git a/photon-core/src/main/java/org/photonvision/common/configuration/HardwareConfig.java b/photon-core/src/main/java/org/photonvision/common/configuration/HardwareConfig.java index 8ec2a21db..2c8323f15 100644 --- a/photon-core/src/main/java/org/photonvision/common/configuration/HardwareConfig.java +++ b/photon-core/src/main/java/org/photonvision/common/configuration/HardwareConfig.java @@ -150,4 +150,53 @@ public class HardwareConfig { || gpuMemUsageCommand != "" || diskUsageCommand != ""; } + + @Override + public String toString() { + return "HardwareConfig [deviceName=" + + deviceName + + ", deviceLogoPath=" + + deviceLogoPath + + ", supportURL=" + + supportURL + + ", ledPins=" + + ledPins + + ", ledSetCommand=" + + ledSetCommand + + ", ledsCanDim=" + + ledsCanDim + + ", ledBrightnessRange=" + + ledBrightnessRange + + ", ledDimCommand=" + + ledDimCommand + + ", ledBlinkCommand=" + + ledBlinkCommand + + ", statusRGBPins=" + + statusRGBPins + + ", cpuTempCommand=" + + cpuTempCommand + + ", cpuMemoryCommand=" + + cpuMemoryCommand + + ", cpuUtilCommand=" + + cpuUtilCommand + + ", cpuThrottleReasonCmd=" + + cpuThrottleReasonCmd + + ", cpuUptimeCommand=" + + cpuUptimeCommand + + ", gpuMemoryCommand=" + + gpuMemoryCommand + + ", ramUtilCommand=" + + ramUtilCommand + + ", gpuMemUsageCommand=" + + gpuMemUsageCommand + + ", diskUsageCommand=" + + diskUsageCommand + + ", restartHardwareCommand=" + + restartHardwareCommand + + ", vendorFOV=" + + vendorFOV + + ", blacklistedResIndices=" + + blacklistedResIndices + + "]"; + } } diff --git a/photon-core/src/main/java/org/photonvision/common/configuration/HardwareSettings.java b/photon-core/src/main/java/org/photonvision/common/configuration/HardwareSettings.java index f5578bb74..3ef424dfe 100644 --- a/photon-core/src/main/java/org/photonvision/common/configuration/HardwareSettings.java +++ b/photon-core/src/main/java/org/photonvision/common/configuration/HardwareSettings.java @@ -19,4 +19,9 @@ package org.photonvision.common.configuration; public class HardwareSettings { public int ledBrightnessPercentage = 100; + + @Override + public String toString() { + return "HardwareSettings [ledBrightnessPercentage=" + ledBrightnessPercentage + "]"; + } } diff --git a/photon-core/src/main/java/org/photonvision/common/configuration/LegacyConfigProvider.java b/photon-core/src/main/java/org/photonvision/common/configuration/LegacyConfigProvider.java new file mode 100644 index 000000000..2ce8c0dbf --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/common/configuration/LegacyConfigProvider.java @@ -0,0 +1,443 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.common.configuration; + +import com.fasterxml.jackson.core.JsonProcessingException; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.time.temporal.TemporalAccessor; +import java.util.*; +import java.util.stream.Collectors; +import org.photonvision.common.logging.LogGroup; +import org.photonvision.common.logging.Logger; +import org.photonvision.common.util.file.FileUtils; +import org.photonvision.common.util.file.JacksonUtils; +import org.photonvision.vision.pipeline.CVPipelineSettings; +import org.photonvision.vision.pipeline.DriverModePipelineSettings; +import org.photonvision.vision.processes.VisionSource; +import org.zeroturnaround.zip.ZipUtil; + +class LegacyConfigProvider extends ConfigProvider { + private static final Logger logger = new Logger(LegacyConfigProvider.class, LogGroup.General); + + public static final String HW_CFG_FNAME = "hardwareConfig.json"; + public static final String HW_SET_FNAME = "hardwareSettings.json"; + public static final String NET_SET_FNAME = "networkSettings.json"; + + private PhotonConfiguration config; + private final File hardwareConfigFile; + private final File hardwareSettingsFile; + private final File networkConfigFile; + private final File camerasFolder; + + final File configDirectoryFile; + + private long saveRequestTimestamp = -1; + private Thread settingsSaveThread; + + public static void saveUploadedSettingsZip(File uploadPath) { + var folderPath = Path.of(System.getProperty("java.io.tmpdir"), "photonvision").toFile(); + folderPath.mkdirs(); + ZipUtil.unpack(uploadPath, folderPath); + FileUtils.deleteDirectory(getRootFolder()); + try { + org.apache.commons.io.FileUtils.copyDirectory(folderPath, getRootFolder().toFile()); + logger.info("Copied settings successfully!"); + } catch (IOException e) { + logger.error("Exception copying uploaded settings!", e); + return; + } + } + + public PhotonConfiguration getConfig() { + return config; + } + + private static Path getRootFolder() { + return Path.of("photonvision_config"); + } + + protected LegacyConfigProvider(Path configDirectoryFile) { + this.configDirectoryFile = new File(configDirectoryFile.toUri()); + this.hardwareConfigFile = + new File(Path.of(configDirectoryFile.toString(), HW_CFG_FNAME).toUri()); + this.hardwareSettingsFile = + new File(Path.of(configDirectoryFile.toString(), HW_SET_FNAME).toUri()); + this.networkConfigFile = + new File(Path.of(configDirectoryFile.toString(), NET_SET_FNAME).toUri()); + this.camerasFolder = new File(Path.of(configDirectoryFile.toString(), "cameras").toUri()); + + settingsSaveThread = new Thread(this::saveAndWriteTask); + settingsSaveThread.start(); + } + + @Override + public void load() { + logger.info("Loading settings..."); + if (!configDirectoryFile.exists()) { + if (configDirectoryFile.mkdirs()) { + logger.debug("Root config folder did not exist. Created!"); + } else { + logger.error("Failed to create root config folder!"); + } + } + if (!configDirectoryFile.canWrite()) { + logger.debug("Making root dir writeable..."); + try { + var success = configDirectoryFile.setWritable(true); + if (success) logger.debug("Set root dir writeable!"); + else logger.error("Could not make root dir writeable!"); + } catch (SecurityException e) { + logger.error("Could not make root dir writeable!", e); + } + } + + HardwareConfig hardwareConfig; + HardwareSettings hardwareSettings; + NetworkConfig networkConfig; + + if (hardwareConfigFile.exists()) { + try { + hardwareConfig = + JacksonUtils.deserialize(hardwareConfigFile.toPath(), HardwareConfig.class); + if (hardwareConfig == null) { + logger.error("Could not deserialize hardware config! Loading defaults"); + hardwareConfig = new HardwareConfig(); + } + } catch (IOException e) { + logger.error("Could not deserialize hardware config! Loading defaults"); + hardwareConfig = new HardwareConfig(); + } + } else { + logger.info("Hardware config does not exist! Loading defaults"); + hardwareConfig = new HardwareConfig(); + } + + if (hardwareSettingsFile.exists()) { + try { + hardwareSettings = + JacksonUtils.deserialize(hardwareSettingsFile.toPath(), HardwareSettings.class); + if (hardwareSettings == null) { + logger.error("Could not deserialize hardware settings! Loading defaults"); + hardwareSettings = new HardwareSettings(); + } + } catch (IOException e) { + logger.error("Could not deserialize hardware settings! Loading defaults"); + hardwareSettings = new HardwareSettings(); + } + } else { + logger.info("Hardware settings does not exist! Loading defaults"); + hardwareSettings = new HardwareSettings(); + } + + if (networkConfigFile.exists()) { + try { + networkConfig = JacksonUtils.deserialize(networkConfigFile.toPath(), NetworkConfig.class); + if (networkConfig == null) { + logger.error("Could not deserialize network config! Loading defaults"); + networkConfig = new NetworkConfig(); + } + } catch (IOException e) { + logger.error("Could not deserialize network config! Loading defaults"); + networkConfig = new NetworkConfig(); + } + } else { + logger.info("Network config file does not exist! Loading defaults"); + networkConfig = new NetworkConfig(); + } + + if (!camerasFolder.exists()) { + if (camerasFolder.mkdirs()) { + logger.debug("Cameras config folder did not exist. Created!"); + } else { + logger.error("Failed to create cameras config folder!"); + } + } + + HashMap cameraConfigurations = loadCameraConfigs(); + + this.config = + new PhotonConfiguration( + hardwareConfig, hardwareSettings, networkConfig, cameraConfigurations); + } + + @Override + public void saveToDisk() { + // Delete old configs + FileUtils.deleteDirectory(camerasFolder.toPath()); + + try { + JacksonUtils.serialize(networkConfigFile.toPath(), config.getNetworkConfig()); + } catch (IOException e) { + logger.error("Could not save network config!", e); + } + try { + JacksonUtils.serialize(hardwareSettingsFile.toPath(), config.getHardwareSettings()); + } catch (IOException e) { + logger.error("Could not save hardware config!", e); + } + + // save all of our cameras + var cameraConfigMap = config.getCameraConfigurations(); + for (var subdirName : cameraConfigMap.keySet()) { + var camConfig = cameraConfigMap.get(subdirName); + var subdir = Path.of(camerasFolder.toPath().toString(), subdirName); + + if (!subdir.toFile().exists()) { + // TODO: check for error + subdir.toFile().mkdirs(); + } + + try { + JacksonUtils.serialize(Path.of(subdir.toString(), "config.json"), camConfig); + } catch (IOException e) { + logger.error("Could not save config.json for " + subdir, e); + } + + try { + JacksonUtils.serialize( + Path.of(subdir.toString(), "drivermode.json"), camConfig.driveModeSettings); + } catch (IOException e) { + logger.error("Could not save drivermode.json for " + subdir, e); + } + + for (var pipe : camConfig.pipelineSettings) { + var pipePath = Path.of(subdir.toString(), "pipelines", pipe.pipelineNickname + ".json"); + + if (!pipePath.getParent().toFile().exists()) { + // TODO: check for error + pipePath.getParent().toFile().mkdirs(); + } + + try { + JacksonUtils.serialize(pipePath, pipe); + } catch (IOException e) { + logger.error("Could not save " + pipe.pipelineNickname + ".json!", e); + } + } + } + logger.info("Settings saved!"); + } + + private HashMap loadCameraConfigs() { + HashMap loadedConfigurations = new HashMap<>(); + try { + var subdirectories = + Files.list(camerasFolder.toPath()) + .filter(f -> f.toFile().isDirectory()) + .collect(Collectors.toList()); + + for (var subdir : subdirectories) { + var cameraConfigPath = Path.of(subdir.toString(), "config.json"); + CameraConfiguration loadedConfig = null; + try { + loadedConfig = + JacksonUtils.deserialize( + cameraConfigPath.toAbsolutePath(), CameraConfiguration.class); + } catch (JsonProcessingException e) { + logger.error("Camera config deserialization failed!", e); + e.printStackTrace(); + } + if (loadedConfig == null) { // If the file could not be deserialized + logger.warn("Could not load camera " + subdir + "'s config.json! Loading " + "default"); + continue; // TODO how do we later try to load this camera if it gets reconnected? + } + + // At this point we have only loaded the base stuff + // We still need to deserialize pipelines, as well as + // driver mode settings + var driverModeFile = Path.of(subdir.toString(), "drivermode.json"); + DriverModePipelineSettings driverMode; + try { + driverMode = + JacksonUtils.deserialize( + driverModeFile.toAbsolutePath(), DriverModePipelineSettings.class); + } catch (JsonProcessingException e) { + logger.error("Could not deserialize drivermode.json! Loading defaults"); + logger.debug(Arrays.toString(e.getStackTrace())); + driverMode = new DriverModePipelineSettings(); + } + if (driverMode == null) { + logger.warn( + "Could not load camera " + subdir + "'s drivermode.json! Loading" + " default"); + driverMode = new DriverModePipelineSettings(); + } + + // Load pipelines by mapping the files within the pipelines subdir + // to their deserialized equivalents + var pipelineSubdirectory = Path.of(subdir.toString(), "pipelines"); + List settings = + pipelineSubdirectory.toFile().exists() + ? Files.list(pipelineSubdirectory) + .filter(p -> p.toFile().isFile()) + .map( + p -> { + var relativizedFilePath = + configDirectoryFile + .toPath() + .toAbsolutePath() + .relativize(p) + .toString(); + try { + return JacksonUtils.deserialize(p, CVPipelineSettings.class); + } catch (JsonProcessingException e) { + logger.error("Exception while deserializing " + relativizedFilePath, e); + } catch (IOException e) { + logger.warn( + "Could not load pipeline at " + + relativizedFilePath + + "! Skipping..."); + } + return null; + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()) + : Collections.emptyList(); + + loadedConfig.driveModeSettings = driverMode; + loadedConfig.addPipelineSettings(settings); + + loadedConfigurations.put(subdir.toFile().getName(), loadedConfig); + } + } catch (IOException e) { + logger.error("Error loading camera configs!", e); + } + return loadedConfigurations; + } + + public void addCameraConfigurations(List sources) { + getConfig().addCameraConfigs(sources); + requestSave(); + } + + public void saveModule(CameraConfiguration config, String uniqueName) { + getConfig().addCameraConfig(uniqueName, config); + requestSave(); + } + + public File getSettingsFolderAsZip() { + File out = Path.of(System.getProperty("java.io.tmpdir"), "photonvision-settings.zip").toFile(); + try { + ZipUtil.pack(configDirectoryFile, out); + } catch (Exception e) { + e.printStackTrace(); + } + return out; + } + + public void setNetworkSettings(NetworkConfig networkConfig) { + getConfig().setNetworkConfig(networkConfig); + requestSave(); + } + + public Path getLogsDir() { + return Path.of(configDirectoryFile.toString(), "logs"); + } + + public Path getCalibDir() { + return Path.of(configDirectoryFile.toString(), "calibImgs"); + } + + public static final String LOG_PREFIX = "photonvision-"; + public static final String LOG_EXT = ".log"; + public static final String LOG_DATE_TIME_FORMAT = "yyyy-M-d_hh-mm-ss"; + + public String taToLogFname(TemporalAccessor date) { + var dateString = DateTimeFormatter.ofPattern(LOG_DATE_TIME_FORMAT).format(date); + return LOG_PREFIX + dateString + LOG_EXT; + } + + public Date logFnameToDate(String fname) throws ParseException { + // Strip away known unneded portions of the log file name + fname = fname.replace(LOG_PREFIX, "").replace(LOG_EXT, ""); + DateFormat format = new SimpleDateFormat(LOG_DATE_TIME_FORMAT); + return format.parse(fname); + } + + public Path getLogPath() { + var logFile = Path.of(this.getLogsDir().toString(), taToLogFname(LocalDateTime.now())).toFile(); + if (!logFile.getParentFile().exists()) logFile.getParentFile().mkdirs(); + return logFile.toPath(); + } + + public Path getImageSavePath() { + var imgFilePath = Path.of(configDirectoryFile.toString(), "imgSaves").toFile(); + if (!imgFilePath.exists()) imgFilePath.mkdirs(); + return imgFilePath.toPath(); + } + + public Path getHardwareConfigFile() { + return this.hardwareConfigFile.toPath(); + } + + public Path getHardwareSettingsFile() { + return this.hardwareSettingsFile.toPath(); + } + + public Path getNetworkConfigFile() { + return this.networkConfigFile.toPath(); + } + + public void saveUploadedHardwareConfig(Path uploadPath) { + FileUtils.deleteFile(this.getHardwareConfigFile()); + FileUtils.copyFile(uploadPath, this.getHardwareConfigFile()); + } + + public void saveUploadedHardwareSettings(Path uploadPath) { + FileUtils.deleteFile(this.getHardwareSettingsFile()); + FileUtils.copyFile(uploadPath, this.getHardwareSettingsFile()); + } + + public void saveUploadedNetworkConfig(Path uploadPath) { + FileUtils.deleteFile(this.getNetworkConfigFile()); + FileUtils.copyFile(uploadPath, this.getNetworkConfigFile()); + } + + public void requestSave() { + logger.trace("Requesting save..."); + saveRequestTimestamp = System.currentTimeMillis(); + } + + private void saveAndWriteTask() { + // Only save if 1 second has past since the request was made + while (!Thread.currentThread().isInterrupted()) { + if (saveRequestTimestamp > 0 && (System.currentTimeMillis() - saveRequestTimestamp) > 1000L) { + saveRequestTimestamp = -1; + logger.debug("Saving to disk..."); + saveToDisk(); + } + + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + logger.error("Exception waiting for settings semaphore", e); + } + } + } + + public void unloadCameraConfigs() { + this.config.getCameraConfigurations().clear(); + } +} diff --git a/photon-core/src/main/java/org/photonvision/common/configuration/NetworkConfig.java b/photon-core/src/main/java/org/photonvision/common/configuration/NetworkConfig.java index 0ff570abc..014147dce 100644 --- a/photon-core/src/main/java/org/photonvision/common/configuration/NetworkConfig.java +++ b/photon-core/src/main/java/org/photonvision/common/configuration/NetworkConfig.java @@ -31,6 +31,7 @@ import org.photonvision.common.networking.NetworkMode; import org.photonvision.common.util.file.JacksonUtils; public class NetworkConfig { + // Can be a integer team number, or a IP address public String ntServerAddress = "0"; public NetworkMode connectionType = NetworkMode.DHCP; public String staticIp = ""; @@ -105,4 +106,29 @@ public class NetworkConfig { public void setShouldManage(boolean shouldManage) { this.shouldManage = shouldManage || Platform.isLinux(); } + + @Override + public String toString() { + return "NetworkConfig [serverAddr=" + + ntServerAddress + + ", connectionType=" + + connectionType + + ", staticIp=" + + staticIp + + ", hostname=" + + hostname + + ", runNTServer=" + + runNTServer + + ", networkManagerIface=" + + networkManagerIface + + ", physicalInterface=" + + physicalInterface + + ", setStaticCommand=" + + setStaticCommand + + ", setDHCPcommand=" + + setDHCPcommand + + ", shouldManage=" + + shouldManage + + "]"; + } } diff --git a/photon-core/src/main/java/org/photonvision/common/configuration/PhotonConfiguration.java b/photon-core/src/main/java/org/photonvision/common/configuration/PhotonConfiguration.java index ba3e75181..1b5431571 100644 --- a/photon-core/src/main/java/org/photonvision/common/configuration/PhotonConfiguration.java +++ b/photon-core/src/main/java/org/photonvision/common/configuration/PhotonConfiguration.java @@ -140,4 +140,17 @@ public class PhotonConfiguration { public List> calibrations; public boolean isFovConfigurable = true; } + + @Override + public String toString() { + return "PhotonConfiguration [hardwareConfig=" + + hardwareConfig + + ", hardwareSettings=" + + hardwareSettings + + ", networkConfig=" + + networkConfig + + ", cameraConfigurations=" + + cameraConfigurations + + "]"; + } } diff --git a/photon-core/src/main/java/org/photonvision/common/configuration/SqlConfigProvider.java b/photon-core/src/main/java/org/photonvision/common/configuration/SqlConfigProvider.java new file mode 100644 index 000000000..3de6035c2 --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/common/configuration/SqlConfigProvider.java @@ -0,0 +1,475 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.common.configuration; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import org.photonvision.common.logging.LogGroup; +import org.photonvision.common.logging.Logger; +import org.photonvision.common.util.file.JacksonUtils; +import org.photonvision.vision.pipeline.CVPipelineSettings; +import org.photonvision.vision.pipeline.DriverModePipelineSettings; + +/** + * Saves settings in a SQLite database file (called photon.sqlite). + * + *

Within this database we have a cameras database, which has one row per camera, and holds: + * unique_name, config_json, drivermode_json, pipeline_jsons. + * + *

Global has one row per global config file (like hardware settings and network settings) + */ +public class SqlConfigProvider extends ConfigProvider { + private final Logger logger = new Logger(SqlConfigProvider.class, LogGroup.Config); + + static class TableKeys { + static final String CAM_UNIQUE_NAME = "unique_name"; + static final String CONFIG_JSON = "config_json"; + static final String DRIVERMODE_JSON = "drivermode_json"; + static final String PIPELINE_JSONS = "pipeline_jsons"; + + static final String NETWORK_CONFIG = "networkConfig"; + static final String HARDWARE_CONFIG = "hardwareConfig"; + static final String HARDWARE_SETTINGS = "hardwareSettings"; + } + + private static final String dbName = "photon.sqlite"; + private final String dbPath; + + private PhotonConfiguration config; + private final Object m_mutex = new Object(); + private final File rootFolder; + + public SqlConfigProvider(Path rootFolder) { + this.rootFolder = rootFolder.toFile(); + dbPath = Path.of(rootFolder.toString(), dbName).toAbsolutePath().toString(); + logger.debug("Using database " + dbPath); + initDatabase(); + } + + public PhotonConfiguration getConfig() { + if (config == null) { + logger.warn("CONFIG IS NULL!"); + } + return config; + } + + private Connection createConn() { + String url = "jdbc:sqlite:" + dbPath; + + try { + var conn = DriverManager.getConnection(url); + conn.setAutoCommit(false); + return conn; + } catch (SQLException e) { + logger.error("Error creating connection", e); + return null; + } + } + + private void tryCommit(Connection conn) { + try { + conn.commit(); + } catch (SQLException e) { + logger.error("Err committing changes: ", e); + try { + conn.rollback(); + } catch (SQLException e1) { + logger.error("Err rolling back changes: ", e); + } + } + } + + private void initDatabase() { + // Make sure root dir exists + + if (!rootFolder.exists()) { + if (rootFolder.mkdirs()) { + logger.debug("Root config folder did not exist. Created!"); + } else { + logger.error("Failed to create root config folder!"); + } + } + + Connection conn = null; + Statement createGlobalTableStatement = null, createCameraTableStatement = null; + try { + conn = createConn(); + if (conn == null) { + logger.error("No connection, cannot init db"); + return; + } + + // Create global settings table. Just a dumb table with list of jsons and their + // name + try { + createGlobalTableStatement = conn.createStatement(); + String sql = + "CREATE TABLE IF NOT EXISTS global (\n" + + " filename TINYTEXT PRIMARY KEY,\n" + + " contents mediumtext NOT NULL\n" + + ");"; + createGlobalTableStatement.execute(sql); + } catch (SQLException e) { + logger.error("Err creating global table", e); + } + + // Create cameras table, key is the camera unique name + try { + createCameraTableStatement = conn.createStatement(); + var sql = + "CREATE TABLE IF NOT EXISTS cameras (\n" + + " unique_name TINYTEXT PRIMARY KEY,\n" + + " config_json text NOT NULL,\n" + + " drivermode_json text NOT NULL,\n" + + " pipeline_jsons mediumtext NOT NULL\n" + + ");"; + createCameraTableStatement.execute(sql); + } catch (SQLException e) { + logger.error("Err creating cameras table", e); + } + + this.tryCommit(conn); + } finally { + try { + if (createGlobalTableStatement != null) createGlobalTableStatement.close(); + if (createCameraTableStatement != null) createCameraTableStatement.close(); + if (conn != null) conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + } + + @Override + public void saveToDisk() { + logger.debug("Saving to disk"); + var conn = createConn(); + if (conn == null) return; + synchronized (m_mutex) { + if (config == null) { + logger.error("Config null! Cannot save"); + return; + } + + saveCameras(conn); + saveGlobal(conn); + tryCommit(conn); + + try { + conn.close(); + } catch (SQLException e) { + logger.error("SQL Err closing connection while saving to disk: ", e); + } + } + + logger.info("Settings saved!"); + } + + @Override + public void load() { + logger.debug("Loading config..."); + var conn = createConn(); + if (conn == null) return; + + synchronized (m_mutex) { + HardwareConfig hardwareConfig; + HardwareSettings hardwareSettings; + NetworkConfig networkConfig; + + try { + hardwareConfig = + JacksonUtils.deserialize( + getOneConfigFile(conn, TableKeys.HARDWARE_CONFIG), HardwareConfig.class); + } catch (IOException e) { + logger.error("Could not deserialize hardware config! Loading defaults"); + hardwareConfig = new HardwareConfig(); + } + + try { + hardwareSettings = + JacksonUtils.deserialize( + getOneConfigFile(conn, TableKeys.HARDWARE_SETTINGS), HardwareSettings.class); + } catch (IOException e) { + logger.error("Could not deserialize hardware settings! Loading defaults"); + hardwareSettings = new HardwareSettings(); + } + + try { + networkConfig = + JacksonUtils.deserialize( + getOneConfigFile(conn, TableKeys.NETWORK_CONFIG), NetworkConfig.class); + } catch (IOException e) { + logger.error("Could not deserialize network config! Loading defaults"); + networkConfig = new NetworkConfig(); + } + + var cams = loadCameraConfigs(conn); + + try { + conn.close(); + } catch (SQLException e) { + logger.error("SQL Err closing connection while loading: ", e); + } + + this.config = new PhotonConfiguration(hardwareConfig, hardwareSettings, networkConfig, cams); + } + } + + private String getOneConfigFile(Connection conn, String filename) { + // Query every single row of the global settings db + PreparedStatement query = null; + try { + query = + conn.prepareStatement("SELECT contents FROM global where filename=\"" + filename + "\""); + + var result = query.executeQuery(); + + while (result.next()) { + var contents = result.getString("contents"); + return contents; + } + } catch (SQLException e) { + logger.error("SQL Err getting file " + filename, e); + } finally { + try { + if (query != null) query.close(); + } catch (SQLException e) { + logger.error("SQL Err closing config file query " + filename, e); + } + } + + return ""; + } + + private void saveCameras(Connection conn) { + try { + // Replace this camera's row with the new settings + var sqlString = + "REPLACE INTO cameras (unique_name, config_json, drivermode_json, pipeline_jsons) VALUES " + + "(?,?,?,?);"; + + for (var c : config.getCameraConfigurations().entrySet()) { + PreparedStatement statement = conn.prepareStatement(sqlString); + + var config = c.getValue(); + statement.setString(1, c.getKey()); + statement.setString(2, JacksonUtils.serializeToString(config)); + statement.setString(3, JacksonUtils.serializeToString(config.driveModeSettings)); + + // Serializing a list of abstract classes sucks. Instead, make it into a array + // of strings, which we can later unpack back into individual settings + List settings = + config.pipelineSettings.stream() + .map( + it -> { + try { + return JacksonUtils.serializeToString(it); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + statement.setString(4, JacksonUtils.serializeToString(settings)); + + statement.executeUpdate(); + } + } catch (SQLException | IOException e) { + logger.error("Err saving cameras", e); + try { + conn.rollback(); + } catch (SQLException e1) { + logger.error("Err rolling back changes: ", e); + } + } + } + + private void addFile(PreparedStatement ps, String key, String value) throws SQLException { + ps.setString(1, key); + ps.setString(2, value); + } + + private void saveGlobal(Connection conn) { + PreparedStatement statement1 = null; + PreparedStatement statement2 = null; + PreparedStatement statement3 = null; + try { + // Replace this camera's row with the new settings + var sqlString = "REPLACE INTO global (filename, contents) VALUES " + "(?,?);"; + + statement1 = conn.prepareStatement(sqlString); + addFile( + statement1, + TableKeys.HARDWARE_SETTINGS, + JacksonUtils.serializeToString(config.getHardwareSettings())); + statement1.executeUpdate(); + + statement2 = conn.prepareStatement(sqlString); + addFile( + statement2, + TableKeys.NETWORK_CONFIG, + JacksonUtils.serializeToString(config.getNetworkConfig())); + statement2.executeUpdate(); + statement2.close(); + + statement3 = conn.prepareStatement(sqlString); + addFile( + statement3, + TableKeys.HARDWARE_CONFIG, + JacksonUtils.serializeToString(config.getHardwareConfig())); + statement3.executeUpdate(); + statement3.close(); + + } catch (SQLException | IOException e) { + logger.error("Err saving global", e); + try { + conn.rollback(); + } catch (SQLException e1) { + logger.error("Err rolling back changes: ", e); + } + } finally { + try { + if (statement1 != null) statement1.close(); + if (statement2 != null) statement2.close(); + if (statement3 != null) statement3.close(); + } catch (SQLException e) { + logger.error("SQL Err closing global settings query ", e); + } + } + } + + private void saveOneFile(String fname, Path path) { + Connection conn = null; + PreparedStatement statement1 = null; + try { + conn = createConn(); + if (conn == null) { + return; + } + // Replace this camera's row with the new settings + var sqlString = "REPLACE INTO global (filename, contents) VALUES " + "(?,?);"; + + statement1 = conn.prepareStatement(sqlString); + addFile(statement1, fname, Files.readString(path)); + statement1.executeUpdate(); + + conn.commit(); + } catch (SQLException | IOException e) { + logger.error("Err saving global", e); + try { + conn.rollback(); + } catch (SQLException e1) { + logger.error("Err rolling back changes: ", e); + } + } finally { + try { + if (statement1 != null) statement1.close(); + conn.close(); + } catch (SQLException e) { + logger.error("SQL Err saving file " + fname, e); + } + } + } + + @Override + public void saveUploadedHardwareConfig(Path uploadPath) { + saveOneFile(TableKeys.HARDWARE_CONFIG, uploadPath); + } + + @Override + public void saveUploadedHardwareSettings(Path uploadPath) { + saveOneFile(TableKeys.HARDWARE_SETTINGS, uploadPath); + } + + @Override + public void saveUploadedNetworkConfig(Path uploadPath) { + saveOneFile(TableKeys.NETWORK_CONFIG, uploadPath); + } + + private HashMap loadCameraConfigs(Connection conn) { + HashMap loadedConfigurations = new HashMap<>(); + + // Querry every single row of the cameras db + PreparedStatement query = null; + try { + query = + conn.prepareStatement( + String.format( + "SELECT %s, %s, %s, %s FROM cameras", + TableKeys.CAM_UNIQUE_NAME, + TableKeys.CONFIG_JSON, + TableKeys.DRIVERMODE_JSON, + TableKeys.PIPELINE_JSONS)); + + var result = query.executeQuery(); + + // Iterate over every row/"camera" in the table + while (result.next()) { + List dummyList = new ArrayList<>(); + + var uniqueName = result.getString(TableKeys.CAM_UNIQUE_NAME); + var config = + JacksonUtils.deserialize( + result.getString(TableKeys.CONFIG_JSON), CameraConfiguration.class); + var driverMode = + JacksonUtils.deserialize( + result.getString(TableKeys.DRIVERMODE_JSON), DriverModePipelineSettings.class); + List pipelineSettings = + JacksonUtils.deserialize( + result.getString(TableKeys.PIPELINE_JSONS), dummyList.getClass()); + + List loadedSettings = new ArrayList<>(); + for (var str : pipelineSettings) { + if (str instanceof String) { + loadedSettings.add(JacksonUtils.deserialize((String) str, CVPipelineSettings.class)); + } + } + + config.pipelineSettings = loadedSettings; + config.driveModeSettings = driverMode; + loadedConfigurations.put(uniqueName, config); + } + } catch (SQLException | IOException e) { + logger.error("Err loading cameras: ", e); + } finally { + try { + if (query != null) query.close(); + } catch (SQLException e) { + logger.error("SQL Err closing connection while loading cameras ", e); + } + } + return loadedConfigurations; + } + + public void setConfig(PhotonConfiguration config) { + this.config = config; + } +} diff --git a/photon-core/src/main/java/org/photonvision/common/logging/LogGroup.java b/photon-core/src/main/java/org/photonvision/common/logging/LogGroup.java index cfdd91861..50524db63 100644 --- a/photon-core/src/main/java/org/photonvision/common/logging/LogGroup.java +++ b/photon-core/src/main/java/org/photonvision/common/logging/LogGroup.java @@ -22,5 +22,6 @@ public enum LogGroup { WebServer, VisionModule, Data, - General + General, + Config } diff --git a/photon-core/src/main/java/org/photonvision/common/logging/Logger.java b/photon-core/src/main/java/org/photonvision/common/logging/Logger.java index 50d0b8fb5..5972e3c75 100644 --- a/photon-core/src/main/java/org/photonvision/common/logging/Logger.java +++ b/photon-core/src/main/java/org/photonvision/common/logging/Logger.java @@ -102,6 +102,7 @@ public class Logger { levelMap.put(LogGroup.WebServer, LogLevel.INFO); levelMap.put(LogGroup.Data, LogLevel.INFO); levelMap.put(LogGroup.VisionModule, LogLevel.INFO); + levelMap.put(LogGroup.Config, LogLevel.INFO); } static { @@ -240,7 +241,7 @@ public class Logger { * @param t */ public void error(String message, Throwable t) { - log(message, LogLevel.ERROR); + log(message + ": " + t.getMessage(), LogLevel.ERROR); log(convertStackTraceToString(t), LogLevel.ERROR, LogLevel.DEBUG); } diff --git a/photon-core/src/main/java/org/photonvision/common/util/file/FileUtils.java b/photon-core/src/main/java/org/photonvision/common/util/file/FileUtils.java index 23b3056c3..b63ed93a7 100644 --- a/photon-core/src/main/java/org/photonvision/common/util/file/FileUtils.java +++ b/photon-core/src/main/java/org/photonvision/common/util/file/FileUtils.java @@ -42,14 +42,13 @@ public class FileUtils { public static void deleteDirectory(Path path) { try { - // create a stream var files = Files.walk(path); // delete directory including files and sub-folders files .sorted(Comparator.reverseOrder()) .map(Path::toFile) - .filter(File::isFile) + // .filter(File::isFile) // we want to delete directories and sub-dirs, too .forEach((var file) -> deleteFile(file.toPath())); // close the stream diff --git a/photon-core/src/main/java/org/photonvision/common/util/file/JacksonUtils.java b/photon-core/src/main/java/org/photonvision/common/util/file/JacksonUtils.java index dd792c497..818a0e51e 100644 --- a/photon-core/src/main/java/org/photonvision/common/util/file/JacksonUtils.java +++ b/photon-core/src/main/java/org/photonvision/common/util/file/JacksonUtils.java @@ -40,6 +40,16 @@ public class JacksonUtils { serialize(path, object, true); } + public static String serializeToString(T object) throws IOException { + PolymorphicTypeValidator ptv = + BasicPolymorphicTypeValidator.builder().allowIfBaseType(object.getClass()).build(); + ObjectMapper objectMapper = + JsonMapper.builder() + .activateDefaultTyping(ptv, ObjectMapper.DefaultTyping.JAVA_LANG_OBJECT) + .build(); + return objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(object); + } + public static void serialize(Path path, T object, boolean forceSync) throws IOException { PolymorphicTypeValidator ptv = BasicPolymorphicTypeValidator.builder().allowIfBaseType(object.getClass()).build(); @@ -51,6 +61,19 @@ public class JacksonUtils { saveJsonString(json, path, forceSync); } + public static T deserialize(String s, Class ref) throws IOException { + PolymorphicTypeValidator ptv = + BasicPolymorphicTypeValidator.builder().allowIfBaseType(ref).build(); + ObjectMapper objectMapper = + JsonMapper.builder() + .configure(JsonReadFeature.ALLOW_JAVA_COMMENTS, true) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + .activateDefaultTyping(ptv, ObjectMapper.DefaultTyping.JAVA_LANG_OBJECT) + .build(); + + return objectMapper.readValue(s, ref); + } + public static T deserialize(Path path, Class ref) throws IOException { PolymorphicTypeValidator ptv = BasicPolymorphicTypeValidator.builder().allowIfBaseType(ref).build(); diff --git a/photon-core/src/test/java/org/photonvision/common/configuration/ConfigTest.java b/photon-core/src/test/java/org/photonvision/common/configuration/ConfigTest.java index e9a6e391d..8aa145600 100644 --- a/photon-core/src/test/java/org/photonvision/common/configuration/ConfigTest.java +++ b/photon-core/src/test/java/org/photonvision/common/configuration/ConfigTest.java @@ -46,7 +46,8 @@ public class ConfigTest { @BeforeAll public static void init() { TestUtils.loadLibraries(); - configMgr = new ConfigManager(Path.of("testconfigdir")); + var path = Path.of("testconfigdir"); + configMgr = new ConfigManager(path, new LegacyConfigProvider(path)); configMgr.load(); Logger.setLevel(LogGroup.General, LogLevel.TRACE); diff --git a/photon-core/src/test/java/org/photonvision/common/configuration/NetworkConfigTest.java b/photon-core/src/test/java/org/photonvision/common/configuration/NetworkConfigTest.java index 701ba3cf8..7c9d6d6f4 100644 --- a/photon-core/src/test/java/org/photonvision/common/configuration/NetworkConfigTest.java +++ b/photon-core/src/test/java/org/photonvision/common/configuration/NetworkConfigTest.java @@ -37,14 +37,14 @@ public class NetworkConfigTest { @Test public void testDeserializeTeamNumberOrNtServerAddress() { { - ConfigManager configMgr = - new ConfigManager(Path.of("test-resources/network-old-team-number")); + var folder = Path.of("test-resources/network-old-team-number"); + var configMgr = new ConfigManager(folder, new LegacyConfigProvider(folder)); configMgr.load(); Assertions.assertEquals("9999", configMgr.getConfig().getNetworkConfig().ntServerAddress); } { - ConfigManager configMgr = - new ConfigManager(Path.of("test-resources/network-new-team-number")); + var folder = Path.of("test-resources/network-new-team-number"); + var configMgr = new ConfigManager(folder, new LegacyConfigProvider(folder)); configMgr.load(); Assertions.assertEquals("9999", configMgr.getConfig().getNetworkConfig().ntServerAddress); } diff --git a/photon-core/src/test/java/org/photonvision/common/configuration/SQLConfigTest.java b/photon-core/src/test/java/org/photonvision/common/configuration/SQLConfigTest.java new file mode 100644 index 000000000..ad2567bc3 --- /dev/null +++ b/photon-core/src/test/java/org/photonvision/common/configuration/SQLConfigTest.java @@ -0,0 +1,68 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.common.configuration; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.nio.file.Path; +import java.util.List; +import org.junit.jupiter.api.*; +import org.photonvision.common.util.TestUtils; +import org.photonvision.vision.camera.CameraType; +import org.photonvision.vision.pipeline.AprilTagPipelineSettings; +import org.photonvision.vision.pipeline.ColoredShapePipelineSettings; +import org.photonvision.vision.pipeline.ReflectivePipelineSettings; + +public class SQLConfigTest { + @BeforeAll + public static void init() { + TestUtils.loadLibraries(); + } + + @Test + public void testLoad() { + var cfgLoader = new SqlConfigProvider(Path.of("jdbc_test")); + + cfgLoader.load(); + + var testcamcfg = + new CameraConfiguration( + "basename", + "a_unique_name", + "a_nick_name", + 69, + "a/path/idk", + CameraType.UsbCamera, + List.of(), + 0); + testcamcfg.pipelineSettings = + List.of( + new ReflectivePipelineSettings(), + new AprilTagPipelineSettings(), + new ColoredShapePipelineSettings()); + + cfgLoader.getConfig().addCameraConfig(testcamcfg); + cfgLoader.getConfig().getNetworkConfig().ntServerAddress = "5940"; + cfgLoader.saveToDisk(); + + cfgLoader.load(); + System.out.println(cfgLoader.getConfig()); + + assertEquals(cfgLoader.getConfig().getNetworkConfig().ntServerAddress, "5940"); + } +} diff --git a/photon-core/src/test/java/org/photonvision/common/util/LogFileManagementTest.java b/photon-core/src/test/java/org/photonvision/common/util/LogFileManagementTest.java index 126d42683..bfd36a6ae 100644 --- a/photon-core/src/test/java/org/photonvision/common/util/LogFileManagementTest.java +++ b/photon-core/src/test/java/org/photonvision/common/util/LogFileManagementTest.java @@ -33,13 +33,13 @@ import org.photonvision.common.logging.Logger; public class LogFileManagementTest { @Test - public void fileCleanupTest() throws IOException { + public void fileCleanupTest() { // Ensure we instantiate the new log correctly ConfigManager.getInstance(); String testDir = ConfigManager.getInstance().getLogsDir().toString() + "/test"; - Files.createDirectories(Path.of(testDir)); + Assertions.assertDoesNotThrow(() -> Files.createDirectories(Path.of(testDir))); // Create a bunch of log files with dummy contents. for (int fileIdx = 0; fileIdx < Logger.MAX_LOGS_TO_KEEP + 5; fileIdx++) { @@ -71,7 +71,11 @@ public class LogFileManagementTest { // Clean uptest directory org.photonvision.common.util.file.FileUtils.deleteDirectory(Path.of(testDir)); - Files.delete(Path.of(testDir)); + try { + Files.delete(Path.of(testDir)); + } catch (IOException e) { + // it's OK if this fails + } } private int countLogFiles(String testDir) { diff --git a/photon-server/src/main/java/org/photonvision/Main.java b/photon-server/src/main/java/org/photonvision/Main.java index 175ee2703..8d399cdb0 100644 --- a/photon-server/src/main/java/org/photonvision/Main.java +++ b/photon-server/src/main/java/org/photonvision/Main.java @@ -334,6 +334,7 @@ public class Main { Logger.setLevel(LogGroup.WebServer, logLevel); Logger.setLevel(LogGroup.VisionModule, logLevel); Logger.setLevel(LogGroup.Data, logLevel); + Logger.setLevel(LogGroup.Config, logLevel); Logger.setLevel(LogGroup.General, logLevel); logger.info("Logging initialized in debug mode.");