diff --git a/photon-core/src/main/java/org/photonvision/common/configuration/ConfigManager.java b/photon-core/src/main/java/org/photonvision/common/configuration/ConfigManager.java index 6f4371302..d8fe57622 100644 --- a/photon-core/src/main/java/org/photonvision/common/configuration/ConfigManager.java +++ b/photon-core/src/main/java/org/photonvision/common/configuration/ConfigManager.java @@ -39,7 +39,7 @@ import org.photonvision.vision.processes.VisionSource; import org.zeroturnaround.zip.ZipUtil; public class ConfigManager { - private static ConfigManager INSTANCE; + static ConfigManager INSTANCE; public static final String HW_CFG_FNAME = "hardwareConfig.json"; public static final String HW_SET_FNAME = "hardwareSettings.json"; diff --git a/photon-core/src/main/java/org/photonvision/common/configuration/LegacyConfigProvider.java b/photon-core/src/main/java/org/photonvision/common/configuration/LegacyConfigProvider.java index 6feadcab3..28de811a9 100644 --- a/photon-core/src/main/java/org/photonvision/common/configuration/LegacyConfigProvider.java +++ b/photon-core/src/main/java/org/photonvision/common/configuration/LegacyConfigProvider.java @@ -218,7 +218,7 @@ class LegacyConfigProvider extends ConfigProvider { hardwareSettings, networkConfig, atfl, - new NeuralNetworkPropertyManager(), + new NeuralNetworkModelsSettings(), cameraConfigurations); } diff --git a/photon-core/src/main/java/org/photonvision/common/configuration/NeuralNetworkModelManager.java b/photon-core/src/main/java/org/photonvision/common/configuration/NeuralNetworkModelManager.java index 4c8eb722b..9180c6363 100644 --- a/photon-core/src/main/java/org/photonvision/common/configuration/NeuralNetworkModelManager.java +++ b/photon-core/src/main/java/org/photonvision/common/configuration/NeuralNetworkModelManager.java @@ -34,7 +34,7 @@ import java.util.Optional; import java.util.jar.JarEntry; import java.util.jar.JarFile; import java.util.stream.Stream; -import org.photonvision.common.configuration.NeuralNetworkPropertyManager.ModelProperties; +import org.photonvision.common.configuration.NeuralNetworkModelsSettings.ModelProperties; import org.photonvision.common.hardware.Platform; import org.photonvision.common.logging.LogGroup; import org.photonvision.common.logging.Logger; @@ -50,20 +50,20 @@ import org.photonvision.vision.objects.RubikModel; * extracted to the filesystem, it will not be extracted again. * *

Each model must have a corresponding {@link ModelProperties} entry in {@link - * NeuralNetworkPropertyManager}. + * NeuralNetworkModelsSettings}. */ public class NeuralNetworkModelManager { /** Singleton instance of the NeuralNetworkModelManager */ private static NeuralNetworkModelManager INSTANCE; - private final List supportedBackends = new ArrayList<>(); + final List supportedBackends = new ArrayList<>(); /** * This function stores the properties of the shipped object detection models. It is stored as a * function so that it can be dynamic, to adjust for the models directory. */ - private NeuralNetworkPropertyManager getShippedProperties(File modelsDirectory) { - NeuralNetworkPropertyManager nnProps = new NeuralNetworkPropertyManager(); + private NeuralNetworkModelsSettings getShippedProperties(File modelsDirectory) { + NeuralNetworkModelsSettings nnProps = new NeuralNetworkModelsSettings(); LinkedList cocoLabels = new LinkedList( @@ -272,7 +272,7 @@ public class NeuralNetworkModelManager { * *

The first model in the list is the default model. */ - private Map> models; + Map> models; /** * Retrieves the model with the specified name, assuming it is available under a supported @@ -324,10 +324,23 @@ public class NeuralNetworkModelManager { logger.error( "Model properties are null. This could mean the config for model " + path - + " was unable to be found in the database."); - return; + + " was unable to be found in the database. Trying legacy..."); + try { + properties = ModelProperties.createFromFilename(path.getFileName().toString()); + + // At this point this property is not serialized or known to our configuration. add to + // NeuralNetworkModelsSettings + ConfigManager.getInstance() + .getConfig() + .neuralNetworkPropertyManager() + .addModelProperties(properties); + } catch (IllegalArgumentException | IOException e) { + logger.error("Failed to translate legacy model filename to properties: " + path, e); + } } + logger.debug(properties.toString()); + if (!supportedBackends.contains(properties.family())) { logger.warn( "Model " @@ -412,7 +425,7 @@ public class NeuralNetworkModelManager { File modelsDirectory = ConfigManager.getInstance().getModelsDirectory(); // Filter shippedProprties by supportedBackends - NeuralNetworkPropertyManager supportedProperties = new NeuralNetworkPropertyManager(); + NeuralNetworkModelsSettings supportedProperties = new NeuralNetworkModelsSettings(); for (ModelProperties model : getShippedProperties(modelsDirectory).getModels()) { if (supportedBackends.contains(model.family())) { supportedProperties.addModelProperties(model); diff --git a/photon-core/src/main/java/org/photonvision/common/configuration/NeuralNetworkModelsSettings.java b/photon-core/src/main/java/org/photonvision/common/configuration/NeuralNetworkModelsSettings.java new file mode 100644 index 000000000..56e230280 --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/common/configuration/NeuralNetworkModelsSettings.java @@ -0,0 +1,277 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.common.configuration; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.photonvision.common.configuration.NeuralNetworkModelManager.Family; +import org.photonvision.common.configuration.NeuralNetworkModelManager.Version; + +public class NeuralNetworkModelsSettings { + /* + * The properties of the model. This is used to determine which model to load. + * The only families currently supported are RKNN and Rubik (custom .tflite) + */ + public record ModelProperties( + @JsonProperty("modelPath") Path modelPath, + @JsonProperty("nickname") String nickname, + @JsonProperty("labels") List labels, + @JsonProperty("resolutionWidth") int resolutionWidth, + @JsonProperty("resolutionHeight") int resolutionHeight, + @JsonProperty("family") Family family, + @JsonProperty("version") Version version) { + @JsonCreator + public ModelProperties {} + + ModelProperties(ModelProperties other) { + this( + other.modelPath, + other.nickname, + other.labels, // note this does not clone the underlying list + other.resolutionWidth, + other.resolutionHeight, + other.family, + other.version); + } + + // In v2025.3.1, this was single string for the model path. but the first argument + // is now nickname + public ModelProperties(@JsonProperty("nickname") String filename) + throws IllegalArgumentException, IOException { + this(createFromFilename(filename)); + } + + // ============= Migration code from v2025.3.1 =========== + + private static Pattern modelPattern = + Pattern.compile("^([a-zA-Z0-9._]+)-(\\d+)-(\\d+)-(yolov(?:5|8|11)[nsmlx]*)\\.rknn$"); + + static ModelProperties createFromFilename(String modelFileName) + throws IllegalArgumentException, IOException { + // Used to point to default models directory + var model = + ConfigManager.getInstance().getModelsDirectory().toPath().resolve(modelFileName).toFile(); + + // Get the model extension and check if it is supported + String modelExtension = model.getName().substring(model.getName().lastIndexOf('.')); + if (!modelExtension.equals(".rknn")) { + throw new IllegalArgumentException("Model " + modelFileName + " is not a supported format"); + } + + var backend = + Arrays.stream(NeuralNetworkModelManager.Family.values()) + .filter(b -> b.extension().equals(modelExtension)) + .findFirst(); + + if (!backend.isPresent()) { + throw new IllegalArgumentException("Model " + modelFileName + " cannot find backend"); + } + + String labelFile = model.getAbsolutePath().replace(backend.get().extension(), "-labels.txt"); + List labels = Files.readAllLines(Paths.get(labelFile)); + + String[] parts = parseRKNNName(modelFileName); + var version = getModelVersion(parts[3]); + int width = Integer.parseInt(parts[1]); + int height = Integer.parseInt(parts[2]); + + return new ModelProperties( + model.toPath(), + model.getName(), + labels, + // all files used to be 640x640 + width, + height, + Family.RKNN, + version); + } + + /** + * Determines the model version based on the model's filename. + * + *

"yolov5" -> "YOLO_V5" + * + *

"yolov8" -> "YOLO_V8" + * + *

"yolov11" -> "YOLO_V11" + * + * @param modelName The model's filename + * @return The model version + */ + private static Version getModelVersion(String modelName) throws IllegalArgumentException { + if (modelName.contains("yolov5")) { + return Version.YOLOV5; + } else if (modelName.contains("yolov8")) { + return Version.YOLOV8; + } else if (modelName.contains("yolov11")) { + return Version.YOLOV11; + } else { + throw new IllegalArgumentException("Unknown model version for model " + modelName); + } + } + + /** + * Parse RKNN name and return the name, width, height, and model type. + * + *

This is static as it is not dependent on the state of the class. + * + * @param modelName the name of the model + * @throws IllegalArgumentException if the model name does not follow the naming convention + * @return an array containing the name, width, height, and model type + */ + public static String[] parseRKNNName(String modelName) { + Matcher modelMatcher = modelPattern.matcher(modelName); + + if (!modelMatcher.matches()) { + throw new IllegalArgumentException( + "Model name must follow the naming convention of name-widthResolution-heightResolution-modelType.rknn"); + } + + return new String[] { + modelMatcher.group(1), modelMatcher.group(2), modelMatcher.group(3), modelMatcher.group(4) + }; + } + } + + // The path to the model is used as the key in the map because it is unique to + // the model, and should not change + @JsonProperty("modelPathToProperties") + private HashMap modelPathToProperties = + new HashMap(); + + /** + * Constructor for the NeuralNetworkProperties class. + * + *

This object holds a LinkedList of {@link ModelProperties} objects + */ + public NeuralNetworkModelsSettings() {} + + /** + * Constructor for the NeuralNetworkProperties class. + * + *

This object holds a LinkedList of {@link ModelProperties} objects. + * + * @param modelPropertiesList When the class is constructed, it will hold the provided list + */ + public NeuralNetworkModelsSettings(HashMap modelPropertiesList) {} + + @Override + public String toString() { + String toReturn = ""; + + toReturn += "NeuralNetworkProperties ["; + + toReturn += modelPathToProperties.toString() + "]"; + + return toReturn; + } + + /** + * Add a model to the list of models. + * + * @param modelProperties + */ + public void addModelProperties(ModelProperties modelProperties) { + modelPathToProperties.put(modelProperties.modelPath, modelProperties); + } + + /** + * Add two Neural Network Properties together. + * + *

Any properties that are the same will be overwritten by the second + * + * @param nnProps + * @return itself, so it can be chained and used fluently + */ + public NeuralNetworkModelsSettings sum(NeuralNetworkModelsSettings nnProps) { + modelPathToProperties.putAll(nnProps.modelPathToProperties); + + return this; + } + + /** + * Remove a model from the list of models. + * + * @param modelPath + * @return True if the model was removed, false if it was not found + */ + public boolean removeModel(Path modelPath) { + return modelPathToProperties.remove(modelPath) != null; + } + + /** + * Get the model properties for a given model path. + * + * @param modelPath + * @return {@link ModelProperties} object + */ + public ModelProperties getModel(Path modelPath) { + return modelPathToProperties.get(modelPath); + } + + /** + * Get all models + * + * @return A list of all models + */ + @JsonIgnore + public ModelProperties[] getModels() { + return modelPathToProperties.values().toArray(new ModelProperties[0]); + } + + /** + * Change the nickname of a {@link ModelProperties} object. + * + * @param modelPath + * @param newName + * @return True if the model was found and renamed, false if it was not found + */ + public boolean renameModel(Path modelPath, String newName) { + ModelProperties temp = modelPathToProperties.get(modelPath); + if (temp != null) { + modelPathToProperties.remove(modelPath); + modelPathToProperties.put( + modelPath, + new ModelProperties( + temp.modelPath, + newName, + temp.labels, + temp.resolutionWidth, + temp.resolutionHeight, + temp.family, + temp.version)); + return true; + } + return false; + } + + public boolean clear() { + modelPathToProperties.clear(); + return true; + } +} diff --git a/photon-core/src/main/java/org/photonvision/common/configuration/NeuralNetworkPropertyManager.java b/photon-core/src/main/java/org/photonvision/common/configuration/NeuralNetworkPropertyManager.java deleted file mode 100644 index 0fa6c31b0..000000000 --- a/photon-core/src/main/java/org/photonvision/common/configuration/NeuralNetworkPropertyManager.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright (C) Photon Vision. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.photonvision.common.configuration; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.nio.file.Path; -import java.util.HashMap; -import java.util.LinkedList; -import org.photonvision.common.configuration.NeuralNetworkModelManager.Family; -import org.photonvision.common.configuration.NeuralNetworkModelManager.Version; - -public class NeuralNetworkPropertyManager { - /* - * The properties of the model. This is used to determine which model to load. - * The only families currently supported are RKNN and Rubik (custom .tflite) - */ - public record ModelProperties( - @JsonProperty("modelPath") Path modelPath, - @JsonProperty("nickname") String nickname, - @JsonProperty("labels") LinkedList labels, - @JsonProperty("resolutionWidth") int resolutionWidth, - @JsonProperty("resolutionHeight") int resolutionHeight, - @JsonProperty("family") Family family, - @JsonProperty("version") Version version) { - @JsonCreator - public ModelProperties { - // Record constructor is automatically annotated with @JsonCreator - } - } - - // The path to the model is used as the key in the map because it is unique to - // the model, and should not change - @JsonProperty("modelPathToProperties") - private HashMap modelPathToProperties = - new HashMap(); - - /** - * Constructor for the NeuralNetworkProperties class. - * - *

This object holds a LinkedList of {@link ModelProperties} objects - */ - public NeuralNetworkPropertyManager() {} - - /** - * Constructor for the NeuralNetworkProperties class. - * - *

This object holds a LinkedList of {@link ModelProperties} objects. - * - * @param modelPropertiesList When the class is constructed, it will hold the provided list - */ - public NeuralNetworkPropertyManager(HashMap modelPropertiesList) {} - - @Override - public String toString() { - String toReturn = ""; - - toReturn += "NeuralNetworkProperties ["; - - toReturn += modelPathToProperties.toString() + "]"; - - return toReturn; - } - - /** - * Add a model to the list of models. - * - * @param modelProperties - */ - public void addModelProperties(ModelProperties modelProperties) { - modelPathToProperties.put(modelProperties.modelPath, modelProperties); - } - - /** - * Add two Neural Network Properties together. - * - *

Any properties that are the same will be overwritten by the second - * - * @param nnProps - * @return itself, so it can be chained and used fluently - */ - public NeuralNetworkPropertyManager sum(NeuralNetworkPropertyManager nnProps) { - modelPathToProperties.putAll(nnProps.modelPathToProperties); - - return this; - } - - /** - * Remove a model from the list of models. - * - * @param modelPath - * @return True if the model was removed, false if it was not found - */ - public boolean removeModel(Path modelPath) { - return modelPathToProperties.remove(modelPath) != null; - } - - /** - * Get the model properties for a given model path. - * - * @param modelPath - * @return {@link ModelProperties} object - */ - public ModelProperties getModel(Path modelPath) { - return modelPathToProperties.get(modelPath); - } - - /** - * Get all models - * - * @return A list of all models - */ - @JsonIgnore - public ModelProperties[] getModels() { - return modelPathToProperties.values().toArray(new ModelProperties[0]); - } - - /** - * Change the nickname of a {@link ModelProperties} object. - * - * @param modelPath - * @param newName - * @return True if the model was found and renamed, false if it was not found - */ - public boolean renameModel(Path modelPath, String newName) { - ModelProperties temp = modelPathToProperties.get(modelPath); - if (temp != null) { - modelPathToProperties.remove(modelPath); - modelPathToProperties.put( - modelPath, - new ModelProperties( - temp.modelPath, - newName, - temp.labels, - temp.resolutionWidth, - temp.resolutionHeight, - temp.family, - temp.version)); - return true; - } - return false; - } - - public boolean clear() { - modelPathToProperties.clear(); - return true; - } -} diff --git a/photon-core/src/main/java/org/photonvision/common/configuration/PhotonConfiguration.java b/photon-core/src/main/java/org/photonvision/common/configuration/PhotonConfiguration.java index 9b01cff12..64bd75444 100644 --- a/photon-core/src/main/java/org/photonvision/common/configuration/PhotonConfiguration.java +++ b/photon-core/src/main/java/org/photonvision/common/configuration/PhotonConfiguration.java @@ -28,7 +28,7 @@ public class PhotonConfiguration { private final HardwareSettings hardwareSettings; private NetworkConfig networkConfig; private AprilTagFieldLayout atfl; - private NeuralNetworkPropertyManager neuralNetworkProperties; + private NeuralNetworkModelsSettings neuralNetworkProperties; private HashMap cameraConfigurations; public PhotonConfiguration( @@ -36,7 +36,7 @@ public class PhotonConfiguration { HardwareSettings hardwareSettings, NetworkConfig networkConfig, AprilTagFieldLayout atfl, - NeuralNetworkPropertyManager neuralNetworkProperties) { + NeuralNetworkModelsSettings neuralNetworkProperties) { this( hardwareConfig, hardwareSettings, @@ -51,7 +51,7 @@ public class PhotonConfiguration { HardwareSettings hardwareSettings, NetworkConfig networkConfig, AprilTagFieldLayout atfl, - NeuralNetworkPropertyManager neuralNetworkProperties, + NeuralNetworkModelsSettings neuralNetworkProperties, HashMap cameraConfigurations) { this.hardwareConfig = hardwareConfig; this.hardwareSettings = hardwareSettings; @@ -67,7 +67,7 @@ public class PhotonConfiguration { new HardwareSettings(), new NetworkConfig(), new AprilTagFieldLayout(List.of(), 0, 0), - new NeuralNetworkPropertyManager()); + new NeuralNetworkModelsSettings()); } public HardwareConfig getHardwareConfig() { @@ -86,7 +86,7 @@ public class PhotonConfiguration { return atfl; } - public NeuralNetworkPropertyManager neuralNetworkPropertyManager() { + public NeuralNetworkModelsSettings neuralNetworkPropertyManager() { return neuralNetworkProperties; } @@ -98,7 +98,7 @@ public class PhotonConfiguration { this.networkConfig = networkConfig; } - public void setNeuralNetworkProperties(NeuralNetworkPropertyManager neuralNetworkProperties) { + public void setNeuralNetworkProperties(NeuralNetworkModelsSettings neuralNetworkProperties) { this.neuralNetworkProperties = neuralNetworkProperties; } @@ -132,6 +132,16 @@ public class PhotonConfiguration { @Override public String toString() { + StringBuilder cameraConfigurationsString = new StringBuilder(); + cameraConfigurations.forEach( + (key, value) -> { + cameraConfigurationsString + .append("\n ") + .append(key) + .append(" -> ") + .append(value.toString()); + }); + return "PhotonConfiguration [\n hardwareConfig=" + hardwareConfig + "\n hardwareSettings=" @@ -142,8 +152,8 @@ public class PhotonConfiguration { + atfl + "\n neuralNetworkProperties=" + neuralNetworkProperties - + "\n cameraConfigurations=" - + cameraConfigurations - + "\n]"; + + "\n cameraConfigurations={" + + cameraConfigurationsString + + "}\n]"; } } diff --git a/photon-core/src/main/java/org/photonvision/common/configuration/SqlConfigProvider.java b/photon-core/src/main/java/org/photonvision/common/configuration/SqlConfigProvider.java index 40d322df9..847405bd8 100644 --- a/photon-core/src/main/java/org/photonvision/common/configuration/SqlConfigProvider.java +++ b/photon-core/src/main/java/org/photonvision/common/configuration/SqlConfigProvider.java @@ -313,8 +313,8 @@ public class SqlConfigProvider extends ConfigProvider { loadConfigOrDefault( conn, GlobalKeys.NEURAL_NETWORK_PROPERTIES, - NeuralNetworkPropertyManager.class, - NeuralNetworkPropertyManager::new); + NeuralNetworkModelsSettings.class, + NeuralNetworkModelsSettings::new); var atfl = loadConfigOrDefault( conn, GlobalKeys.ATFL_CONFIG_FILE, AprilTagFieldLayout.class, this::atflDefault); @@ -612,52 +612,65 @@ public class SqlConfigProvider extends ConfigProvider { // Iterate over every row/"camera" in the table while (result.next()) { - List dummyList = new ArrayList<>(); + String uniqueName = ""; + try { + List dummyList = new ArrayList<>(); - var uniqueName = result.getString(Columns.CAM_UNIQUE_NAME); + uniqueName = result.getString(Columns.CAM_UNIQUE_NAME); - // A horrifying hack to keep backward compat with otherpaths - // We -really- need to delete this -stupid- otherpaths column. I hate it. - var configStr = result.getString(Columns.CAM_CONFIG_JSON); - CameraConfiguration config = JacksonUtils.deserialize(configStr, CameraConfiguration.class); + // A horrifying hack to keep backward compat with otherpaths + // We -really- need to delete this -stupid- otherpaths column. I hate it. + var configStr = result.getString(Columns.CAM_CONFIG_JSON); + CameraConfiguration config = + JacksonUtils.deserialize(configStr, CameraConfiguration.class); - if (config.matchedCameraInfo == null) { - logger.info("Legacy CameraConfiguration detected - upgrading"); + if (config.matchedCameraInfo == null) { + logger.info("Legacy CameraConfiguration detected - upgrading"); - // manually create the matchedCameraInfo ourselves. Need to upgrade: - // baseName, path, otherPaths, cameraType, usbvid/pid -> matchedCameraInfo - config.matchedCameraInfo = - JacksonUtils.deserialize(configStr, LegacyCameraConfigStruct.class).matchedCameraInfo; + // manually create the matchedCameraInfo ourselves. Need to upgrade: + // baseName, path, otherPaths, cameraType, usbvid/pid -> matchedCameraInfo + config.matchedCameraInfo = + JacksonUtils.deserialize(configStr, LegacyCameraConfigStruct.class) + .matchedCameraInfo; - // Except that otherPaths used to be its own column. so hack that in here as well - var otherPaths = + // Except that otherPaths used to be its own column. so hack that in here as well + var otherPaths = + JacksonUtils.deserialize( + result.getString(Columns.CAM_OTHERPATHS_JSON), String[].class); + if (config.matchedCameraInfo instanceof UsbCameraInfo usbInfo) { + usbInfo.otherPaths = otherPaths; + } + } + + var driverMode = JacksonUtils.deserialize( - result.getString(Columns.CAM_OTHERPATHS_JSON), String[].class); - if (config.matchedCameraInfo instanceof UsbCameraInfo usbInfo) { - usbInfo.otherPaths = otherPaths; + result.getString(Columns.CAM_DRIVERMODE_JSON), DriverModePipelineSettings.class); + List pipelineSettings = + JacksonUtils.deserialize( + result.getString(Columns.CAM_PIPELINE_JSONS), dummyList.getClass()); + + List loadedSettings = new ArrayList<>(); + for (var setting : pipelineSettings) { + if (setting instanceof String str) { + try { + loadedSettings.add(JacksonUtils.deserialize(str, CVPipelineSettings.class)); + } catch (IOException e) { + logger.error( + "Could not deserialize pipeline setting for camera " + config.nickname, e); + } + } } + + config.pipelineSettings = loadedSettings; + config.driveModeSettings = driverMode; + loadedConfigurations.put(uniqueName, config); + } catch (IOException e) { + logger.error( + "Could not deserialize camera configuration " + uniqueName + " from database!", e); } - - var driverMode = - JacksonUtils.deserialize( - result.getString(Columns.CAM_DRIVERMODE_JSON), DriverModePipelineSettings.class); - List pipelineSettings = - JacksonUtils.deserialize( - result.getString(Columns.CAM_PIPELINE_JSONS), dummyList.getClass()); - - List loadedSettings = new ArrayList<>(); - for (var setting : pipelineSettings) { - if (setting instanceof String str) { - loadedSettings.add(JacksonUtils.deserialize(str, CVPipelineSettings.class)); - } - } - - config.pipelineSettings = loadedSettings; - config.driveModeSettings = driverMode; - loadedConfigurations.put(uniqueName, config); } - } catch (SQLException | IOException e) { - logger.error("Err loading cameras: ", e); + } catch (SQLException e) { + logger.error("Err querying database to load cameras: ", e); } finally { try { if (query != null) query.close(); diff --git a/photon-core/src/main/java/org/photonvision/common/dataflow/websocket/UIGeneralSettings.java b/photon-core/src/main/java/org/photonvision/common/dataflow/websocket/UIGeneralSettings.java index d824d1557..069d11aee 100644 --- a/photon-core/src/main/java/org/photonvision/common/dataflow/websocket/UIGeneralSettings.java +++ b/photon-core/src/main/java/org/photonvision/common/dataflow/websocket/UIGeneralSettings.java @@ -18,14 +18,14 @@ package org.photonvision.common.dataflow.websocket; import java.util.List; -import org.photonvision.common.configuration.NeuralNetworkPropertyManager; +import org.photonvision.common.configuration.NeuralNetworkModelsSettings; public class UIGeneralSettings { public UIGeneralSettings( String version, String gpuAcceleration, boolean mrCalWorking, - NeuralNetworkPropertyManager.ModelProperties[] availableModels, + NeuralNetworkModelsSettings.ModelProperties[] availableModels, List supportedBackends, String hardwareModel, String hardwarePlatform, @@ -45,7 +45,7 @@ public class UIGeneralSettings { public String version; public String gpuAcceleration; public boolean mrCalWorking; - public NeuralNetworkPropertyManager.ModelProperties[] availableModels; + public NeuralNetworkModelsSettings.ModelProperties[] availableModels; public List supportedBackends; public String hardwareModel; public String hardwarePlatform; diff --git a/photon-core/src/main/java/org/photonvision/common/util/file/JacksonUtils.java b/photon-core/src/main/java/org/photonvision/common/util/file/JacksonUtils.java index e2d8e1324..d2edab277 100644 --- a/photon-core/src/main/java/org/photonvision/common/util/file/JacksonUtils.java +++ b/photon-core/src/main/java/org/photonvision/common/util/file/JacksonUtils.java @@ -18,6 +18,7 @@ package org.photonvision.common.util.file; import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.StreamReadFeature; import com.fasterxml.jackson.core.json.JsonReadFeature; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.DeserializationFeature; @@ -80,6 +81,7 @@ public class JacksonUtils { pathModule.addKeyDeserializer(Path.class, new PathKeyDeserializer()); return JsonMapper.builder() + .enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION) .configure(JsonReadFeature.ALLOW_JAVA_COMMENTS, true) .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) .activateDefaultTyping(ptv, ObjectMapper.DefaultTyping.JAVA_LANG_OBJECT) diff --git a/photon-core/src/main/java/org/photonvision/vision/objects/Model.java b/photon-core/src/main/java/org/photonvision/vision/objects/Model.java index f4db19f26..a661395ed 100644 --- a/photon-core/src/main/java/org/photonvision/vision/objects/Model.java +++ b/photon-core/src/main/java/org/photonvision/vision/objects/Model.java @@ -18,7 +18,7 @@ package org.photonvision.vision.objects; import org.photonvision.common.configuration.NeuralNetworkModelManager.Family; -import org.photonvision.common.configuration.NeuralNetworkPropertyManager.ModelProperties; +import org.photonvision.common.configuration.NeuralNetworkModelsSettings.ModelProperties; public interface Model { public ObjectDetector load(); diff --git a/photon-core/src/main/java/org/photonvision/vision/objects/NullModel.java b/photon-core/src/main/java/org/photonvision/vision/objects/NullModel.java index 0a10f1e64..117d3d110 100644 --- a/photon-core/src/main/java/org/photonvision/vision/objects/NullModel.java +++ b/photon-core/src/main/java/org/photonvision/vision/objects/NullModel.java @@ -20,7 +20,7 @@ package org.photonvision.vision.objects; import java.util.List; import org.opencv.core.Mat; import org.photonvision.common.configuration.NeuralNetworkModelManager.Family; -import org.photonvision.common.configuration.NeuralNetworkPropertyManager.ModelProperties; +import org.photonvision.common.configuration.NeuralNetworkModelsSettings.ModelProperties; import org.photonvision.vision.pipe.impl.NeuralNetworkPipeResult; /** diff --git a/photon-core/src/main/java/org/photonvision/vision/objects/RknnModel.java b/photon-core/src/main/java/org/photonvision/vision/objects/RknnModel.java index 1be547ccb..858330b4a 100644 --- a/photon-core/src/main/java/org/photonvision/vision/objects/RknnModel.java +++ b/photon-core/src/main/java/org/photonvision/vision/objects/RknnModel.java @@ -21,7 +21,7 @@ import java.io.File; import org.opencv.core.Size; import org.photonvision.common.configuration.NeuralNetworkModelManager.Family; import org.photonvision.common.configuration.NeuralNetworkModelManager.Version; -import org.photonvision.common.configuration.NeuralNetworkPropertyManager.ModelProperties; +import org.photonvision.common.configuration.NeuralNetworkModelsSettings.ModelProperties; public class RknnModel implements Model { public final File modelFile; diff --git a/photon-core/src/main/java/org/photonvision/vision/objects/RubikModel.java b/photon-core/src/main/java/org/photonvision/vision/objects/RubikModel.java index 99f713410..1b4de695f 100644 --- a/photon-core/src/main/java/org/photonvision/vision/objects/RubikModel.java +++ b/photon-core/src/main/java/org/photonvision/vision/objects/RubikModel.java @@ -21,7 +21,7 @@ import java.io.File; import org.opencv.core.Size; import org.photonvision.common.configuration.NeuralNetworkModelManager.Family; import org.photonvision.common.configuration.NeuralNetworkModelManager.Version; -import org.photonvision.common.configuration.NeuralNetworkPropertyManager.ModelProperties; +import org.photonvision.common.configuration.NeuralNetworkModelsSettings.ModelProperties; public class RubikModel implements Model { public final File modelFile; diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/ObjectDetectionPipelineSettings.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/ObjectDetectionPipelineSettings.java index 67428559a..4af55c7d6 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipeline/ObjectDetectionPipelineSettings.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipeline/ObjectDetectionPipelineSettings.java @@ -18,13 +18,13 @@ package org.photonvision.vision.pipeline; import org.photonvision.common.configuration.NeuralNetworkModelManager; -import org.photonvision.common.configuration.NeuralNetworkPropertyManager; +import org.photonvision.common.configuration.NeuralNetworkModelsSettings; import org.photonvision.vision.objects.Model; public class ObjectDetectionPipelineSettings extends AdvancedPipelineSettings { public double confidence; public double nms; // non maximal suppression - public NeuralNetworkPropertyManager.ModelProperties model; + public NeuralNetworkModelsSettings.ModelProperties model; public ObjectDetectionPipelineSettings() { super(); diff --git a/photon-core/src/main/java/org/photonvision/vision/processes/VisionModuleChangeSubscriber.java b/photon-core/src/main/java/org/photonvision/vision/processes/VisionModuleChangeSubscriber.java index 1b40864ff..ffdd27fc6 100644 --- a/photon-core/src/main/java/org/photonvision/vision/processes/VisionModuleChangeSubscriber.java +++ b/photon-core/src/main/java/org/photonvision/vision/processes/VisionModuleChangeSubscriber.java @@ -25,7 +25,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.locks.ReentrantLock; import org.opencv.core.Point; -import org.photonvision.common.configuration.NeuralNetworkPropertyManager.ModelProperties; +import org.photonvision.common.configuration.NeuralNetworkModelsSettings.ModelProperties; import org.photonvision.common.dataflow.DataChangeSubscriber; import org.photonvision.common.dataflow.events.DataChangeEvent; import org.photonvision.common.dataflow.events.IncomingWebSocketEvent; diff --git a/photon-core/src/test/java/org/photonvision/common/configuration/NeuralNetworkPropertyManagerTest.java b/photon-core/src/test/java/org/photonvision/common/configuration/NeuralNetworkPropertyManagerTest.java index 061b4d7d4..14dd6db0e 100644 --- a/photon-core/src/test/java/org/photonvision/common/configuration/NeuralNetworkPropertyManagerTest.java +++ b/photon-core/src/test/java/org/photonvision/common/configuration/NeuralNetworkPropertyManagerTest.java @@ -25,13 +25,13 @@ import java.util.LinkedList; import org.junit.jupiter.api.Test; import org.photonvision.common.configuration.NeuralNetworkModelManager.Family; import org.photonvision.common.configuration.NeuralNetworkModelManager.Version; -import org.photonvision.common.configuration.NeuralNetworkPropertyManager.ModelProperties; +import org.photonvision.common.configuration.NeuralNetworkModelsSettings.ModelProperties; import org.photonvision.common.util.file.JacksonUtils; public class NeuralNetworkPropertyManagerTest { @Test void testSerialization() { - var nnpm = new NeuralNetworkPropertyManager(); + var nnpm = new NeuralNetworkModelsSettings(); // Path is always serialized as absolute; for the test to pass, this must also be made absolute nnpm.addModelProperties( new ModelProperties( @@ -45,7 +45,7 @@ public class NeuralNetworkPropertyManagerTest { String result = assertDoesNotThrow(() -> JacksonUtils.serializeToString(nnpm)); var deserializedNnpm = assertDoesNotThrow( - () -> JacksonUtils.deserialize(result, NeuralNetworkPropertyManager.class)); + () -> JacksonUtils.deserialize(result, NeuralNetworkModelsSettings.class)); assertEquals(nnpm.getModels()[0], deserializedNnpm.getModels()[0]); } } diff --git a/photon-core/src/test/java/org/photonvision/common/configuration/SQLConfigTest.java b/photon-core/src/test/java/org/photonvision/common/configuration/SQLConfigTest.java index 642c09487..a965e3557 100644 --- a/photon-core/src/test/java/org/photonvision/common/configuration/SQLConfigTest.java +++ b/photon-core/src/test/java/org/photonvision/common/configuration/SQLConfigTest.java @@ -20,6 +20,7 @@ package org.photonvision.common.configuration; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import com.fasterxml.jackson.core.JsonProcessingException; import edu.wpi.first.cscore.UsbCameraInfo; import java.nio.file.Path; import java.util.List; @@ -28,11 +29,13 @@ import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.photonvision.common.LoadJNI; +import org.photonvision.common.configuration.NeuralNetworkModelManager.Family; import org.photonvision.common.util.TestUtils; import org.photonvision.vision.camera.CameraQuirk; import org.photonvision.vision.camera.PVCameraInfo; import org.photonvision.vision.pipeline.AprilTagPipelineSettings; import org.photonvision.vision.pipeline.ColoredShapePipelineSettings; +import org.photonvision.vision.pipeline.ObjectDetectionPipelineSettings; import org.photonvision.vision.pipeline.ReflectivePipelineSettings; public class SQLConfigTest { @@ -104,4 +107,52 @@ public class SQLConfigTest { .hasQuirk(c)); } } + + void common2025p3p1Assertions(PhotonConfiguration config) { + // Make sure we got 8 cameras + assertEquals(8, config.getCameraConfigurations().size()); + + // Make sure exactly 2 have object detection pipelines + long count = + config.getCameraConfigurations().values().stream() + .filter( + c -> + c.pipelineSettings.stream() + .anyMatch(s -> s instanceof ObjectDetectionPipelineSettings)) + .count(); + assertEquals(2, count); + } + + @Test + public void testLoadNewNNMM() throws JsonProcessingException { + var folder = TestUtils.getConfigDirectoriesPath(false).resolve("2025.3.1-old-nnmm"); + var cfgManager = new ConfigManager(folder, new SqlConfigProvider(folder)); + + // Replace global configmanager + ConfigManager.INSTANCE = cfgManager; + + assertDoesNotThrow(cfgManager::load); + + System.out.println(cfgManager.getConfig()); + common2025p3p1Assertions(cfgManager.getConfig()); + + // And we now see two models + NeuralNetworkModelManager.getInstance(); + // force us to allow RKNN + NeuralNetworkModelManager.getInstance().supportedBackends.add(Family.RKNN); + NeuralNetworkModelManager.getInstance().discoverModels(); + assertEquals(5, NeuralNetworkModelManager.getInstance().models.get(Family.RKNN).size()); + + ConfigManager.getInstance().saveToDisk(); + + // Now that we have the config saved, load it again + var reloadedProvider = new SqlConfigProvider(folder); + reloadedProvider.load(); + common2025p3p1Assertions(reloadedProvider.getConfig()); + + // And make sure NNPM has all 5 models + assertEquals(5, reloadedProvider.getConfig().neuralNetworkPropertyManager().getModels().length); + + ConfigManager.INSTANCE = null; + } } diff --git a/photon-server/src/main/java/org/photonvision/server/RequestHandler.java b/photon-server/src/main/java/org/photonvision/server/RequestHandler.java index 63ffd2ec3..c6ed9b8d2 100644 --- a/photon-server/src/main/java/org/photonvision/server/RequestHandler.java +++ b/photon-server/src/main/java/org/photonvision/server/RequestHandler.java @@ -39,7 +39,7 @@ import org.opencv.imgcodecs.Imgcodecs; import org.photonvision.common.configuration.ConfigManager; import org.photonvision.common.configuration.NetworkConfig; import org.photonvision.common.configuration.NeuralNetworkModelManager; -import org.photonvision.common.configuration.NeuralNetworkPropertyManager.ModelProperties; +import org.photonvision.common.configuration.NeuralNetworkModelsSettings.ModelProperties; import org.photonvision.common.dataflow.DataChangeDestination; import org.photonvision.common.dataflow.DataChangeService; import org.photonvision.common.dataflow.events.IncomingWebSocketEvent; diff --git a/test-resources/old_configs/2025.3.1-old-nnmm/models/algaeV1-640-640-yolov8n-labels.txt b/test-resources/old_configs/2025.3.1-old-nnmm/models/algaeV1-640-640-yolov8n-labels.txt new file mode 100644 index 000000000..51335db7e --- /dev/null +++ b/test-resources/old_configs/2025.3.1-old-nnmm/models/algaeV1-640-640-yolov8n-labels.txt @@ -0,0 +1 @@ +algae diff --git a/test-resources/old_configs/2025.3.1-old-nnmm/models/algaeV1-640-640-yolov8n.rknn b/test-resources/old_configs/2025.3.1-old-nnmm/models/algaeV1-640-640-yolov8n.rknn new file mode 100644 index 000000000..e69de29bb diff --git a/test-resources/old_configs/2025.3.1-old-nnmm/models/chunkyfuel-640-640-yolov11-labels.txt b/test-resources/old_configs/2025.3.1-old-nnmm/models/chunkyfuel-640-640-yolov11-labels.txt new file mode 100644 index 000000000..78a0ae075 --- /dev/null +++ b/test-resources/old_configs/2025.3.1-old-nnmm/models/chunkyfuel-640-640-yolov11-labels.txt @@ -0,0 +1,2 @@ +bumpers +fuel diff --git a/test-resources/old_configs/2025.3.1-old-nnmm/models/chunkyfuel-640-640-yolov11.rknn b/test-resources/old_configs/2025.3.1-old-nnmm/models/chunkyfuel-640-640-yolov11.rknn new file mode 100644 index 000000000..e69de29bb diff --git a/test-resources/old_configs/2025.3.1-old-nnmm/models/coraldetector-640-640-yolov11-labels.txt b/test-resources/old_configs/2025.3.1-old-nnmm/models/coraldetector-640-640-yolov11-labels.txt new file mode 100644 index 000000000..6ef754ce8 --- /dev/null +++ b/test-resources/old_configs/2025.3.1-old-nnmm/models/coraldetector-640-640-yolov11-labels.txt @@ -0,0 +1,2 @@ +Coral +Algae diff --git a/test-resources/old_configs/2025.3.1-old-nnmm/models/coraldetector-640-640-yolov11.rknn b/test-resources/old_configs/2025.3.1-old-nnmm/models/coraldetector-640-640-yolov11.rknn new file mode 100644 index 000000000..e69de29bb diff --git a/test-resources/old_configs/2025.3.1-old-nnmm/models/fuelv2-640-640-yolov11-labels.txt b/test-resources/old_configs/2025.3.1-old-nnmm/models/fuelv2-640-640-yolov11-labels.txt new file mode 100644 index 000000000..95943651d --- /dev/null +++ b/test-resources/old_configs/2025.3.1-old-nnmm/models/fuelv2-640-640-yolov11-labels.txt @@ -0,0 +1 @@ +fuel diff --git a/test-resources/old_configs/2025.3.1-old-nnmm/models/fuelv2-640-640-yolov11.rknn b/test-resources/old_configs/2025.3.1-old-nnmm/models/fuelv2-640-640-yolov11.rknn new file mode 100644 index 000000000..e69de29bb diff --git a/test-resources/old_configs/2025.3.1-old-nnmm/models/fuelv3-640-640-yolov11-labels.txt b/test-resources/old_configs/2025.3.1-old-nnmm/models/fuelv3-640-640-yolov11-labels.txt new file mode 100644 index 000000000..4cb462f89 --- /dev/null +++ b/test-resources/old_configs/2025.3.1-old-nnmm/models/fuelv3-640-640-yolov11-labels.txt @@ -0,0 +1,2 @@ +Bumper +Fuel diff --git a/test-resources/old_configs/2025.3.1-old-nnmm/models/fuelv3-640-640-yolov11.rknn b/test-resources/old_configs/2025.3.1-old-nnmm/models/fuelv3-640-640-yolov11.rknn new file mode 100644 index 000000000..e69de29bb diff --git a/test-resources/old_configs/2025.3.1-old-nnmm/photon.sqlite b/test-resources/old_configs/2025.3.1-old-nnmm/photon.sqlite new file mode 100644 index 000000000..ad6e7212e Binary files /dev/null and b/test-resources/old_configs/2025.3.1-old-nnmm/photon.sqlite differ