From 7b67f6bebf3e5aacdb9b0df75369c4b5da40b34b Mon Sep 17 00:00:00 2001 From: Mohammad Durrani <46766905+mdurrani808@users.noreply.github.com> Date: Mon, 15 Jan 2024 22:28:34 -0500 Subject: [PATCH] Add RKNN / Object Detection Pipeline (#1144) Tested on Orange Pi 5 and Cool Pi 4B. Merge with parts of the OpenCV DNN PR. Adds support for YOLOv5s models for Rockchip CPUs with a NPU. Right now hard coded to a note model from alex_idk. Very much still incubating and largely untested. --- .styleguide | 1 + build.gradle | 2 + photon-client/index.html | 2 +- photon-client/package-lock.json | 8 +- photon-client/package.json | 6 +- .../dashboard/CameraAndPipelineSelectCard.vue | 12 +- .../components/dashboard/ConfigOptions.vue | 28 +++- .../dashboard/tabs/ObjectDetectionTab.vue | 35 +++++ .../components/dashboard/tabs/TargetsTab.vue | 16 ++ .../stores/settings/GeneralSettingsStore.ts | 6 +- .../src/types/PhotonTrackingTypes.ts | 4 + photon-client/src/types/PipelineTypes.ts | 32 +++- photon-client/src/types/SettingTypes.ts | 1 + photon-client/src/types/WebsocketDataTypes.ts | 3 +- photon-client/vite.config.ts | 19 +-- photon-core/build.gradle | 4 +- .../common/configuration/ConfigManager.java | 7 + .../NeuralNetworkModelManager.java | 98 +++++++++++++ .../configuration/PhotonConfiguration.java | 4 +- .../dataflow/websocket/UIDataPublisher.java | 1 + .../common/hardware/Platform.java | 19 ++- .../org/photonvision/jni/PhotonJNICommon.java | 25 ++-- .../org/photonvision/jni/RknnDetectorJNI.java | 138 ++++++++++++++++++ .../photonvision/mrcal/MrCalJNILoader.java | 28 +++- .../org/photonvision/vision/pipe/CVPipe.java | 4 + .../vision/pipe/impl/ArucoDetectionPipe.java | 7 + .../vision/pipe/impl/Calibrate3dPipe.java | 2 +- .../impl}/Calibrate3dPipeline.java | 8 +- .../pipe/impl/NeuralNetworkPipeResult.java | 32 ++++ .../vision/pipe/impl/RknnDetectionPipe.java | 69 +++++++++ .../vision/pipeline/CVPipeline.java | 11 +- .../vision/pipeline/CVPipelineSettings.java | 3 +- .../pipeline/ObjectDetectionPipeline.java | 94 ++++++++++++ .../ObjectDetectionPipelineSettings.java | 34 +++++ .../vision/pipeline/PipelineType.java | 5 +- .../pipeline/result/CVPipelineResult.java | 25 +++- .../vision/processes/PipelineManager.java | 19 ++- .../vision/target/TrackedTarget.java | 62 ++++++++ .../vision/pipeline/Calibrate3dPipeTest.java | 1 + .../src/main/java/org/photonvision/Main.java | 17 ++- .../src/main/resources/models/labels.txt | 1 + .../models/note-640-640-yolov5s.rknn | Bin 0 -> 8261246 bytes 42 files changed, 830 insertions(+), 63 deletions(-) create mode 100644 photon-client/src/components/dashboard/tabs/ObjectDetectionTab.vue create mode 100644 photon-core/src/main/java/org/photonvision/common/configuration/NeuralNetworkModelManager.java create mode 100644 photon-core/src/main/java/org/photonvision/jni/RknnDetectorJNI.java rename photon-core/src/main/java/org/photonvision/vision/{pipeline => pipe/impl}/Calibrate3dPipeline.java (97%) create mode 100644 photon-core/src/main/java/org/photonvision/vision/pipe/impl/NeuralNetworkPipeResult.java create mode 100644 photon-core/src/main/java/org/photonvision/vision/pipe/impl/RknnDetectionPipe.java create mode 100644 photon-core/src/main/java/org/photonvision/vision/pipeline/ObjectDetectionPipeline.java create mode 100644 photon-core/src/main/java/org/photonvision/vision/pipeline/ObjectDetectionPipelineSettings.java create mode 100644 photon-server/src/main/resources/models/labels.txt create mode 100644 photon-server/src/main/resources/models/note-640-640-yolov5s.rknn diff --git a/.styleguide b/.styleguide index ff0e63b45..376e3b22c 100644 --- a/.styleguide +++ b/.styleguide @@ -18,6 +18,7 @@ modifiableFileExclude { \.dll$ \.webp$ \.ico$ + \.rknn$ gradlew } diff --git a/build.gradle b/build.gradle index b70cf7850..6d4dfd9f7 100644 --- a/build.gradle +++ b/build.gradle @@ -30,9 +30,11 @@ ext { joglVersion = "2.4.0-rc-20200307" javalinVersion = "5.6.2" photonGlDriverLibVersion = "dev-v2023.1.0-9-g75fc678" + rknnVersion = "dev-v2024.0.0-30-g001b5ec" frcYear = "2024" mrcalVersion = "dev-v2024.0.0-7-gc976aaa"; + pubVersion = versionString isDev = pubVersion.startsWith("dev") diff --git a/photon-client/index.html b/photon-client/index.html index 3547e54f0..533f161dd 100644 --- a/photon-client/index.html +++ b/photon-client/index.html @@ -1,4 +1,4 @@ - + diff --git a/photon-client/package-lock.json b/photon-client/package-lock.json index a3794eaed..5ce803504 100644 --- a/photon-client/package-lock.json +++ b/photon-client/package-lock.json @@ -31,7 +31,7 @@ "eslint": "^8.56.0", "eslint-plugin-vue": "^9.19.2", "npm-run-all": "^4.1.5", - "prettier": "^3.1.1", + "prettier": "3.2.2", "sass": "~1.32", "sass-loader": "^13.3.2", "terser": "^5.14.2", @@ -3917,9 +3917,9 @@ } }, "node_modules/prettier": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.1.1.tgz", - "integrity": "sha512-22UbSzg8luF4UuZtzgiUOfcGM8s4tjBv6dJRT7j275NXsy2jb4aJa4NNveul5x4eqlF1wuhuR2RElK71RvmVaw==", + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.2.2.tgz", + "integrity": "sha512-HTByuKZzw7utPiDO523Tt2pLtEyK7OibUD9suEJQrPUCYQqrHr74GGX6VidMrovbf/I50mPqr8j/II6oBAuc5A==", "dev": true, "bin": { "prettier": "bin/prettier.cjs" diff --git a/photon-client/package.json b/photon-client/package.json index a69b55602..a2ab77c4c 100644 --- a/photon-client/package.json +++ b/photon-client/package.json @@ -27,17 +27,17 @@ }, "devDependencies": { "@rushstack/eslint-patch": "^1.3.2", - "@vue/eslint-config-prettier": "^9.0.0", - "@vue/eslint-config-typescript": "^12.0.0", - "prettier": "^3.1.1", "@types/node": "^16.11.45", "@types/three": "^0.160.0", "@vitejs/plugin-vue2": "^2.3.1", + "@vue/eslint-config-prettier": "^9.0.0", + "@vue/eslint-config-typescript": "^12.0.0", "@vue/tsconfig": "^0.5.1", "deepmerge": "^4.3.1", "eslint": "^8.56.0", "eslint-plugin-vue": "^9.19.2", "npm-run-all": "^4.1.5", + "prettier": "3.2.2", "sass": "~1.32", "sass-loader": "^13.3.2", "terser": "^5.14.2", diff --git a/photon-client/src/components/dashboard/CameraAndPipelineSelectCard.vue b/photon-client/src/components/dashboard/CameraAndPipelineSelectCard.vue index cec1db4f9..a42264186 100644 --- a/photon-client/src/components/dashboard/CameraAndPipelineSelectCard.vue +++ b/photon-client/src/components/dashboard/CameraAndPipelineSelectCard.vue @@ -7,6 +7,7 @@ import { computed, ref } from "vue"; import PvIcon from "@/components/common/pv-icon.vue"; import PvInput from "@/components/common/pv-input.vue"; import { PipelineType } from "@/types/PipelineTypes"; +import { useSettingsStore } from "@/stores/settings/GeneralSettingsStore"; const changeCurrentCameraIndex = (index: number) => { useCameraSettingsStore().setCurrentCameraIndex(index, true); @@ -24,6 +25,8 @@ const changeCurrentCameraIndex = (index: number) => { case PipelineType.Aruco: pipelineType.value = WebsocketPipelineType.Aruco; break; + case PipelineType.ObjectDetection: + pipelineType.value = WebsocketPipelineType.ObjectDetection; } }; @@ -154,6 +157,9 @@ const pipelineTypesWrapper = computed<{ name: string; value: number }[]>(() => { { name: "AprilTag", value: WebsocketPipelineType.AprilTag }, { name: "Aruco", value: WebsocketPipelineType.Aruco } ]; + if (useSettingsStore().general.rknnSupported) { + pipelineTypes.push({ name: "Object Detection", value: WebsocketPipelineType.ObjectDetection }); + } if (useCameraSettingsStore().isDriverMode) { pipelineTypes.push({ name: "Driver Mode", value: WebsocketPipelineType.DriverMode }); @@ -208,6 +214,9 @@ useCameraSettingsStore().$subscribe((mutation, state) => { case PipelineType.Aruco: pipelineType.value = WebsocketPipelineType.Aruco; break; + case PipelineType.ObjectDetection: + pipelineType.value = WebsocketPipelineType.ObjectDetection; + break; } }); @@ -354,7 +363,8 @@ useCameraSettingsStore().$subscribe((mutation, state) => { { name: 'Reflective', value: WebsocketPipelineType.Reflective }, { name: 'Colored Shape', value: WebsocketPipelineType.ColoredShape }, { name: 'AprilTag', value: WebsocketPipelineType.AprilTag }, - { name: 'Aruco', value: WebsocketPipelineType.Aruco } + { name: 'Aruco', value: WebsocketPipelineType.Aruco }, + { name: 'Object Detection', value: WebsocketPipelineType.ObjectDetection } ]" /> diff --git a/photon-client/src/components/dashboard/ConfigOptions.vue b/photon-client/src/components/dashboard/ConfigOptions.vue index 6ed0156f9..c1023f217 100644 --- a/photon-client/src/components/dashboard/ConfigOptions.vue +++ b/photon-client/src/components/dashboard/ConfigOptions.vue @@ -8,6 +8,7 @@ import ThresholdTab from "@/components/dashboard/tabs/ThresholdTab.vue"; import ContoursTab from "@/components/dashboard/tabs/ContoursTab.vue"; import AprilTagTab from "@/components/dashboard/tabs/AprilTagTab.vue"; import ArucoTab from "@/components/dashboard/tabs/ArucoTab.vue"; +import ObjectDetectionTab from "@/components/dashboard/tabs/ObjectDetectionTab.vue"; import OutputTab from "@/components/dashboard/tabs/OutputTab.vue"; import TargetsTab from "@/components/dashboard/tabs/TargetsTab.vue"; import PnPTab from "@/components/dashboard/tabs/PnPTab.vue"; @@ -40,6 +41,10 @@ const allTabs = Object.freeze({ tabName: "Aruco", component: ArucoTab }, + objectDetectionTab: { + tabName: "Object Detection", + component: ObjectDetectionTab + }, outputTab: { tabName: "Output", component: OutputTab @@ -75,6 +80,7 @@ const getTabGroups = (): ConfigOption[][] => { allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, + allTabs.objectDetectionTab, allTabs.outputTab ], [allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab] @@ -82,14 +88,21 @@ const getTabGroups = (): ConfigOption[][] => { } else if (lgAndDown) { return [ [allTabs.inputTab], - [allTabs.thresholdTab, allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, allTabs.outputTab], + [ + allTabs.thresholdTab, + allTabs.contoursTab, + allTabs.apriltagTab, + allTabs.arucoTab, + allTabs.objectDetectionTab, + allTabs.outputTab + ], [allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab] ]; } else if (xl) { return [ [allTabs.inputTab], [allTabs.thresholdTab], - [allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, allTabs.outputTab], + [allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, allTabs.objectDetectionTab, allTabs.outputTab], [allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab] ]; } @@ -103,17 +116,20 @@ const tabGroups = computed(() => { const allow3d = useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled; const isAprilTag = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.AprilTag; const isAruco = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.Aruco; + const isObjectDetection = + useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.ObjectDetection; return getTabGroups() .map((tabGroup) => tabGroup.filter( (tabConfig) => !(!allow3d && tabConfig.tabName === "3D") && //Filter out 3D tab any time 3D isn't calibrated - !((!allow3d || isAprilTag || isAruco) && tabConfig.tabName === "PnP") && //Filter out the PnP config tab if 3D isn't available, or we're doing AprilTags - !((isAprilTag || isAruco) && tabConfig.tabName === "Threshold") && //Filter out threshold tab if we're doing AprilTags - !((isAprilTag || isAruco) && tabConfig.tabName === "Contours") && //Filter out contours if we're doing AprilTags + !((!allow3d || isAprilTag || isAruco || isObjectDetection) && tabConfig.tabName === "PnP") && //Filter out the PnP config tab if 3D isn't available, or we're doing AprilTags + !((isAprilTag || isAruco || isObjectDetection) && tabConfig.tabName === "Threshold") && //Filter out threshold tab if we're doing AprilTags + !((isAprilTag || isAruco || isObjectDetection) && tabConfig.tabName === "Contours") && //Filter out contours if we're doing AprilTags !(!isAprilTag && tabConfig.tabName === "AprilTag") && //Filter out apriltag unless we actually are doing AprilTags - !(!isAruco && tabConfig.tabName === "Aruco") //Filter out aruco unless we actually are doing Aruco + !(!isAruco && tabConfig.tabName === "Aruco") && + !(!isObjectDetection && tabConfig.tabName === "Object Detection") //Filter out aruco unless we actually are doing Aruco ) ) .filter((it) => it.length); // Remove empty tab groups diff --git a/photon-client/src/components/dashboard/tabs/ObjectDetectionTab.vue b/photon-client/src/components/dashboard/tabs/ObjectDetectionTab.vue new file mode 100644 index 000000000..646c1fcb2 --- /dev/null +++ b/photon-client/src/components/dashboard/tabs/ObjectDetectionTab.vue @@ -0,0 +1,35 @@ + + + diff --git a/photon-client/src/components/dashboard/tabs/TargetsTab.vue b/photon-client/src/components/dashboard/tabs/TargetsTab.vue index 87d32dcb4..1e2caf893 100644 --- a/photon-client/src/components/dashboard/tabs/TargetsTab.vue +++ b/photon-client/src/components/dashboard/tabs/TargetsTab.vue @@ -48,6 +48,10 @@ const resetCurrentBuffer = () => { > Fiducial ID +