Backport maven changes

* LL3

* Update build.yml

.github/workflows/build.yml

@@ -2,123 +2,14 @@ name: Build
 
 on:
   push:
-    branches: [ master ]
+    branches: 
+      - master
     tags:
       - 'v*'
   pull_request:
     branches: [ master ]
 
 jobs:
-  build-client:
-    name: "PhotonClient Build"
-    defaults:
-      run:
-        working-directory: photon-client
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: 18
-      - name: Install Dependencies
-        run: npm ci
-      - name: Build Production Client
-        run: npm run build
-      - uses: actions/upload-artifact@v4
-        with:
-          name: built-client
-          path: photon-client/dist/
-  build-examples:
-    name: "Build Examples"
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Fetch tags
-        run: git fetch --tags --force
-      - name: Install Java 17
-        uses: actions/setup-java@v4
-        with:
-          java-version: 17
-          distribution: temurin
-      # Need to publish to maven local first, so that C++ sim can pick it up
-      # Still haven't figured out how to make the vendordep file be copied before trying to build examples
-      - name: Publish photonlib to maven local
-        run: |
-          chmod +x gradlew
-          ./gradlew publishtomavenlocal -x check
-      - name: Build Java examples
-        working-directory: photonlib-java-examples
-        run: |
-          chmod +x gradlew
-          ./gradlew copyPhotonlib -x check
-          ./gradlew build -x check --max-workers 2
-      - name: Build C++ examples
-        working-directory: photonlib-cpp-examples
-        run: |
-          chmod +x gradlew
-          ./gradlew copyPhotonlib -x check
-          ./gradlew build -x check --max-workers 2
-  build-gradle:
-    name: "Gradle Build"
-    runs-on: ubuntu-22.04
-    steps:
-      # Checkout code.
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Fetch tags
-        run: git fetch --tags --force
-      - name: Install Java 17
-        uses: actions/setup-java@v3
-        with:
-          java-version: 17
-          distribution: temurin
-      - name: Install mrcal deps
-        run: sudo apt-get update && sudo apt-get install -y libcholmod3 liblapack3 libsuitesparseconfig5
-      - name: Gradle Build
-        run: |
-          chmod +x gradlew
-          ./gradlew build -x check --max-workers 2
-      - name: Gradle Tests
-        run: ./gradlew testHeadless -i --max-workers 1 --stacktrace
-      - name: Gradle Coverage
-        run: ./gradlew jacocoTestReport --max-workers 1
-      - name: Publish Coverage Report
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./photon-server/build/reports/jacoco/test/jacocoTestReport.xml
-      - name: Publish Core Coverage Report
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./photon-core/build/reports/jacoco/test/jacocoTestReport.xml
-  build-offline-docs:
-    name: "Build Offline Docs"
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          repository: 'PhotonVision/photonvision-docs.git'
-          ref: master
-      - uses: actions/setup-python@v5
-        with:
-          python-version: '3.9'
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install sphinx sphinx_rtd_theme sphinx-tabs sphinxext-opengraph doc8
-          pip install -r requirements.txt
-      - name: Build the docs
-        run: |
-          make html
-      - uses: actions/upload-artifact@master
-        with:
-          name: built-docs
-          path: build/html
   build-photonlib-host:
     env:
       MACOSX_DEPLOYMENT_TARGET: 12
@@ -184,149 +75,7 @@ jobs:
       - name: Publish
         run: |
           chmod +x gradlew
-          ./gradlew photon-lib:publish
+          ./gradlew photon-lib:publish photon-targeting:publish
         env:
           ARTIFACTORY_API_KEY: ${{ secrets.ARTIFACTORY_API_KEY }}
         if: github.event_name == 'push'
-  build-package:
-    needs: [build-client, build-gradle, build-offline-docs]
-
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - os: windows-latest
-            artifact-name: Win64
-            architecture: x64
-            arch-override: none
-          - os: macos-latest
-            artifact-name: macOS
-            architecture: x64
-            arch-override: none
-          - os: ubuntu-latest
-            artifact-name: Linux
-            architecture: x64
-            arch-override: none
-          - os: macos-latest
-            artifact-name: macOSArm
-            architecture: x64
-            arch-override: macarm64
-          - os: ubuntu-latest
-            artifact-name: LinuxArm32
-            architecture: x64
-            arch-override: linuxarm32
-          - os: ubuntu-latest
-            artifact-name: LinuxArm64
-            architecture: x64
-            arch-override: linuxarm64
-
-    runs-on: ${{ matrix.os }}
-    name: "Build fat JAR - ${{ matrix.artifact-name }}"
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Install Java 17
-        uses: actions/setup-java@v4
-        with:
-          java-version: 17
-          distribution: temurin
-      - run: |
-          rm -rf photon-server/src/main/resources/web/*
-          mkdir -p photon-server/src/main/resources/web/docs
-        if: ${{ (matrix.os) != 'windows-latest' }}
-      - run: |
-          del photon-server\src\main\resources\web\*.*
-          mkdir photon-server\src\main\resources\web\docs
-        if: ${{ (matrix.os) == 'windows-latest' }}
-      - uses: actions/download-artifact@v4
-        with:
-          name: built-client
-          path: photon-server/src/main/resources/web/
-      - uses: actions/download-artifact@v4
-        with:
-          name: built-docs
-          path: photon-server/src/main/resources/web/docs
-      - run: |
-          chmod +x gradlew
-          ./gradlew photon-server:shadowJar --max-workers 2 -PArchOverride=${{ matrix.arch-override }}
-        if: ${{ (matrix.arch-override != 'none') }}
-      - run: |
-          chmod +x gradlew
-          ./gradlew photon-server:shadowJar --max-workers 2
-        if: ${{ (matrix.arch-override == 'none') }}
-      - uses: actions/upload-artifact@v4
-        with:
-          name: jar-${{ matrix.artifact-name }}
-          path: photon-server/build/libs
-  build-image:
-    needs: [build-package]
-
-    if: ${{ github.event_name != 'pull_request' }}
-
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - os: ubuntu-latest
-            artifact-name: LinuxArm64
-            image_suffix: RaspberryPi
-            image_url: https://github.com/PhotonVision/photon-image-modifier/releases/download/v2024.0.4/photonvision_raspi.img.xz
-          - os: ubuntu-latest
-            artifact-name: LinuxArm64
-            image_suffix: limelight2
-            image_url: https://github.com/PhotonVision/photon-image-modifier/releases/download/v2024.0.4/photonvision_limelight.img.xz
-          - os: ubuntu-latest
-            artifact-name: LinuxArm64
-            image_suffix: orangepi5
-            image_url: https://github.com/PhotonVision/photon-image-modifier/releases/download/v2024.0.4/photonvision_opi5.img.xz
-
-    runs-on: ${{ matrix.os }}
-    name: "Build image - ${{ matrix.image_url }}"
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - uses: actions/download-artifact@v4
-        with:
-          name: jar-${{ matrix.artifact-name }}
-      # TODO- replace with the arm-runner action and run this inside of the chroot. but this works for now.
-      - name: Generate image
-        run: |
-          chmod +x scripts/generatePiImage.sh
-          ./scripts/generatePiImage.sh ${{ matrix.image_url }} ${{ matrix.image_suffix }}
-      - uses: actions/upload-artifact@v4
-        name: Upload image
-        with:
-          name: image-${{ matrix.image_suffix }}
-          path: photonvision*.xz
-  release:
-    needs: [build-package, build-image]
-    runs-on: ubuntu-22.04
-    steps:
-      # Download literally every single artifact. This also downloads client and docs,
-      # but the filtering below won't pick these up (I hope)
-      - uses: actions/download-artifact@v4
-      - run: find
-      # Push to dev release
-      - uses: pyTooling/Actions/releaser@r0
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          tag: 'Dev'
-          rm: true
-          files: |
-            **/*.xz
-            **/*.jar
-        if: github.event_name == 'push'
-      # Upload all jars and xz archives
-      - uses: softprops/action-gh-release@v1
-        with:
-          files: |
-            **/*.xz
-            **/*.jar
-        if: startsWith(github.ref, 'refs/tags/v')
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/documentation.yml

@@ -1,92 +0,0 @@
-name: Documentation
-
-on:
-  push:
-    # For now, run on all commits to master
-    branches: [ master ]
-    # and also all tags starting with v
-    tags:
-      - 'v*'
-
-# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
-permissions:
-  contents: read
-  pages: write
-  id-token: write
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build-client:
-    name: "PhotonClient Build"
-    defaults:
-      run:
-        working-directory: photon-client
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: 18
-      - name: Install Dependencies
-        run: npm ci
-      - name: Build Production Client
-        run: npm run build-demo
-      - uses: actions/upload-artifact@v4
-        with:
-          name: built-client
-          path: photon-client/dist/
-
-  run_docs:
-    runs-on: "ubuntu-22.04"
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-    - name: Fetch tags
-      run: git fetch --tags --force
-    - name: Install Java 17
-      uses: actions/setup-java@v3
-      with:
-        java-version: 17
-        distribution: temurin
-
-    - name: Build javadocs/doxygen
-      run: |
-        chmod +x gradlew
-        ./gradlew docs:generateJavaDocs docs:doxygen
-
-    - uses: actions/upload-artifact@v4
-      with:
-        name: built-docs
-        path: docs/build/docs
-
-  release:
-    needs: [build-client, run_docs]
-
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-
-    runs-on: ubuntu-22.04
-    steps:
-
-      # Download literally every single artifact.
-      - uses: actions/download-artifact@v4
-
-      - run: find .
-
-      - name: Setup Pages
-        uses: actions/configure-pages@v4
-      - name: Upload artifact
-        uses: actions/upload-pages-artifact@v3
-        with:
-          # Upload entire repository
-          path: '.'
-      - name: Deploy to GitHub Pages
-        id: deployment
-        uses: actions/deploy-pages@v4

.github/workflows/lint-format.yml

@@ -1,88 +0,0 @@
-name: Lint and Format
-
-on:
-  push:
-    branches: [ master ]
-    tags:
-      - 'v*'
-  pull_request:
-    branches: [ master ]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  wpiformat:
-    name: "wpiformat"
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v3
-      - name: Fetch all history and metadata
-        run: |
-          git fetch --prune --unshallow
-          git checkout -b pr
-          git branch -f master origin/master
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.8
-      - name: Install wpiformat
-        run: pip3 install wpiformat
-      - name: Run
-        run: wpiformat
-      - name: Check output
-        run: git --no-pager diff --exit-code HEAD
-      - name: Generate diff
-        run: git diff HEAD > wpiformat-fixes.patch
-        if: ${{ failure() }}
-      - uses: actions/upload-artifact@v3
-        with:
-          name: wpiformat fixes
-          path: wpiformat-fixes.patch
-        if: ${{ failure() }}
-  javaformat:
-    name: "Java Formatting"
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - uses: actions/setup-java@v3
-        with:
-          java-version: 17
-          distribution: temurin
-      - run: |
-          chmod +x gradlew
-          ./gradlew spotlessCheck
-
-  client-lint-format:
-    name: "PhotonClient Lint and Formatting"
-    defaults:
-      run:
-        working-directory: photon-client
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v3
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: 18
-      - name: Install Dependencies
-        run: npm ci
-      - name: Check Linting
-        run: npm run lint-ci
-      - name: Check Formatting
-        run: npm run format-ci
-  server-index:
-    name: "Check server index.html not changed"
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v3
-      - name: Fetch all history and metadata
-        run: |
-          git fetch --prune --unshallow
-          git checkout -b pr
-          git branch -f master origin/master
-      - name: Check index.html not changed
-        run: git --no-pager diff --exit-code origin/master photon-server/src/main/resources/web/index.html

.github/workflows/python.yml

@@ -1,60 +0,0 @@
-name: Build and Distribute PhotonLibPy
-
-permissions:
-  id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
-
-on:
-  push:
-    branches: [ master ]
-    tags:
-      - 'v*'
-  pull_request:
-    branches: [ master ]
-
-jobs:
-  buildAndDeploy:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: 3.11
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install setuptools wheel pytest
-
-      - name: Build wheel
-        working-directory: ./photon-lib/py
-        run: |
-          python setup.py sdist bdist_wheel
-
-      - name: Run Unit Tests
-        working-directory: ./photon-lib/py
-        run: |
-            pip install --no-cache-dir dist/*.whl
-            pytest
-
-
-      - name: Upload artifacts
-        uses: actions/upload-artifact@master
-        with:
-          name: dist
-          path: ./photon-lib/py/dist/
-
-      - name: Publish package distributions to TestPyPI
-        # Only upload on tags
-        if: startsWith(github.ref, 'refs/tags/v')
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          packages_dir: ./photon-lib/py/dist/
-
-    permissions:
-      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing

.styleguide

@@ -18,6 +18,7 @@ modifiableFileExclude {
   \.dll$
   \.webp$
   \.ico$
+  \.rknn$
   gradlew
 }
 

build.gradle

@@ -1,5 +1,7 @@
+import edu.wpi.first.toolchain.*
+
 plugins {
-    id "com.diffplug.spotless" version "6.22.0"
+    id "com.diffplug.spotless" version "6.24.0"
     id "edu.wpi.first.NativeUtils" version "2024.6.1" apply false
     id "edu.wpi.first.wpilib.repositories.WPILibRepositoriesPlugin" version "2020.2"
     id "edu.wpi.first.GradleRIO" version "2024.1.1"
@@ -11,8 +13,9 @@ allprojects {
     repositories {
         mavenCentral()
         mavenLocal()
-        maven { url = "https://maven.photonvision.org/repository/internal/" }
-        maven { url = "https://maven.photonvision.org/repository/snapshots/" }
+        maven { url = "https://maven.photonvision.org/releases" }
+        maven { url = "https://maven.photonvision.org/snapshots" }
+        maven { url = "https://jogamp.org/deployment/maven/" }
     }
     wpilibRepositories.addAllReleaseRepositories(it)
     wpilibRepositories.addAllDevelopmentRepositories(it)
@@ -28,9 +31,11 @@ ext {
     joglVersion = "2.4.0-rc-20200307"
     javalinVersion = "5.6.2"
     photonGlDriverLibVersion = "dev-v2023.1.0-9-g75fc678"
+    rknnVersion = "dev-v2024.0.0-30-g001b5ec"
     frcYear = "2024"
     mrcalVersion = "dev-v2024.0.0-7-gc976aaa";
 
+
     pubVersion = versionString
     isDev = pubVersion.startsWith("dev")
 
@@ -46,6 +51,10 @@ ext {
     println("Building for platform " + jniPlatform + " wpilib: " + wpilibNativeName)
     println("Using Wpilib: " + wpilibVersion)
     println("Using OpenCV: " + openCVversion)
+
+
+    photonMavenURL = 'https://maven.photonvision.org/' + (isDev ? 'snapshots' : 'releases');
+    println("Publishing Photonlib to " + photonMavenURL)
 }
 
 spotless {
@@ -96,3 +105,7 @@ spotless {
 wrapper {
     gradleVersion '8.4'
 }
+
+ext.getCurrentArch = {
+    return NativePlatforms.desktop
+}

photon-client/index.html

@@ -1,4 +1,4 @@
-<!DOCTYPE html>
+<!doctype html>
 <html lang="en">
   <head>
     <meta charset="UTF-8" />

photon-client/package-lock.json

@@ -31,7 +31,7 @@
         "eslint": "^8.56.0",
         "eslint-plugin-vue": "^9.19.2",
         "npm-run-all": "^4.1.5",
-        "prettier": "^3.1.1",
+        "prettier": "3.2.2",
         "sass": "~1.32",
         "sass-loader": "^13.3.2",
         "terser": "^5.14.2",
@@ -3917,9 +3917,9 @@
       }
     },
     "node_modules/prettier": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.1.1.tgz",
-      "integrity": "sha512-22UbSzg8luF4UuZtzgiUOfcGM8s4tjBv6dJRT7j275NXsy2jb4aJa4NNveul5x4eqlF1wuhuR2RElK71RvmVaw==",
+      "version": "3.2.2",
+      "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.2.2.tgz",
+      "integrity": "sha512-HTByuKZzw7utPiDO523Tt2pLtEyK7OibUD9suEJQrPUCYQqrHr74GGX6VidMrovbf/I50mPqr8j/II6oBAuc5A==",
       "dev": true,
       "bin": {
         "prettier": "bin/prettier.cjs"

photon-client/package.json

@@ -27,17 +27,17 @@
   },
   "devDependencies": {
     "@rushstack/eslint-patch": "^1.3.2",
-    "@vue/eslint-config-prettier": "^9.0.0",
-    "@vue/eslint-config-typescript": "^12.0.0",
-    "prettier": "^3.1.1",
     "@types/node": "^16.11.45",
     "@types/three": "^0.160.0",
     "@vitejs/plugin-vue2": "^2.3.1",
+    "@vue/eslint-config-prettier": "^9.0.0",
+    "@vue/eslint-config-typescript": "^12.0.0",
     "@vue/tsconfig": "^0.5.1",
     "deepmerge": "^4.3.1",
     "eslint": "^8.56.0",
     "eslint-plugin-vue": "^9.19.2",
     "npm-run-all": "^4.1.5",
+    "prettier": "3.2.2",
     "sass": "~1.32",
     "sass-loader": "^13.3.2",
     "terser": "^5.14.2",

photon-client/src/components/dashboard/CameraAndPipelineSelectCard.vue

@@ -7,6 +7,7 @@ import { computed, ref } from "vue";
 import PvIcon from "@/components/common/pv-icon.vue";
 import PvInput from "@/components/common/pv-input.vue";
 import { PipelineType } from "@/types/PipelineTypes";
+import { useSettingsStore } from "@/stores/settings/GeneralSettingsStore";
 
 const changeCurrentCameraIndex = (index: number) => {
   useCameraSettingsStore().setCurrentCameraIndex(index, true);
@@ -24,6 +25,8 @@ const changeCurrentCameraIndex = (index: number) => {
     case PipelineType.Aruco:
       pipelineType.value = WebsocketPipelineType.Aruco;
       break;
+    case PipelineType.ObjectDetection:
+      pipelineType.value = WebsocketPipelineType.ObjectDetection;
   }
 };
 
@@ -154,6 +157,9 @@ const pipelineTypesWrapper = computed<{ name: string; value: number }[]>(() => {
     { name: "AprilTag", value: WebsocketPipelineType.AprilTag },
     { name: "Aruco", value: WebsocketPipelineType.Aruco }
   ];
+  if (useSettingsStore().general.rknnSupported) {
+    pipelineTypes.push({ name: "Object Detection", value: WebsocketPipelineType.ObjectDetection });
+  }
 
   if (useCameraSettingsStore().isDriverMode) {
     pipelineTypes.push({ name: "Driver Mode", value: WebsocketPipelineType.DriverMode });
@@ -208,6 +214,9 @@ useCameraSettingsStore().$subscribe((mutation, state) => {
     case PipelineType.Aruco:
       pipelineType.value = WebsocketPipelineType.Aruco;
       break;
+    case PipelineType.ObjectDetection:
+      pipelineType.value = WebsocketPipelineType.ObjectDetection;
+      break;
   }
 });
 </script>
@@ -354,7 +363,8 @@ useCameraSettingsStore().$subscribe((mutation, state) => {
               { name: 'Reflective', value: WebsocketPipelineType.Reflective },
               { name: 'Colored Shape', value: WebsocketPipelineType.ColoredShape },
               { name: 'AprilTag', value: WebsocketPipelineType.AprilTag },
-              { name: 'Aruco', value: WebsocketPipelineType.Aruco }
+              { name: 'Aruco', value: WebsocketPipelineType.Aruco },
+              { name: 'Object Detection', value: WebsocketPipelineType.ObjectDetection }
             ]"
           />
         </v-card-text>

photon-client/src/components/dashboard/ConfigOptions.vue

@@ -8,6 +8,7 @@ import ThresholdTab from "@/components/dashboard/tabs/ThresholdTab.vue";
 import ContoursTab from "@/components/dashboard/tabs/ContoursTab.vue";
 import AprilTagTab from "@/components/dashboard/tabs/AprilTagTab.vue";
 import ArucoTab from "@/components/dashboard/tabs/ArucoTab.vue";
+import ObjectDetectionTab from "@/components/dashboard/tabs/ObjectDetectionTab.vue";
 import OutputTab from "@/components/dashboard/tabs/OutputTab.vue";
 import TargetsTab from "@/components/dashboard/tabs/TargetsTab.vue";
 import PnPTab from "@/components/dashboard/tabs/PnPTab.vue";
@@ -40,6 +41,10 @@ const allTabs = Object.freeze({
     tabName: "Aruco",
     component: ArucoTab
   },
+  objectDetectionTab: {
+    tabName: "Object Detection",
+    component: ObjectDetectionTab
+  },
   outputTab: {
     tabName: "Output",
     component: OutputTab
@@ -75,6 +80,7 @@ const getTabGroups = (): ConfigOption[][] => {
         allTabs.contoursTab,
         allTabs.apriltagTab,
         allTabs.arucoTab,
+        allTabs.objectDetectionTab,
         allTabs.outputTab
       ],
       [allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab]
@@ -82,14 +88,21 @@ const getTabGroups = (): ConfigOption[][] => {
   } else if (lgAndDown) {
     return [
       [allTabs.inputTab],
-      [allTabs.thresholdTab, allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, allTabs.outputTab],
+      [
+        allTabs.thresholdTab,
+        allTabs.contoursTab,
+        allTabs.apriltagTab,
+        allTabs.arucoTab,
+        allTabs.objectDetectionTab,
+        allTabs.outputTab
+      ],
       [allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab]
     ];
   } else if (xl) {
     return [
       [allTabs.inputTab],
       [allTabs.thresholdTab],
-      [allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, allTabs.outputTab],
+      [allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, allTabs.objectDetectionTab, allTabs.outputTab],
       [allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab]
     ];
   }
@@ -103,17 +116,20 @@ const tabGroups = computed<ConfigOption[][]>(() => {
   const allow3d = useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled;
   const isAprilTag = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.AprilTag;
   const isAruco = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.Aruco;
+  const isObjectDetection =
+    useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.ObjectDetection;
 
   return getTabGroups()
     .map((tabGroup) =>
       tabGroup.filter(
         (tabConfig) =>
           !(!allow3d && tabConfig.tabName === "3D") && //Filter out 3D tab any time 3D isn't calibrated
-          !((!allow3d || isAprilTag || isAruco) && tabConfig.tabName === "PnP") && //Filter out the PnP config tab if 3D isn't available, or we're doing AprilTags
-          !((isAprilTag || isAruco) && tabConfig.tabName === "Threshold") && //Filter out threshold tab if we're doing AprilTags
-          !((isAprilTag || isAruco) && tabConfig.tabName === "Contours") && //Filter out contours if we're doing AprilTags
+          !((!allow3d || isAprilTag || isAruco || isObjectDetection) && tabConfig.tabName === "PnP") && //Filter out the PnP config tab if 3D isn't available, or we're doing AprilTags
+          !((isAprilTag || isAruco || isObjectDetection) && tabConfig.tabName === "Threshold") && //Filter out threshold tab if we're doing AprilTags
+          !((isAprilTag || isAruco || isObjectDetection) && tabConfig.tabName === "Contours") && //Filter out contours if we're doing AprilTags
           !(!isAprilTag && tabConfig.tabName === "AprilTag") && //Filter out apriltag unless we actually are doing AprilTags
-          !(!isAruco && tabConfig.tabName === "Aruco") //Filter out aruco unless we actually are doing Aruco
+          !(!isAruco && tabConfig.tabName === "Aruco") &&
+          !(!isObjectDetection && tabConfig.tabName === "Object Detection") //Filter out aruco unless we actually are doing Aruco
       )
     )
     .filter((it) => it.length); // Remove empty tab groups

photon-client/src/components/dashboard/tabs/ObjectDetectionTab.vue

@@ -0,0 +1,35 @@
+<script setup lang="ts">
+import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
+import { PipelineType } from "@/types/PipelineTypes";
+import PvSlider from "@/components/common/pv-slider.vue";
+import { computed, getCurrentInstance } from "vue";
+import { useStateStore } from "@/stores/StateStore";
+
+// TODO fix pipeline typing in order to fix this, the store settings call should be able to infer that only valid pipeline type settings are exposed based on pre-checks for the entire config section
+// Defer reference to store access method
+const currentPipelineSettings = useCameraSettingsStore().currentPipelineSettings;
+
+const interactiveCols = computed(
+  () =>
+    (getCurrentInstance()?.proxy.$vuetify.breakpoint.mdAndDown || false) &&
+    (!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode)
+)
+  ? 9
+  : 8;
+</script>
+
+<template>
+  <div v-if="currentPipelineSettings.pipelineType === PipelineType.ObjectDetection">
+    <pv-slider
+      v-model="currentPipelineSettings.confidence"
+      class="pt-2"
+      :slider-cols="interactiveCols"
+      label="Confidence"
+      tooltip="The minimum confidence for a detection to be considered valid. Bigger numbers mean fewer but more probable detections are allowed through."
+      :min="0"
+      :max="1"
+      :step="0.01"
+      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ confidence: value }, false)"
+    />
+  </div>
+</template>

photon-client/src/components/dashboard/tabs/TargetsTab.vue

@@ -48,6 +48,10 @@ const resetCurrentBuffer = () => {
               >
                 Fiducial ID
               </th>
+              <template v-if="currentPipelineSettings.pipelineType === PipelineType.ObjectDetection">
+                <th class="text-center white--text">Class</th>
+                <th class="text-center white--text">Confidence</th>
+              </template>
               <template v-if="!useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled">
                 <th class="text-center white--text">Pitch &theta;&deg;</th>
                 <th class="text-center white--text">Yaw &theta;&deg;</th>
@@ -85,6 +89,18 @@ const resetCurrentBuffer = () => {
               >
                 {{ target.fiducialId }}
               </td>
+              <td
+                v-if="currentPipelineSettings.pipelineType === PipelineType.ObjectDetection"
+                class="text-center white--text"
+              >
+                {{ useStateStore().currentPipelineResults?.classNames[target.classId] }}
+              </td>
+              <td
+                v-if="currentPipelineSettings.pipelineType === PipelineType.ObjectDetection"
+                class="text-center white--text"
+              >
+                {{ target.confidence.toFixed(2) }}
+              </td>
               <template v-if="!useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled">
                 <td class="text-center">{{ target.pitch.toFixed(2) }}&deg;</td>
                 <td class="text-center">{{ target.yaw.toFixed(2) }}&deg;</td>

photon-client/src/stores/settings/GeneralSettingsStore.ts

@@ -27,7 +27,8 @@ export const useSettingsStore = defineStore("settings", {
       gpuAcceleration: undefined,
       hardwareModel: undefined,
       hardwarePlatform: undefined,
-      mrCalWorking: true
+      mrCalWorking: true,
+      rknnSupported: false
     },
     network: {
       ntServerAddress: "",
@@ -99,7 +100,8 @@ export const useSettingsStore = defineStore("settings", {
         hardwareModel: data.general.hardwareModel || undefined,
         hardwarePlatform: data.general.hardwarePlatform || undefined,
         gpuAcceleration: data.general.gpuAcceleration || undefined,
-        mrCalWorking: data.general.mrCalWorking
+        mrCalWorking: data.general.mrCalWorking,
+        rknnSupported: data.general.rknnSupported
       };
       this.lighting = data.lighting;
       this.network = data.networkSettings;

photon-client/src/types/PhotonTrackingTypes.ts

@@ -54,6 +54,8 @@ export interface PhotonTarget {
   ambiguity: number;
   // -1 if not set
   fiducialId: number;
+  confidence: number;
+  classId: number;
   // undefined if 3d isn't enabled
   pose?: Transform3d;
 }
@@ -70,4 +72,6 @@ export interface PipelineResult {
   targets: PhotonTarget[];
   // undefined if multitag failed or non-tag pipeline
   multitagResult?: MultitagResult;
+  // Object detection class names -- empty if not doing object detection
+  classNames: string[];
 }

photon-client/src/types/PipelineTypes.ts

@@ -5,7 +5,8 @@ export enum PipelineType {
   Reflective = 2,
   ColoredShape = 3,
   AprilTag = 4,
-  Aruco = 5
+  Aruco = 5,
+  ObjectDetection = 6
 }
 
 export enum AprilTagFamily {
@@ -281,14 +282,39 @@ export const DefaultArucoPipelineSettings: ArucoPipelineSettings = {
   doSingleTargetAlways: false
 };
 
+export interface ObjectDetectionPipelineSettings extends PipelineSettings {
+  pipelineType: PipelineType.ObjectDetection;
+  confidence: number;
+  nms: number;
+  box_thresh: number;
+}
+export type ConfigurableObjectDetectionPipelineSettings = Partial<
+  Omit<ObjectDetectionPipelineSettings, "pipelineType">
+> &
+  ConfigurablePipelineSettings;
+export const DefaultObjectDetectionPipelineSettings: ObjectDetectionPipelineSettings = {
+  ...DefaultPipelineSettings,
+  pipelineType: PipelineType.ObjectDetection,
+  cameraGain: 20,
+  targetModel: TargetModel.InfiniteRechargeHighGoalOuter,
+  ledMode: true,
+  outputShowMultipleTargets: false,
+  cameraExposure: 6,
+  confidence: 0.9,
+  nms: 0.45,
+  box_thresh: 0.25
+};
+
 export type ActivePipelineSettings =
   | ReflectivePipelineSettings
   | ColoredShapePipelineSettings
   | AprilTagPipelineSettings
-  | ArucoPipelineSettings;
+  | ArucoPipelineSettings
+  | ObjectDetectionPipelineSettings;
 
 export type ActiveConfigurablePipelineSettings =
   | ConfigurableReflectivePipelineSettings
   | ConfigurableColoredShapePipelineSettings
   | ConfigurableAprilTagPipelineSettings
-  | ConfigurableArucoPipelineSettings;
+  | ConfigurableArucoPipelineSettings
+  | ConfigurableObjectDetectionPipelineSettings;

photon-client/src/types/SettingTypes.ts

@@ -7,6 +7,7 @@ export interface GeneralSettings {
   hardwareModel?: string;
   hardwarePlatform?: string;
   mrCalWorking: boolean;
+  rknnSupported: boolean;
 }
 
 export interface MetricData {

photon-client/src/types/WebsocketDataTypes.ts

@@ -101,5 +101,6 @@ export enum WebsocketPipelineType {
   Reflective = 0,
   ColoredShape = 1,
   AprilTag = 2,
-  Aruco = 3
+  Aruco = 3,
+  ObjectDetection = 4
 }

photon-client/vite.config.ts

@@ -10,25 +10,22 @@ export default defineConfig({
   plugins: [
     Vue2(),
     Components({
-      resolvers: [
-          VuetifyResolver()
-      ],
+      resolvers: [VuetifyResolver()],
       dts: true,
       transformer: "vue2",
-      types: [{
-        from: "vue-router",
-        names: ["RouterLink", "RouterView"]
-      }],
+      types: [
+        {
+          from: "vue-router",
+          names: ["RouterLink", "RouterView"]
+        }
+      ],
       version: 2.7
     })
   ],
   css: {
     preprocessorOptions: {
       sass: {
-        additionalData: [
-          "@import \"@/assets/styles/variables.scss\"",
-          ""
-        ].join("\n")
+        additionalData: ['@import "@/assets/styles/variables.scss"', ""].join("\n")
       }
     }
   },

photon-core/build.gradle

@@ -1,7 +1,31 @@
+plugins {
+    id 'edu.wpi.first.WpilibTools' version '1.3.0'
+}
+
 import java.nio.file.Path
 
 apply from: "${rootDir}/shared/common.gradle"
 
+wpilibTools.deps.wpilibVersion = wpi.versions.wpilibVersion.get()
+
+def nativeConfigName = 'wpilibNatives'
+def nativeConfig = configurations.create(nativeConfigName)
+
+def nativeTasks = wpilibTools.createExtractionTasks {
+    configurationName = nativeConfigName
+}
+
+nativeTasks.addToSourceSetResources(sourceSets.main)
+
+nativeConfig.dependencies.add wpilibTools.deps.wpilib("wpimath")
+nativeConfig.dependencies.add wpilibTools.deps.wpilib("wpinet")
+nativeConfig.dependencies.add wpilibTools.deps.wpilib("wpiutil")
+nativeConfig.dependencies.add wpilibTools.deps.wpilib("ntcore")
+nativeConfig.dependencies.add wpilibTools.deps.wpilib("cscore")
+nativeConfig.dependencies.add wpilibTools.deps.wpilib("apriltag")
+nativeConfig.dependencies.add wpilibTools.deps.wpilib("hal")
+nativeConfig.dependencies.add wpilibTools.deps.wpilibOpenCv("frc" + wpi.frcYear.get(), wpi.versions.opencvVersion.get())
+
 dependencies {
     // JOGL stuff (currently we only distribute for aarch64, which is Pi 4)
     implementation "org.jogamp.gluegen:gluegen-rt:$joglVersion"
@@ -13,7 +37,9 @@ dependencies {
     implementation 'org.zeroturnaround:zt-zip:1.14'
 
     implementation "org.xerial:sqlite-jdbc:3.41.0.0"
-
+    def rknnjniversion = "dev-v2024.0.0-44-g8022c40"
+    implementation "org.photonvision:rknn_jni-jni:$rknnjniversion:linuxarm64"
+    implementation "org.photonvision:rknn_jni-java:$rknnjniversion"
     implementation "org.photonvision:photon-libcamera-gl-driver-jni:$photonGlDriverLibVersion:linuxarm64"
     implementation "org.photonvision:photon-libcamera-gl-driver-java:$photonGlDriverLibVersion"
 

photon-core/src/main/java/org/photonvision/common/configuration/ConfigManager.java

@@ -296,4 +296,11 @@ public class ConfigManager {
             }
         }
     }
+
+    /** Get (and create if not present) the subfolder where ML models are stored */
+    public File getModelsDirectory() {
+        var ret = new File(configDirectoryFile, "models");
+        if (!ret.exists()) ret.mkdirs();
+        return ret;
+    }
 }

photon-core/src/main/java/org/photonvision/common/configuration/NeuralNetworkModelManager.java

@@ -0,0 +1,98 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.common.configuration;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import org.photonvision.common.logging.LogGroup;
+import org.photonvision.common.logging.Logger;
+
+public class NeuralNetworkModelManager {
+    private static NeuralNetworkModelManager INSTANCE;
+    private static final Logger logger = new Logger(NeuralNetworkModelManager.class, LogGroup.Config);
+
+    private final String MODEL_NAME = "note-640-640-yolov5s.rknn";
+    private File defaultModelFile;
+    private List<String> labels;
+
+    public static NeuralNetworkModelManager getInstance() {
+        if (INSTANCE == null) {
+            INSTANCE = new NeuralNetworkModelManager();
+        }
+        return INSTANCE;
+    }
+
+    /**
+     * Perform initial setup and extract default model from JAR to the filesystem
+     *
+     * @param modelsFolder Where models live
+     */
+    public void initialize(File modelsFolder) {
+        var modelResourcePath = "/models/" + MODEL_NAME;
+        this.defaultModelFile = new File(modelsFolder, MODEL_NAME);
+        extractResource(modelResourcePath, defaultModelFile);
+
+        File labelsFile = new File(modelsFolder, "labels.txt");
+        var labelResourcePath = "/models/" + labelsFile.getName();
+        extractResource(labelResourcePath, labelsFile);
+
+        try {
+            labels = Files.readAllLines(Paths.get(labelsFile.getPath()));
+        } catch (IOException e) {
+            logger.error("Error reading labels.txt", e);
+        }
+    }
+
+    private void extractResource(String resourcePath, File outputFile) {
+        try (var in = NeuralNetworkModelManager.class.getResourceAsStream(resourcePath)) {
+            if (in == null) {
+                logger.error("Failed to find jar resource at " + resourcePath);
+                return;
+            }
+
+            if (!outputFile.exists()) {
+                try (FileOutputStream fos = new FileOutputStream(outputFile)) {
+                    int read = -1;
+                    byte[] buffer = new byte[1024];
+                    while ((read = in.read(buffer)) != -1) {
+                        fos.write(buffer, 0, read);
+                    }
+                } catch (IOException e) {
+                    logger.error("Error extracting resource to " + outputFile.toPath().toString(), e);
+                }
+            } else {
+                logger.info(
+                        "File " + outputFile.toPath().toString() + " already exists. Skipping extraction.");
+            }
+        } catch (IOException e) {
+            logger.error("Error finding jar resource " + resourcePath, e);
+        }
+    }
+
+    public File getDefaultRknnModel() {
+        return defaultModelFile;
+    }
+
+    public List<String> getLabels() {
+        return labels;
+    }
+}

photon-core/src/main/java/org/photonvision/common/configuration/PhotonConfiguration.java

@@ -27,6 +27,7 @@ import org.photonvision.PhotonVersion;
 import org.photonvision.common.hardware.Platform;
 import org.photonvision.common.networking.NetworkUtils;
 import org.photonvision.common.util.SerializationUtils;
+import org.photonvision.jni.RknnDetectorJNI;
 import org.photonvision.mrcal.MrCalJNILoader;
 import org.photonvision.raspi.LibCameraJNILoader;
 import org.photonvision.vision.calibration.CameraCalibrationCoefficients;
@@ -142,7 +143,8 @@ public class PhotonConfiguration {
                 LibCameraJNILoader.isSupported()
                         ? "Zerocopy Libcamera Working"
                         : ""); // TODO add support for other types of GPU accel
-        generalSubmap.put("mrCalWorking", MrCalJNILoader.isWorking());
+        generalSubmap.put("mrCalWorking", MrCalJNILoader.getInstance().isLoaded());
+        generalSubmap.put("rknnSupported", RknnDetectorJNI.getInstance().isLoaded());
         generalSubmap.put("hardwareModel", hardwareConfig.deviceName);
         generalSubmap.put("hardwarePlatform", Platform.getPlatformName());
         settingsSubmap.put("general", generalSubmap);

photon-core/src/main/java/org/photonvision/common/dataflow/websocket/UIDataPublisher.java

@@ -52,6 +52,7 @@ public class UIDataPublisher implements CVPipelineResultConsumer {
             uiTargets.add(t.toHashMap());
         }
         dataMap.put("targets", uiTargets);
+        dataMap.put("classNames", result.objectDetectionClassNames);
 
         // Only send Multitag Results if they are present, similar to 3d pose
         if (result.multiTagResult.estimatedPose.isPresent) {

photon-core/src/main/java/org/photonvision/common/hardware/Platform.java

@@ -43,6 +43,7 @@ public enum Platform {
             true,
             OSType.LINUX,
             true), // Raspberry Pi 3/4 with a 64-bit image
+    LINUX_RK3588_64("Linux AARCH 64-bit with RK3588", "linuxarm64", false, OSType.LINUX, true),
     LINUX_AARCH64(
             "Linux AARCH64", "linuxarm64", false, OSType.LINUX, true), // Jetson Nano, Jetson TX2
 
@@ -94,6 +95,10 @@ public enum Platform {
         return currentPlatform.osType == OSType.LINUX;
     }
 
+    public static boolean isRK3588() {
+        return Platform.isOrangePi() || Platform.isCoolPi4b();
+    }
+
     public static boolean isRaspberryPi() {
         return currentPlatform.isPi;
     }
@@ -186,7 +191,11 @@ public enum Platform {
                 return LINUX_32;
             } else if (RuntimeDetector.isArm64()) {
                 // TODO - os detection needed?
-                return LINUX_AARCH64;
+                if (isOrangePi()) {
+                    return LINUX_RK3588_64;
+                } else {
+                    return LINUX_AARCH64;
+                }
             } else if (RuntimeDetector.isArm32()) {
                 return LINUX_ARM32;
             } else {
@@ -204,6 +213,14 @@ public enum Platform {
         return fileHasText("/proc/cpuinfo", "Raspberry Pi");
     }
 
+    private static boolean isOrangePi() {
+        return fileHasText("/proc/device-tree/model", "Orange Pi 5");
+    }
+
+    private static boolean isCoolPi4b() {
+        return fileHasText("/proc/device-tree/model", "CoolPi 4B");
+    }
+
     private static boolean isJetsonSBC() {
         // https://forums.developer.nvidia.com/t/how-to-recognize-jetson-nano-device/146624
         return fileHasText("/proc/device-tree/model", "NVIDIA Jetson");

photon-core/src/main/java/org/photonvision/common/util/TestUtils.java

@@ -144,6 +144,24 @@ public class TestUtils {
         }
     }
 
+    public enum WPI2024Images {
+        kBackAmpZone_117in,
+        kSpeakerCenter_143in;
+
+        public static double FOV = 68.5;
+
+        public final Path path;
+
+        Path getPath() {
+            var filename = this.toString().substring(1);
+            return Path.of("2024", filename + ".jpg");
+        }
+
+        WPI2024Images() {
+            this.path = getPath();
+        }
+    }
+
     public enum WPI2023Apriltags {
         k162_36_Angle,
         k162_36_Straight,

photon-core/src/main/java/org/photonvision/jni/PhotonJNICommon.java

@@ -26,12 +26,15 @@ import org.photonvision.common.logging.LogGroup;
 import org.photonvision.common.logging.Logger;
 
 public abstract class PhotonJNICommon {
-    static boolean libraryLoaded = false;
+    public abstract boolean isLoaded();
+
+    public abstract void setLoaded(boolean state);
+
     protected static Logger logger = null;
 
-    protected static synchronized void forceLoad(Class<?> clazz, List<String> libraries)
-            throws IOException {
-        if (libraryLoaded) return;
+    protected static synchronized void forceLoad(
+            PhotonJNICommon instance, Class<?> clazz, List<String> libraries) throws IOException {
+        if (instance.isLoaded()) return;
         if (logger == null) logger = new Logger(clazz, LogGroup.Camera);
 
         for (var libraryName : libraries) {
@@ -42,7 +45,7 @@ public abstract class PhotonJNICommon {
                 var in = clazz.getResourceAsStream("/nativelibraries/" + arch_name + "/" + nativeLibName);
 
                 if (in == null) {
-                    libraryLoaded = false;
+                    instance.setLoaded(false);
                     return;
                 }
 
@@ -69,15 +72,11 @@ public abstract class PhotonJNICommon {
                 break;
             }
         }
-        libraryLoaded = true;
+        instance.setLoaded(true);
     }
 
-    protected static synchronized void forceLoad(Class<?> clazz, String libraryName)
-            throws IOException {
-        forceLoad(clazz, List.of(libraryName));
-    }
-
-    public static boolean isWorking() {
-        return libraryLoaded;
+    protected static synchronized void forceLoad(
+            PhotonJNICommon instance, Class<?> clazz, String libraryName) throws IOException {
+        forceLoad(instance, clazz, List.of(libraryName));
     }
 }

photon-core/src/main/java/org/photonvision/jni/RknnDetectorJNI.java

@@ -0,0 +1,138 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.jni;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.stream.Collectors;
+import org.photonvision.common.logging.LogGroup;
+import org.photonvision.common.logging.Logger;
+import org.photonvision.common.util.TestUtils;
+import org.photonvision.rknn.RknnJNI;
+import org.photonvision.rknn.RknnJNI.RknnResult;
+import org.photonvision.vision.opencv.CVMat;
+import org.photonvision.vision.pipe.impl.NeuralNetworkPipeResult;
+
+public class RknnDetectorJNI extends PhotonJNICommon {
+    private static final Logger logger = new Logger(RknnDetectorJNI.class, LogGroup.General);
+    private boolean isLoaded;
+    private static RknnDetectorJNI instance = null;
+
+    private RknnDetectorJNI() {
+        isLoaded = false;
+    }
+
+    public static RknnDetectorJNI getInstance() {
+        if (instance == null) instance = new RknnDetectorJNI();
+
+        return instance;
+    }
+
+    public static synchronized void forceLoad() throws IOException {
+        TestUtils.loadLibraries();
+
+        forceLoad(getInstance(), RknnDetectorJNI.class, List.of("rga", "rknnrt", "rknn_jni"));
+    }
+
+    @Override
+    public boolean isLoaded() {
+        return isLoaded;
+    }
+
+    @Override
+    public void setLoaded(boolean state) {
+        isLoaded = state;
+    }
+
+    public static class RknnObjectDetector {
+        long objPointer = -1;
+        private List<String> labels;
+        private final Object lock = new Object();
+
+        private static final CopyOnWriteArrayList<Long> detectors = new CopyOnWriteArrayList<>();
+
+        public RknnObjectDetector(String modelPath, List<String> labels) {
+            synchronized (lock) {
+                objPointer = RknnJNI.create(modelPath, labels.size());
+                detectors.add(objPointer);
+                System.out.println(
+                        "Created " + objPointer + "! Detectors: " + Arrays.toString(detectors.toArray()));
+            }
+            this.labels = labels;
+        }
+
+        public List<String> getClasses() {
+            return labels;
+        }
+
+        /**
+         * Detect forwards using this model
+         *
+         * @param in The image to process
+         * @param nmsThresh Non-maximum supression threshold. Probably should not change
+         * @param boxThresh Minimum confidence for a box to be added. Basically just confidence
+         *     threshold
+         */
+        public List<NeuralNetworkPipeResult> detect(CVMat in, double nmsThresh, double boxThresh) {
+            RknnResult[] ret;
+            synchronized (lock) {
+                // We can technically be asked to detect and the lock might be acquired _after_ release has
+                // been called. This would mean objPointer would be invalid which would call everything to
+                // explode.
+                if (objPointer > 0) {
+                    ret = RknnJNI.detect(objPointer, in.getMat().getNativeObjAddr(), nmsThresh, boxThresh);
+                } else {
+                    logger.warn("Detect called after destroy -- giving up");
+                    return List.of();
+                }
+            }
+            if (ret == null) {
+                return List.of();
+            }
+            return List.of(ret).stream()
+                    .map(it -> new NeuralNetworkPipeResult(it.rect, it.class_id, it.conf))
+                    .collect(Collectors.toList());
+        }
+
+        public void release() {
+            synchronized (lock) {
+                if (objPointer > 0) {
+                    RknnJNI.destroy(objPointer);
+                    detectors.remove(objPointer);
+                    System.out.println(
+                            "Killed " + objPointer + "! Detectors: " + Arrays.toString(detectors.toArray()));
+                    objPointer = -1;
+                } else {
+                    logger.error("RKNN Detector has already been destroyed!");
+                }
+            }
+        }
+    }
+
+    // public static void createRknnDetector() {
+    //     objPointer =
+    //             RknnJNI.create(
+    //                     NeuralNetworkModelManager.getInstance()
+    //                             .getDefaultRknnModel()
+    //                             .getAbsolutePath()
+    //                             .toString(),
+    //                     NeuralNetworkModelManager.getInstance().getLabels().size());
+    // }
+}

photon-core/src/main/java/org/photonvision/mrcal/MrCalJNILoader.java

@@ -24,6 +24,19 @@ import org.photonvision.common.util.TestUtils;
 import org.photonvision.jni.PhotonJNICommon;
 
 public class MrCalJNILoader extends PhotonJNICommon {
+    private boolean isLoaded;
+    private static MrCalJNILoader instance = null;
+
+    private MrCalJNILoader() {
+        isLoaded = false;
+    }
+
+    public static synchronized MrCalJNILoader getInstance() {
+        if (instance == null) instance = new MrCalJNILoader();
+
+        return instance;
+    }
+
     public static synchronized void forceLoad() throws IOException {
         // Force load opencv
         TestUtils.loadLibraries();
@@ -32,6 +45,7 @@ public class MrCalJNILoader extends PhotonJNICommon {
         if (Platform.isWindows()) {
             // Order is correct to match dependencies of libraries
             forceLoad(
+                    MrCalJNILoader.getInstance(),
                     MrCalJNILoader.class,
                     List.of(
                             "libamd",
@@ -40,17 +54,28 @@ public class MrCalJNILoader extends PhotonJNICommon {
                             "libccolamd",
                             "openblas",
                             "libgcc_s_seh-1",
+                            "libquadmath-0",
                             "libgfortran-5",
                             "liblapack",
                             "libcholmod",
                             "mrcal_jni"));
         } else {
             // Nothing else to do on linux
-            forceLoad(MrCalJNILoader.class, List.of("mrcal_jni"));
+            forceLoad(MrCalJNILoader.getInstance(), MrCalJNILoader.class, List.of("mrcal_jni"));
         }
 
-        if (!MrCalJNILoader.isWorking()) {
+        if (!MrCalJNILoader.getInstance().isLoaded()) {
             throw new IOException("Unable to load mrcal JNI!");
         }
     }
+
+    @Override
+    public boolean isLoaded() {
+        return isLoaded;
+    }
+
+    @Override
+    public void setLoaded(boolean state) {
+        isLoaded = state;
+    }
 }

photon-core/src/main/java/org/photonvision/vision/pipe/CVPipe.java

@@ -33,6 +33,10 @@ public abstract class CVPipe<I, O, P> {
         this.params = params;
     }
 
+    public P getParams() {
+        return this.params;
+    }
+
     /**
      * Runs the process for the pipe.
      *

photon-core/src/main/java/org/photonvision/vision/pipe/impl/ArucoDetectionPipe.java

@@ -44,6 +44,13 @@ public class ArucoDetectionPipe
     @Override
     protected List<ArucoDetectionResult> process(CVMat in) {
         var imgMat = in.getMat();
+
+        // Sanity check -- image should not be empty
+        if (imgMat.empty()) {
+            // give up is best we can do here
+            return List.of();
+        }
+
         var detections = photonDetector.detect(imgMat);
         // manually do corner refinement ourselves
         if (params.useCornerRefinement) {

photon-core/src/main/java/org/photonvision/vision/pipe/impl/Calibrate3dPipe.java

@@ -77,7 +77,7 @@ public class Calibrate3dPipe
 
         CameraCalibrationCoefficients ret;
         var start = System.nanoTime();
-        if (MrCalJNILoader.isWorking() && params.useMrCal) {
+        if (MrCalJNILoader.getInstance().isLoaded() && params.useMrCal) {
             logger.debug("Calibrating with mrcal!");
             ret = calibrateMrcal(in);
         } else {

photon-core/src/main/java/org/photonvision/vision/pipe/impl/Calibrate3dPipeline.java

@@ -15,7 +15,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package org.photonvision.vision.pipeline;
+package org.photonvision.vision.pipe.impl;
 
 import edu.wpi.first.math.util.Units;
 import java.util.ArrayList;
@@ -36,10 +36,10 @@ import org.photonvision.vision.frame.FrameThresholdType;
 import org.photonvision.vision.opencv.CVMat;
 import org.photonvision.vision.opencv.ImageRotationMode;
 import org.photonvision.vision.pipe.CVPipe.CVPipeResult;
-import org.photonvision.vision.pipe.impl.CalculateFPSPipe;
-import org.photonvision.vision.pipe.impl.Calibrate3dPipe;
-import org.photonvision.vision.pipe.impl.FindBoardCornersPipe;
 import org.photonvision.vision.pipe.impl.FindBoardCornersPipe.FindBoardCornersPipeResult;
+import org.photonvision.vision.pipeline.CVPipeline;
+import org.photonvision.vision.pipeline.Calibration3dPipelineSettings;
+import org.photonvision.vision.pipeline.UICalibrationData;
 import org.photonvision.vision.pipeline.result.CVPipelineResult;
 import org.photonvision.vision.pipeline.result.CalibrationPipelineResult;
 

photon-core/src/main/java/org/photonvision/vision/pipe/impl/NeuralNetworkPipeResult.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.vision.pipe.impl;
+
+import org.opencv.core.Rect2d;
+
+public class NeuralNetworkPipeResult {
+    public NeuralNetworkPipeResult(Rect2d box2, Integer classIdx, Float confidence) {
+        box = box2;
+        this.classIdx = classIdx;
+        this.confidence = confidence;
+    }
+
+    public final int classIdx;
+    public final Rect2d box;
+    public final double confidence;
+}

photon-core/src/main/java/org/photonvision/vision/pipe/impl/RknnDetectionPipe.java

@@ -0,0 +1,69 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.vision.pipe.impl;
+
+import java.util.List;
+import org.photonvision.common.configuration.NeuralNetworkModelManager;
+import org.photonvision.jni.RknnDetectorJNI.RknnObjectDetector;
+import org.photonvision.vision.opencv.CVMat;
+import org.photonvision.vision.opencv.Releasable;
+import org.photonvision.vision.pipe.CVPipe;
+
+public class RknnDetectionPipe
+        extends CVPipe<CVMat, List<NeuralNetworkPipeResult>, RknnDetectionPipe.RknnDetectionPipeParams>
+        implements Releasable {
+    private RknnObjectDetector detector;
+
+    public RknnDetectionPipe() {
+        // For now this is hard-coded to defaults. Should be refactored into set pipe params, though.
+        // And ideally a little wrapper helper for only changing native stuff on content change created.
+        this.detector =
+                new RknnObjectDetector(
+                        NeuralNetworkModelManager.getInstance().getDefaultRknnModel().getAbsolutePath(),
+                        NeuralNetworkModelManager.getInstance().getLabels());
+    }
+
+    @Override
+    protected List<NeuralNetworkPipeResult> process(CVMat in) {
+        var frame = in.getMat();
+
+        // Make sure we don't get a weird empty frame
+        if (frame.empty()) {
+            return List.of();
+        }
+
+        return detector.detect(in, params.nms, params.confidence);
+    }
+
+    public static class RknnDetectionPipeParams {
+        public double confidence;
+        public double nms;
+        public int max_detections;
+
+        public RknnDetectionPipeParams() {}
+    }
+
+    public List<String> getClassNames() {
+        return detector.getClasses();
+    }
+
+    @Override
+    public void release() {
+        detector.release();
+    }
+}

photon-core/src/main/java/org/photonvision/vision/pipeline/CVPipeline.java

@@ -21,9 +21,11 @@ import org.photonvision.vision.camera.QuirkyCamera;
 import org.photonvision.vision.frame.Frame;
 import org.photonvision.vision.frame.FrameStaticProperties;
 import org.photonvision.vision.frame.FrameThresholdType;
+import org.photonvision.vision.opencv.Releasable;
 import org.photonvision.vision.pipeline.result.CVPipelineResult;
 
-public abstract class CVPipeline<R extends CVPipelineResult, S extends CVPipelineSettings> {
+public abstract class CVPipeline<R extends CVPipelineResult, S extends CVPipelineSettings>
+        implements Releasable {
     protected S settings;
     protected FrameStaticProperties frameStaticProperties;
     protected QuirkyCamera cameraQuirks;
@@ -75,4 +77,11 @@ public abstract class CVPipeline<R extends CVPipelineResult, S extends CVPipelin
 
         return result;
     }
+
+    /**
+     * Release any native memory associated with this pipeline. Called by pipelinemanager at pipeline
+     * switch. Stubbed out, but override if needed.
+     */
+    @Override
+    public void release() {}
 }

photon-core/src/main/java/org/photonvision/vision/pipeline/CVPipelineSettings.java

@@ -32,7 +32,8 @@ import org.photonvision.vision.opencv.ImageRotationMode;
     @JsonSubTypes.Type(value = ReflectivePipelineSettings.class),
     @JsonSubTypes.Type(value = DriverModePipelineSettings.class),
     @JsonSubTypes.Type(value = AprilTagPipelineSettings.class),
-    @JsonSubTypes.Type(value = ArucoPipelineSettings.class)
+    @JsonSubTypes.Type(value = ArucoPipelineSettings.class),
+    @JsonSubTypes.Type(value = ObjectDetectionPipelineSettings.class)
 })
 public class CVPipelineSettings implements Cloneable {
     public int pipelineIndex = 0;

photon-core/src/main/java/org/photonvision/vision/pipeline/ObjectDetectionPipeline.java

@@ -0,0 +1,94 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.vision.pipeline;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.photonvision.vision.frame.Frame;
+import org.photonvision.vision.frame.FrameThresholdType;
+import org.photonvision.vision.pipe.CVPipe.CVPipeResult;
+import org.photonvision.vision.pipe.impl.*;
+import org.photonvision.vision.pipe.impl.RknnDetectionPipe.RknnDetectionPipeParams;
+import org.photonvision.vision.pipeline.result.CVPipelineResult;
+import org.photonvision.vision.target.TrackedTarget;
+import org.photonvision.vision.target.TrackedTarget.TargetCalculationParameters;
+
+public class ObjectDetectionPipeline
+        extends CVPipeline<CVPipelineResult, ObjectDetectionPipelineSettings> {
+    private final CalculateFPSPipe calculateFPSPipe = new CalculateFPSPipe();
+    private final RknnDetectionPipe rknnPipe = new RknnDetectionPipe();
+
+    private static final FrameThresholdType PROCESSING_TYPE = FrameThresholdType.NONE;
+
+    public ObjectDetectionPipeline() {
+        super(PROCESSING_TYPE);
+        settings = new ObjectDetectionPipelineSettings();
+    }
+
+    public ObjectDetectionPipeline(ObjectDetectionPipelineSettings settings) {
+        super(PROCESSING_TYPE);
+        this.settings = settings;
+    }
+
+    @Override
+    protected void setPipeParamsImpl() {
+        // this needs to be based off of the current backend selected!!
+        var params = new RknnDetectionPipeParams();
+        params.confidence = settings.confidence;
+        params.nms = settings.nms;
+        rknnPipe.setParams(params);
+    }
+
+    @Override
+    protected CVPipelineResult process(Frame input_frame, ObjectDetectionPipelineSettings settings) {
+        long sumPipeNanosElapsed = 0;
+
+        // ***************** change based on backend ***********************
+
+        CVPipeResult<List<NeuralNetworkPipeResult>> ret = rknnPipe.run(input_frame.colorImage);
+        sumPipeNanosElapsed += ret.nanosElapsed;
+        List<NeuralNetworkPipeResult> targetList;
+
+        targetList = ret.output;
+        var names = rknnPipe.getClassNames();
+
+        input_frame.colorImage.getMat().copyTo(input_frame.processedImage.getMat());
+
+        // ***************** change based on backend ***********************
+
+        List<TrackedTarget> targets = new ArrayList<>();
+
+        for (var t : targetList) {
+            targets.add(
+                    new TrackedTarget(
+                            t,
+                            new TargetCalculationParameters(
+                                    false, null, null, null, null, frameStaticProperties)));
+        }
+
+        var fpsResult = calculateFPSPipe.run(null);
+        var fps = fpsResult.output;
+
+        return new CVPipelineResult(sumPipeNanosElapsed, fps, targets, input_frame, names);
+    }
+
+    @Override
+    public void release() {
+        rknnPipe.release();
+    }
+}

photon-core/src/main/java/org/photonvision/vision/pipeline/ObjectDetectionPipelineSettings.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.vision.pipeline;
+
+public class ObjectDetectionPipelineSettings extends AdvancedPipelineSettings {
+    public double confidence;
+    public double nms; // non maximal suppression
+
+    public ObjectDetectionPipelineSettings() {
+        super();
+        this.pipelineType = PipelineType.ObjectDetection; // TODO: FIX this
+        this.outputShowMultipleTargets = true;
+        cameraExposure = 20;
+        cameraAutoExposure = false;
+        ledMode = false;
+        confidence = .9;
+        nms = .45;
+    }
+}

photon-core/src/main/java/org/photonvision/vision/pipeline/PipelineType.java

@@ -17,6 +17,8 @@
 
 package org.photonvision.vision.pipeline;
 
+import org.photonvision.vision.pipe.impl.Calibrate3dPipeline;
+
 @SuppressWarnings("rawtypes")
 public enum PipelineType {
     Calib3d(-2, Calibrate3dPipeline.class),
@@ -24,7 +26,8 @@ public enum PipelineType {
     Reflective(0, ReflectivePipeline.class),
     ColoredShape(1, ColoredShapePipeline.class),
     AprilTag(2, AprilTagPipeline.class),
-    Aruco(3, ArucoPipeline.class);
+    Aruco(3, ArucoPipeline.class),
+    ObjectDetection(4, ObjectDetectionPipeline.class);
 
     public final int baseIndex;
     public final Class clazz;

photon-core/src/main/java/org/photonvision/vision/pipeline/result/CVPipelineResult.java

@@ -32,10 +32,20 @@ public class CVPipelineResult implements Releasable {
     public final List<TrackedTarget> targets;
     public final Frame inputAndOutputFrame;
     public MultiTargetPNPResult multiTagResult;
+    public final List<String> objectDetectionClassNames;
 
     public CVPipelineResult(
             double processingNanos, double fps, List<TrackedTarget> targets, Frame inputFrame) {
-        this(processingNanos, fps, targets, new MultiTargetPNPResult(), inputFrame);
+        this(processingNanos, fps, targets, new MultiTargetPNPResult(), inputFrame, List.of());
+    }
+
+    public CVPipelineResult(
+            double processingNanos,
+            double fps,
+            List<TrackedTarget> targets,
+            Frame inputFrame,
+            List<String> classNames) {
+        this(processingNanos, fps, targets, new MultiTargetPNPResult(), inputFrame, classNames);
     }
 
     public CVPipelineResult(
@@ -44,10 +54,21 @@ public class CVPipelineResult implements Releasable {
             List<TrackedTarget> targets,
             MultiTargetPNPResult multiTagResult,
             Frame inputFrame) {
+        this(processingNanos, fps, targets, multiTagResult, inputFrame, List.of());
+    }
+
+    public CVPipelineResult(
+            double processingNanos,
+            double fps,
+            List<TrackedTarget> targets,
+            MultiTargetPNPResult multiTagResult,
+            Frame inputFrame,
+            List<String> classNames) {
         this.processingNanos = processingNanos;
         this.fps = fps;
         this.targets = targets != null ? targets : Collections.emptyList();
         this.multiTagResult = multiTagResult;
+        this.objectDetectionClassNames = classNames;
 
         this.inputAndOutputFrame = inputFrame;
     }
@@ -57,7 +78,7 @@ public class CVPipelineResult implements Releasable {
             double fps,
             List<TrackedTarget> targets,
             MultiTargetPNPResult multiTagResult) {
-        this(processingNanos, fps, targets, multiTagResult, null);
+        this(processingNanos, fps, targets, multiTagResult, null, List.of());
     }
 
     public boolean hasTargets() {

photon-core/src/main/java/org/photonvision/vision/processes/PipelineManager.java

@@ -27,6 +27,7 @@ import org.photonvision.common.dataflow.DataChangeService;
 import org.photonvision.common.dataflow.events.OutgoingUIEvent;
 import org.photonvision.common.logging.LogGroup;
 import org.photonvision.common.logging.Logger;
+import org.photonvision.vision.pipe.impl.Calibrate3dPipeline;
 import org.photonvision.vision.pipeline.*;
 
 @SuppressWarnings({"rawtypes", "unused"})
@@ -41,7 +42,7 @@ public class PipelineManager {
     protected final DriverModePipeline driverModePipeline = new DriverModePipeline();
 
     /** Index of the currently active pipeline. Defaults to 0. */
-    private int currentPipelineIndex = 0;
+    private int currentPipelineIndex = DRIVERMODE_INDEX;
 
     /** The currently active pipeline. */
     private CVPipeline currentUserPipeline = driverModePipeline;
@@ -188,6 +189,11 @@ public class PipelineManager {
             return;
         }
 
+        // Cleanup potential old native resources before swapping over
+        if (currentUserPipeline != null) {
+            currentUserPipeline.release();
+        }
+
         currentPipelineIndex = newIndex;
         if (newIndex >= 0) {
             var desiredPipelineSettings = userPipelineSettings.get(currentPipelineIndex);
@@ -212,6 +218,11 @@ public class PipelineManager {
                     logger.debug("Creating Aruco Pipeline");
                     currentUserPipeline = new ArucoPipeline((ArucoPipelineSettings) desiredPipelineSettings);
                     break;
+                case ObjectDetection:
+                    logger.debug("Creating ObjectDetection Pipeline");
+                    currentUserPipeline =
+                            new ObjectDetectionPipeline(
+                                    (ObjectDetectionPipelineSettings) desiredPipelineSettings);
                 default:
                     // Can be calib3d or drivermode, both of which are special cases
                     break;
@@ -313,6 +324,12 @@ public class PipelineManager {
                     added.pipelineNickname = nickname;
                     return added;
                 }
+            case ObjectDetection:
+                {
+                    var added = new ObjectDetectionPipelineSettings();
+                    added.pipelineNickname = nickname;
+                    return added;
+                }
             default:
                 {
                     logger.error("Got invalid pipeline type: " + type);

photon-core/src/main/java/org/photonvision/vision/target/TrackedTarget.java

@@ -27,6 +27,7 @@ import org.opencv.core.Mat;
 import org.opencv.core.MatOfPoint;
 import org.opencv.core.MatOfPoint2f;
 import org.opencv.core.Point;
+import org.opencv.core.Rect2d;
 import org.opencv.core.RotatedRect;
 import org.photonvision.common.util.SerializationUtils;
 import org.photonvision.common.util.math.MathUtils;
@@ -38,6 +39,7 @@ import org.photonvision.vision.opencv.CVShape;
 import org.photonvision.vision.opencv.Contour;
 import org.photonvision.vision.opencv.DualOffsetValues;
 import org.photonvision.vision.opencv.Releasable;
+import org.photonvision.vision.pipe.impl.NeuralNetworkPipeResult;
 
 public class TrackedTarget implements Releasable {
     public final Contour m_mainContour;
@@ -65,6 +67,9 @@ public class TrackedTarget implements Releasable {
 
     private Mat m_cameraRelativeTvec, m_cameraRelativeRvec;
 
+    private int m_classId = -1;
+    private double m_confidence = -1;
+
     public TrackedTarget(
             PotentialTarget origTarget, TargetCalculationParameters params, CVShape shape) {
         this.m_mainContour = origTarget.m_mainContour;
@@ -154,6 +159,61 @@ public class TrackedTarget implements Releasable {
         m_robotOffsetPoint = new Point();
     }
 
+    public TrackedTarget(
+            Rect2d box, int class_id, double confidence, TargetCalculationParameters params) {
+        m_targetOffsetPoint = new Point(box.x + box.width / 2.0, box.y + box.height / 2.0);
+        m_robotOffsetPoint = new Point();
+
+        var yawPitch =
+                TargetCalculations.calculateYawPitch(
+                        params.cameraCenterPoint.x,
+                        box.x + box.width / 2.0,
+                        params.horizontalFocalLength,
+                        params.cameraCenterPoint.y,
+                        box.y + box.height / 2.0,
+                        params.verticalFocalLength);
+        m_yaw = yawPitch.getFirst();
+        m_pitch = yawPitch.getSecond();
+
+        Point[] cornerPoints =
+                new Point[] {
+                    // Box.x/y is the top-left corner, not the center
+                    new Point(box.x, box.y), // tl
+                    new Point(box.x + box.width, box.y), // tr
+                    new Point(box.x + box.width, box.y + box.height), // br
+                    new Point(box.x, box.y + box.height), // bl
+                };
+
+        m_targetCorners = List.of(cornerPoints);
+        MatOfPoint contourMat = new MatOfPoint(cornerPoints);
+        m_approximateBoundingPolygon = new MatOfPoint2f(cornerPoints);
+
+        m_mainContour = new Contour(contourMat);
+        m_area = m_mainContour.getArea() / params.imageArea * 100;
+
+        m_classId = class_id;
+        m_confidence = confidence;
+    }
+
+    public TrackedTarget(
+            NeuralNetworkPipeResult t, TargetCalculationParameters targetCalculationParameters) {
+        this(t.box, t.classIdx, t.confidence, targetCalculationParameters);
+    }
+
+    /**
+     * @return Returns the confidence of the detection ranging from 0 - 1.
+     */
+    public double getConfidence() {
+        return m_confidence;
+    }
+
+    /**
+     * @return O-indexed class index for the detected object.
+     */
+    public double getClassID() {
+        return m_classId;
+    }
+
     public TrackedTarget(
             ArucoDetectionResult result,
             AprilTagPoseEstimate tagPose,
@@ -388,6 +448,8 @@ public class TrackedTarget implements Releasable {
         ret.put("skew", getSkew());
         ret.put("area", getArea());
         ret.put("ambiguity", getPoseAmbiguity());
+        ret.put("confidence", m_confidence);
+        ret.put("classId", m_classId);
 
         var bestCameraToTarget3d = getBestCameraToTarget3d();
         if (bestCameraToTarget3d != null) {

photon-core/src/test/java/org/photonvision/vision/pipeline/Calibrate3dPipeTest.java

@@ -44,6 +44,7 @@ import org.photonvision.vision.frame.FrameDivisor;
 import org.photonvision.vision.frame.FrameStaticProperties;
 import org.photonvision.vision.frame.FrameThresholdType;
 import org.photonvision.vision.opencv.CVMat;
+import org.photonvision.vision.pipe.impl.Calibrate3dPipeline;
 
 public class Calibrate3dPipeTest {
     @BeforeAll

photon-lib/build.gradle

@@ -1,9 +1,3 @@
-plugins {
-    id 'edu.wpi.first.WpilibTools' version '1.3.0'
-}
-
-apply plugin: "edu.wpi.first.NativeUtils"
-
 import java.nio.file.Path
 
 ext {
@@ -13,9 +7,125 @@ ext {
     generatedHeaders = "src/generate/native/include"
 }
 
-apply from: "${rootDir}/shared/javacpp/setupBuild.gradle"
+apply plugin: 'cpp'
+apply plugin: 'google-test-test-suite'
+apply plugin: 'edu.wpi.first.NativeUtils'
+
+apply from: "${rootDir}/shared/config.gradle"
+apply from: "${rootDir}/shared/javacommon.gradle"
+
 apply from: "${rootDir}/versioningHelper.gradle"
 
+nativeUtils {
+    exportsConfigs {
+        "${nativeName}" {}
+    }
+}
+
+model {
+    components {
+        "${nativeName}"(NativeLibrarySpec) {
+            sources {
+                cpp {
+                    source {
+                        srcDirs 'src/main/native/cpp', "$buildDir/generated/source/proto/main/cpp"
+                        include '**/*.cpp', '**/*.cc'
+                    }
+                    exportedHeaders {
+                        srcDirs 'src/main/native/include', "$buildDir/generated/source/proto/main/cpp"
+                        if (project.hasProperty('generatedHeaders')) {
+                            srcDir generatedHeaders
+                        }
+                        include "**/*.h"
+                    }
+                }
+            }
+
+            binaries.all {
+                it.tasks.withType(CppCompile) {
+                    it.dependsOn generateProto
+                }
+                if(project.hasProperty('includePhotonTargeting')) {
+                    lib project: ':photon-targeting', library: 'photontargeting', linkage: 'shared'
+                }
+            }
+
+            nativeUtils.useRequiredLibrary(it, "wpilib_shared")
+            nativeUtils.useRequiredLibrary(it, "apriltag_shared")
+            nativeUtils.useRequiredLibrary(it, "opencv_shared")
+        }
+    }
+    testSuites {
+        "${nativeName}Test"(GoogleTestTestSuiteSpec) {
+            for(NativeComponentSpec c : $.components) {
+                if (c.name == nativeName) {
+                    testing c
+                    break
+                }
+            }
+            sources {
+                cpp {
+                    source {
+                        srcDirs 'src/test/native/cpp'
+                        include '**/*.cpp'
+                    }
+                    exportedHeaders {
+                        srcDirs 'src/test/native/include', "$buildDir/generated/source/proto/main/cpp"
+                    }
+                }
+            }
+
+            binaries.all {
+                it.tasks.withType(CppCompile) {
+                    it.dependsOn generateProto
+                }
+                if(project.hasProperty('includePhotonTargeting')) {
+                    lib project: ':photon-targeting', library: 'photontargeting', linkage: 'shared'
+                }
+            }
+
+            nativeUtils.useRequiredLibrary(it, "cscore_shared")
+            nativeUtils.useRequiredLibrary(it, "cameraserver_shared")
+            nativeUtils.useRequiredLibrary(it, "wpilib_executable_shared")
+            nativeUtils.useRequiredLibrary(it, "googletest_static")
+            nativeUtils.useRequiredLibrary(it, "apriltag_shared")
+            nativeUtils.useRequiredLibrary(it, "opencv_shared")
+        }
+    }
+
+    tasks {
+        def c = $.testSuites
+        project.tasks.create('runCpp', Exec) {
+            description = "Run the photon-lib executable"
+            def found = false
+            def systemArch = getCurrentArch()
+            c.each {
+                if (it in GoogleTestTestSuiteSpec && it.name == "${nativeName}Test") {
+                    it.binaries.each {
+                        if (!found) {
+                            def arch = it.targetPlatform.name
+                            if (arch == systemArch) {
+                                dependsOn it.tasks.install
+                                commandLine it.tasks.install.runScriptFile.get().asFile.toString()
+                                def filePath = it.tasks.install.installDirectory.get().toString() + File.separatorChar + 'lib'
+                                test.dependsOn it.tasks.install
+                                test.systemProperty 'java.library.path', filePath
+                                test.environment 'LD_LIBRARY_PATH', filePath
+                                test.environment 'DYLD_LIBRARY_PATH', filePath
+                                test.workingDir filePath
+
+                                found = true
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+apply from: "${rootDir}/shared/javacpp/publish.gradle"
+
 // Include the version file in the distributed sources
 cppHeadersZip {
     from('src/generate/native/include') {

photon-lib/py/setup.py

@@ -16,10 +16,24 @@ m = re.search(
 # which should be PEP440 compliant
 if m:
     versionString = m.group(0)
-    prefix = m.group(1)
-    maturity = m.group(2)
-    suffix = m.group(3).replace(".", "")
-    versionString = f"{prefix}.{maturity}.{suffix}"
+    # Hack -- for strings like v2024.1.1, do NOT add matruity/suffix
+    if len(m.group(2)) > 0:
+        print("using beta group matcher")
+        prefix = m.group(1)
+        maturity = m.group(2)
+        suffix = m.group(3).replace(".", "")
+        versionString = f"{prefix}.{maturity}.{suffix}"
+    else:
+        split = gitDescribeResult.split("-")
+        if len(split) == 3:
+            year, commits, sha = split
+            # Chop off leading v from "v2024.1.2", and use "post" for commits to master since
+            versionString = f"{year[1:]}post{commits}"
+            print("using dev release " + versionString)
+        else:
+            year = gitDescribeResult
+            versionString = year[1:]
+            print("using full release " + versionString)
 
 
 else:

photon-lib/src/generate/photonlib.json.in

@@ -8,7 +8,7 @@
     "https://maven.photonvision.org/repository/internal",
     "https://maven.photonvision.org/repository/snapshots"
   ],
-  "jsonUrl": "https://maven.photonvision.org/repository/internal/org/photonvision/PhotonLib-json/1.0/PhotonLib-json-1.0.json",
+  "jsonUrl": "https://maven.photonvision.org/repository/internal/org/photonvision/photonlib-json/1.0/photonlib-json-1.0.json",
   "jniDependencies": [],
   "cppDependencies": [
     {

photon-lib/src/test/java/org/photonvision/OpenCVTest.java

@@ -26,31 +26,23 @@ package org.photonvision;
 
 import static org.junit.jupiter.api.Assertions.*;
 
-import edu.wpi.first.apriltag.jni.AprilTagJNI;
 import edu.wpi.first.cscore.CameraServerCvJNI;
-import edu.wpi.first.cscore.CameraServerJNI;
-import edu.wpi.first.hal.JNIWrapper;
 import edu.wpi.first.math.MathUtil;
 import edu.wpi.first.math.geometry.Pose3d;
 import edu.wpi.first.math.geometry.Rotation2d;
 import edu.wpi.first.math.geometry.Rotation3d;
 import edu.wpi.first.math.geometry.Transform3d;
 import edu.wpi.first.math.geometry.Translation3d;
-import edu.wpi.first.net.WPINetJNI;
 import edu.wpi.first.networktables.NetworkTableInstance;
-import edu.wpi.first.networktables.NetworkTablesJNI;
-import edu.wpi.first.util.CombinedRuntimeLoader;
-import edu.wpi.first.util.WPIUtilJNI;
+import java.io.IOException;
 import java.util.List;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
-import org.opencv.core.Core;
 import org.photonvision.estimation.CameraTargetRelation;
 import org.photonvision.estimation.OpenCVHelp;
 import org.photonvision.estimation.RotTrlTransform3d;
 import org.photonvision.estimation.TargetModel;
 import org.photonvision.simulation.SimCameraProperties;
-import org.photonvision.simulation.VisionSystemSim;
 import org.photonvision.simulation.VisionTargetSim;
 
 public class OpenCVTest {
@@ -84,28 +76,8 @@ public class OpenCVTest {
     private static final SimCameraProperties prop = new SimCameraProperties();
 
     @BeforeAll
-    public static void setUp() {
-        JNIWrapper.Helper.setExtractOnStaticLoad(false);
-        WPIUtilJNI.Helper.setExtractOnStaticLoad(false);
-        NetworkTablesJNI.Helper.setExtractOnStaticLoad(false);
-        WPINetJNI.Helper.setExtractOnStaticLoad(false);
-        CameraServerJNI.Helper.setExtractOnStaticLoad(false);
-        CameraServerCvJNI.Helper.setExtractOnStaticLoad(false);
-        AprilTagJNI.Helper.setExtractOnStaticLoad(false);
-
-        try {
-            CombinedRuntimeLoader.loadLibraries(
-                    VisionSystemSim.class,
-                    "wpiutiljni",
-                    "ntcorejni",
-                    "wpinetjni",
-                    "wpiHaljni",
-                    Core.NATIVE_LIBRARY_NAME,
-                    "cscorejni",
-                    "apriltagjni");
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
+    public static void setUp() throws IOException {
+        CameraServerCvJNI.forceLoad();
 
         // NT live for debug purposes
         NetworkTableInstance.getDefault().startServer();

photon-lib/src/test/java/org/photonvision/PhotonPoseEstimatorTest.java

@@ -30,17 +30,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import edu.wpi.first.apriltag.AprilTag;
 import edu.wpi.first.apriltag.AprilTagFieldLayout;
-import edu.wpi.first.hal.JNIWrapper;
 import edu.wpi.first.math.geometry.Pose3d;
 import edu.wpi.first.math.geometry.Rotation3d;
 import edu.wpi.first.math.geometry.Transform3d;
 import edu.wpi.first.math.geometry.Translation3d;
 import edu.wpi.first.math.util.Units;
-import edu.wpi.first.net.WPINetJNI;
-import edu.wpi.first.networktables.NetworkTablesJNI;
-import edu.wpi.first.util.CombinedRuntimeLoader;
-import edu.wpi.first.util.WPIUtilJNI;
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
@@ -56,19 +50,6 @@ class PhotonPoseEstimatorTest {
 
     @BeforeAll
     public static void init() {
-        JNIWrapper.Helper.setExtractOnStaticLoad(false);
-        WPIUtilJNI.Helper.setExtractOnStaticLoad(false);
-        NetworkTablesJNI.Helper.setExtractOnStaticLoad(false);
-        WPINetJNI.Helper.setExtractOnStaticLoad(false);
-
-        try {
-            CombinedRuntimeLoader.loadLibraries(
-                    PhotonPoseEstimatorTest.class, "wpiutiljni", "ntcorejni", "wpinetjni", "wpiHaljni");
-        } catch (IOException e) {
-            // TODO Auto-generated catch block
-            e.printStackTrace();
-        }
-
         List<AprilTag> tagList = new ArrayList<>(2);
         tagList.add(new AprilTag(0, new Pose3d(3, 3, 3, new Rotation3d())));
         tagList.add(new AprilTag(1, new Pose3d(5, 5, 5, new Rotation3d())));

photon-lib/src/test/java/org/photonvision/VisionSystemSimTest.java

@@ -30,10 +30,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import edu.wpi.first.apriltag.AprilTag;
 import edu.wpi.first.apriltag.AprilTagFieldLayout;
-import edu.wpi.first.apriltag.jni.AprilTagJNI;
-import edu.wpi.first.cscore.CameraServerCvJNI;
-import edu.wpi.first.cscore.CameraServerJNI;
-import edu.wpi.first.hal.JNIWrapper;
 import edu.wpi.first.math.geometry.Pose2d;
 import edu.wpi.first.math.geometry.Pose3d;
 import edu.wpi.first.math.geometry.Rotation2d;
@@ -42,11 +38,7 @@ import edu.wpi.first.math.geometry.Transform3d;
 import edu.wpi.first.math.geometry.Translation2d;
 import edu.wpi.first.math.geometry.Translation3d;
 import edu.wpi.first.math.util.Units;
-import edu.wpi.first.net.WPINetJNI;
 import edu.wpi.first.networktables.NetworkTableInstance;
-import edu.wpi.first.networktables.NetworkTablesJNI;
-import edu.wpi.first.util.CombinedRuntimeLoader;
-import edu.wpi.first.util.WPIUtilJNI;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Stream;
@@ -58,7 +50,6 @@ import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
 import org.junit.jupiter.params.provider.ValueSource;
-import org.opencv.core.Core;
 import org.photonvision.estimation.TargetModel;
 import org.photonvision.estimation.VisionEstimation;
 import org.photonvision.simulation.PhotonCameraSim;
@@ -85,28 +76,6 @@ class VisionSystemSimTest {
 
     @BeforeAll
     public static void setUp() {
-        JNIWrapper.Helper.setExtractOnStaticLoad(false);
-        WPIUtilJNI.Helper.setExtractOnStaticLoad(false);
-        NetworkTablesJNI.Helper.setExtractOnStaticLoad(false);
-        WPINetJNI.Helper.setExtractOnStaticLoad(false);
-        CameraServerJNI.Helper.setExtractOnStaticLoad(false);
-        CameraServerCvJNI.Helper.setExtractOnStaticLoad(false);
-        AprilTagJNI.Helper.setExtractOnStaticLoad(false);
-
-        try {
-            CombinedRuntimeLoader.loadLibraries(
-                    VisionSystemSim.class,
-                    "wpiutiljni",
-                    "ntcorejni",
-                    "wpinetjni",
-                    "wpiHaljni",
-                    Core.NATIVE_LIBRARY_NAME,
-                    "cscorejni",
-                    "apriltagjni");
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-
         // NT live for debug purposes
         NetworkTableInstance.getDefault().startServer();
 

photon-lib/src/test/java/org/photonvision/estimation/ApriltagWorkbenchTest.java

@@ -26,14 +26,8 @@ package org.photonvision.estimation;
 
 import edu.wpi.first.apriltag.AprilTagFieldLayout;
 import edu.wpi.first.apriltag.AprilTagFields;
-import edu.wpi.first.cscore.CameraServerCvJNI;
-import edu.wpi.first.hal.JNIWrapper;
 import edu.wpi.first.math.geometry.Transform3d;
-import edu.wpi.first.net.WPINetJNI;
 import edu.wpi.first.networktables.NetworkTableInstance;
-import edu.wpi.first.networktables.NetworkTablesJNI;
-import edu.wpi.first.util.CombinedRuntimeLoader;
-import edu.wpi.first.util.WPIUtilJNI;
 import edu.wpi.first.wpilibj.smartdashboard.Field2d;
 import edu.wpi.first.wpilibj.smartdashboard.SmartDashboard;
 import java.io.IOException;
@@ -44,25 +38,6 @@ import org.photonvision.PhotonPoseEstimator;
 public class ApriltagWorkbenchTest {
     @BeforeAll
     public static void setUp() {
-        JNIWrapper.Helper.setExtractOnStaticLoad(false);
-        WPIUtilJNI.Helper.setExtractOnStaticLoad(false);
-        NetworkTablesJNI.Helper.setExtractOnStaticLoad(false);
-        WPINetJNI.Helper.setExtractOnStaticLoad(false);
-        CameraServerCvJNI.Helper.setExtractOnStaticLoad(false);
-
-        try {
-            CombinedRuntimeLoader.loadLibraries(
-                    ApriltagWorkbenchTest.class,
-                    "wpiutiljni",
-                    "ntcorejni",
-                    "wpinetjni",
-                    "wpiHaljni",
-                    "cscorejnicvstatic");
-        } catch (Exception e) {
-            // TODO Auto-generated catch block
-            e.printStackTrace();
-        }
-
         // No version check for testing
         PhotonCamera.setVersionCheckEnabled(false);
     }

photon-server/src/main/java/org/photonvision/Main.java

@@ -27,6 +27,7 @@ import java.util.stream.Collectors;
 import org.apache.commons.cli.*;
 import org.photonvision.common.configuration.CameraConfiguration;
 import org.photonvision.common.configuration.ConfigManager;
+import org.photonvision.common.configuration.NeuralNetworkModelManager;
 import org.photonvision.common.dataflow.networktables.NetworkTablesManager;
 import org.photonvision.common.hardware.HardwareManager;
 import org.photonvision.common.hardware.PiVersion;
@@ -37,9 +38,11 @@ import org.photonvision.common.logging.Logger;
 import org.photonvision.common.networking.NetworkManager;
 import org.photonvision.common.util.TestUtils;
 import org.photonvision.common.util.numbers.IntegerCouple;
+import org.photonvision.jni.RknnDetectorJNI;
 import org.photonvision.mrcal.MrCalJNILoader;
 import org.photonvision.raspi.LibCameraJNILoader;
 import org.photonvision.server.Server;
+import org.photonvision.vision.apriltag.AprilTagFamily;
 import org.photonvision.vision.camera.FileVisionSource;
 import org.photonvision.vision.opencv.CVMat;
 import org.photonvision.vision.opencv.ContourGroupingMode;
@@ -261,6 +264,34 @@ public class Main {
             camConf2023.pipelineSettings = psList2023;
         }
 
+        CameraConfiguration camConf2024 =
+                ConfigManager.getInstance().getConfig().getCameraConfigurations().get("WPI2024");
+        if (camConf2024 == null || true) {
+            camConf2024 =
+                    new CameraConfiguration(
+                            "WPI2024",
+                            TestUtils.getResourcesFolderPath(true)
+                                    .resolve("testimages")
+                                    .resolve(TestUtils.WPI2024Images.kSpeakerCenter_143in.path)
+                                    .toString());
+
+            camConf2024.FOV = TestUtils.WPI2024Images.FOV;
+            // same camera as 2023
+            camConf2024.calibrations.add(TestUtils.get2023LifeCamCoeffs(true));
+
+            var pipeline2024 = new AprilTagPipelineSettings();
+            var path_split = Path.of(camConf2024.path).getFileName().toString();
+            pipeline2024.pipelineNickname = path_split.replace(".jpg", "");
+            pipeline2024.targetModel = TargetModel.kAprilTag6p5in_36h11;
+            pipeline2024.tagFamily = AprilTagFamily.kTag36h11;
+            pipeline2024.inputShouldShow = true;
+            pipeline2024.solvePNPEnabled = true;
+
+            var psList2024 = new ArrayList<CVPipelineSettings>();
+            psList2024.add(pipeline2024);
+            camConf2024.pipelineSettings = psList2024;
+        }
+
         // Colored shape testing
         var camConfShape =
                 ConfigManager.getInstance().getConfig().getCameraConfigurations().get("Shape");
@@ -290,12 +321,14 @@ public class Main {
         var fvs2020 = new FileVisionSource(camConf2020);
         var fvs2022 = new FileVisionSource(camConf2022);
         var fvs2023 = new FileVisionSource(camConf2023);
+        var fvs2024 = new FileVisionSource(camConf2024);
 
-        collectedSources.add(fvs2023);
-        collectedSources.add(fvs2022);
-        collectedSources.add(fvsShape);
-        collectedSources.add(fvs2020);
-        collectedSources.add(fvs2019);
+        collectedSources.add(fvs2024);
+        // collectedSources.add(fvs2023);
+        // collectedSources.add(fvs2022);
+        // collectedSources.add(fvsShape);
+        // collectedSources.add(fvs2020);
+        // collectedSources.add(fvs2019);
 
         ConfigManager.getInstance().unloadCameraConfigs();
         VisionModuleManager.getInstance().addSources(collectedSources).forEach(VisionModule::start);
@@ -317,7 +350,15 @@ public class Main {
         } catch (IOException e) {
             logger.error("Failed to load libcamera-JNI!", e);
         }
-
+        try {
+            if (Platform.isRK3588()) {
+                RknnDetectorJNI.forceLoad();
+            } else {
+                logger.error("Platform does not support RKNN based machine learning!");
+            }
+        } catch (IOException e) {
+            logger.error("Failed to load rknn-JNI!", e);
+        }
         try {
             MrCalJNILoader.forceLoad();
         } catch (IOException e) {
@@ -333,7 +374,6 @@ public class Main {
         } catch (ParseException e) {
             logger.error("Failed to parse command-line options!", e);
         }
-
         CVMat.enablePrint(false);
         PipelineProfiler.enablePrint(false);
 
@@ -368,6 +408,10 @@ public class Main {
         NetworkTablesManager.getInstance()
                 .setConfig(ConfigManager.getInstance().getConfig().getNetworkConfig());
 
+        logger.info("Loading ML models");
+        NeuralNetworkModelManager.getInstance()
+                .initialize(ConfigManager.getInstance().getModelsDirectory());
+
         if (!isTestMode) {
             logger.debug("Loading VisionSourceManager...");
             VisionSourceManager.getInstance()

photon-server/src/main/resources/models/labels.txt

@@ -0,0 +1 @@
+note

photon-targeting/build.gradle

@@ -1,9 +1,122 @@
-plugins {
-    id 'edu.wpi.first.WpilibTools' version '1.3.0'
-}
-
 ext {
     nativeName = "photontargeting"
 }
 
-apply from: "${rootDir}/shared/javacpp/setupBuild.gradle"
+apply plugin: 'cpp'
+apply plugin: 'google-test-test-suite'
+apply plugin: 'edu.wpi.first.NativeUtils'
+
+apply from: "${rootDir}/shared/config.gradle"
+apply from: "${rootDir}/shared/javacommon.gradle"
+
+apply from: "${rootDir}/versioningHelper.gradle"
+
+nativeUtils {
+    exportsConfigs {
+        "${nativeName}" {}
+    }
+}
+
+model {
+    components {
+        "${nativeName}"(NativeLibrarySpec) {
+            sources {
+                cpp {
+                    source {
+                        srcDirs 'src/main/native/cpp', "$buildDir/generated/source/proto/main/cpp"
+                        include '**/*.cpp', '**/*.cc'
+                    }
+                    exportedHeaders {
+                        srcDirs 'src/main/native/include', "$buildDir/generated/source/proto/main/cpp"
+                        if (project.hasProperty('generatedHeaders')) {
+                            srcDir generatedHeaders
+                        }
+                        include "**/*.h"
+                    }
+                }
+            }
+
+            binaries.all {
+                it.tasks.withType(CppCompile) {
+                    it.dependsOn generateProto
+                }
+                if(project.hasProperty('includePhotonTargeting')) {
+                    lib project: ':photon-targeting', library: 'photontargeting', linkage: 'shared'
+                }
+            }
+
+            nativeUtils.useRequiredLibrary(it, "wpilib_shared")
+            nativeUtils.useRequiredLibrary(it, "apriltag_shared")
+            nativeUtils.useRequiredLibrary(it, "opencv_shared")
+        }
+    }
+    testSuites {
+        "${nativeName}Test"(GoogleTestTestSuiteSpec) {
+            for(NativeComponentSpec c : $.components) {
+                if (c.name == nativeName) {
+                    testing c
+                    break
+                }
+            }
+            sources {
+                cpp {
+                    source {
+                        srcDirs 'src/test/native/cpp'
+                        include '**/*.cpp'
+                    }
+                    exportedHeaders {
+                        srcDirs 'src/test/native/include', "$buildDir/generated/source/proto/main/cpp"
+                    }
+                }
+            }
+
+            binaries.all {
+                it.tasks.withType(CppCompile) {
+                    it.dependsOn generateProto
+                }
+                if(project.hasProperty('includePhotonTargeting')) {
+                    lib project: ':photon-targeting', library: 'photontargeting', linkage: 'shared'
+                }
+            }
+
+            nativeUtils.useRequiredLibrary(it, "cscore_shared")
+            nativeUtils.useRequiredLibrary(it, "cameraserver_shared")
+            nativeUtils.useRequiredLibrary(it, "wpilib_executable_shared")
+            nativeUtils.useRequiredLibrary(it, "googletest_static")
+            nativeUtils.useRequiredLibrary(it, "apriltag_shared")
+            nativeUtils.useRequiredLibrary(it, "opencv_shared")
+        }
+    }
+
+    tasks {
+        def c = $.testSuites
+        project.tasks.create('runCpp', Exec) {
+            description = "Run the photon-lib executable"
+            def found = false
+            def systemArch = getCurrentArch()
+            c.each {
+                if (it in GoogleTestTestSuiteSpec && it.name == "${nativeName}Test") {
+                    it.binaries.each {
+                        if (!found) {
+                            def arch = it.targetPlatform.name
+                            if (arch == systemArch) {
+                                dependsOn it.tasks.install
+                                commandLine it.tasks.install.runScriptFile.get().asFile.toString()
+                                def filePath = it.tasks.install.installDirectory.get().toString() + File.separatorChar + 'lib'
+                                test.dependsOn it.tasks.install
+                                test.systemProperty 'java.library.path', filePath
+                                test.environment 'LD_LIBRARY_PATH', filePath
+                                test.environment 'DYLD_LIBRARY_PATH', filePath
+                                test.workingDir filePath
+
+                                found = true
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+apply from: "${rootDir}/shared/javacpp/publish.gradle"

photon-targeting/src/main/java/org/photonvision/targeting/MultiTargetPNPResult.java

@@ -17,13 +17,14 @@
 
 package org.photonvision.targeting;
 
+import edu.wpi.first.util.protobuf.ProtobufSerializable;
 import java.util.ArrayList;
 import java.util.List;
 import org.photonvision.common.dataflow.structures.Packet;
 import org.photonvision.common.dataflow.structures.PacketSerde;
 import org.photonvision.targeting.proto.MultiTargetPNPResultProto;
 
-public class MultiTargetPNPResult {
+public class MultiTargetPNPResult implements ProtobufSerializable {
     // Seeing 32 apriltags at once seems like a sane limit
     private static final int MAX_IDS = 32;
 

photon-targeting/src/main/java/org/photonvision/targeting/PNPResult.java

@@ -18,6 +18,7 @@
 package org.photonvision.targeting;
 
 import edu.wpi.first.math.geometry.Transform3d;
+import edu.wpi.first.util.protobuf.ProtobufSerializable;
 import org.photonvision.common.dataflow.structures.Packet;
 import org.photonvision.common.dataflow.structures.PacketSerde;
 import org.photonvision.targeting.proto.PNPResultProto;
@@ -32,7 +33,7 @@ import org.photonvision.utils.PacketUtils;
  * <p>Note that the coordinate frame of these transforms depends on the implementing solvePnP
  * method.
  */
-public class PNPResult {
+public class PNPResult implements ProtobufSerializable {
     /**
      * If this result is valid. A false value indicates there was an error in estimation, and this
      * result should not be used.

photon-targeting/src/main/java/org/photonvision/targeting/PhotonPipelineResult.java

@@ -17,6 +17,7 @@
 
 package org.photonvision.targeting;
 
+import edu.wpi.first.util.protobuf.ProtobufSerializable;
 import java.util.ArrayList;
 import java.util.List;
 import org.photonvision.common.dataflow.structures.Packet;
@@ -24,7 +25,7 @@ import org.photonvision.common.dataflow.structures.PacketSerde;
 import org.photonvision.targeting.proto.PhotonPipelineResultProto;
 
 /** Represents a pipeline result from a PhotonCamera. */
-public class PhotonPipelineResult {
+public class PhotonPipelineResult implements ProtobufSerializable {
     private static boolean HAS_WARNED = false;
 
     // Targets to store.

photon-targeting/src/main/java/org/photonvision/targeting/PhotonTrackedTarget.java

@@ -18,6 +18,7 @@
 package org.photonvision.targeting;
 
 import edu.wpi.first.math.geometry.Transform3d;
+import edu.wpi.first.util.protobuf.ProtobufSerializable;
 import java.util.ArrayList;
 import java.util.List;
 import org.photonvision.common.dataflow.structures.Packet;
@@ -25,7 +26,7 @@ import org.photonvision.common.dataflow.structures.PacketSerde;
 import org.photonvision.targeting.proto.PhotonTrackedTargetProto;
 import org.photonvision.utils.PacketUtils;
 
-public class PhotonTrackedTarget {
+public class PhotonTrackedTarget implements ProtobufSerializable {
     private static final int MAX_CORNERS = 8;
 
     private final double yaw;

photon-targeting/src/main/java/org/photonvision/targeting/TargetCorner.java

@@ -17,6 +17,7 @@
 
 package org.photonvision.targeting;
 
+import edu.wpi.first.util.protobuf.ProtobufSerializable;
 import java.util.Objects;
 import org.photonvision.common.dataflow.structures.Packet;
 import org.photonvision.common.dataflow.structures.PacketSerde;
@@ -26,7 +27,7 @@ import org.photonvision.targeting.proto.TargetCornerProto;
  * Represents a point in an image at the corner of the minimum-area bounding rectangle, in pixels.
  * Origin at the top left, plus-x to the right, plus-y down.
  */
-public class TargetCorner {
+public class TargetCorner implements ProtobufSerializable {
     public final double x;
     public final double y;
 

photonlib-cpp-examples/build.gradle

@@ -7,7 +7,7 @@ allprojects {
     repositories {
         mavenCentral()
         mavenLocal()
-        maven { url = "https://maven.photonvision.org/repository/internal/" }
+        maven { url = "https://maven.photonvision.org/releases" }
     }
 }
 

photonlib-java-examples/build.gradle

@@ -9,7 +9,7 @@ allprojects {
     repositories {
         mavenCentral()
         mavenLocal()
-        maven { url = "https://maven.photonvision.org/repository/internal/" }
+        maven { url = "https://maven.photonvision.org/releases" }
     }
 }
 

scripts/armrunner.sh

@@ -0,0 +1,12 @@
+###
+# Alternative ARM Runner installer to setup PhotonVision JAR
+# for ARM based builds such as Raspberry Pi, Orange Pi, etc.
+# This assumes that the image provided to arm-runner-action contains
+# the servicefile needed to auto-launch PhotonVision.
+###
+NEW_JAR=$(realpath $(find . -name photonvision\*-linuxarm64.jar))
+echo "Using jar: " $(basename $NEW_JAR)
+
+DEST_PV_LOCATION=/opt/photonvision
+sudo mkdir -p $DEST_PV_LOCATION
+sudo cp $NEW_JAR ${DEST_PV_LOCATION}/photonvision.jar

shared/javacommon.gradle

@@ -79,7 +79,7 @@ publishing {
 
     repositories {
         maven {
-            url ('https://maven.photonvision.org/repository/' + (isDev ? 'snapshots' : 'internal'))
+            url(photonMavenURL)
             credentials {
                 username 'ghactions'
                 password System.getenv("ARTIFACTORY_API_KEY")

shared/javacpp/publish.gradle

@@ -70,7 +70,7 @@ model {
 
         repositories {
             maven {
-                url ('https://maven.photonvision.org/repository/' + (isDev ? 'snapshots' : 'internal'))
+                url(photonMavenURL)
                 credentials {
                     username 'ghactions'
                     password System.getenv("ARTIFACTORY_API_KEY")

test-resources/testimages/2024/README.txt

@@ -0,0 +1,13 @@
+This folder contains images of the AprilTags on the 2024 playing field. All measurements included
+are approximate and are intended to give a rough idea of views from various distances from field
+elements. All images were taken with a Microsoft Lifecam at 1280x720 resolution.
+Camera height: ~29.75 in
+Camera ~20 degrees pitch above horizontal
+The folder includes 2 types of images:
+1. "General Field" images which provide a
+general idea of the view from various spots on the field with a camera at this
+height and angle.
+2. Specific field element images which include a measurement away from a specific field element.
+The Amp, Loading, and Stage photos are measured to the face of the respective elements,
+approximately perpendicular from the face. The remaining photos are all measured to the center of
+the subwoofer face, perpendicular or diagonal as necessary.