Backport maven changes

Still in testing, might break our docs for now

.github/workflows/build.yml

@@ -2,123 +2,14 @@ name: Build
 
 on:
   push:
-    branches: [ master ]
+    branches: 
+      - master
     tags:
       - 'v*'
   pull_request:
     branches: [ master ]
 
 jobs:
-  build-client:
-    name: "PhotonClient Build"
-    defaults:
-      run:
-        working-directory: photon-client
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: 18
-      - name: Install Dependencies
-        run: npm ci
-      - name: Build Production Client
-        run: npm run build
-      - uses: actions/upload-artifact@v4
-        with:
-          name: built-client
-          path: photon-client/dist/
-  build-examples:
-    name: "Build Examples"
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Fetch tags
-        run: git fetch --tags --force
-      - name: Install Java 17
-        uses: actions/setup-java@v4
-        with:
-          java-version: 17
-          distribution: temurin
-      # Need to publish to maven local first, so that C++ sim can pick it up
-      # Still haven't figured out how to make the vendordep file be copied before trying to build examples
-      - name: Publish photonlib to maven local
-        run: |
-          chmod +x gradlew
-          ./gradlew publishtomavenlocal -x check
-      - name: Build Java examples
-        working-directory: photonlib-java-examples
-        run: |
-          chmod +x gradlew
-          ./gradlew copyPhotonlib -x check
-          ./gradlew build -x check --max-workers 2
-      - name: Build C++ examples
-        working-directory: photonlib-cpp-examples
-        run: |
-          chmod +x gradlew
-          ./gradlew copyPhotonlib -x check
-          ./gradlew build -x check --max-workers 2
-  build-gradle:
-    name: "Gradle Build"
-    runs-on: ubuntu-22.04
-    steps:
-      # Checkout code.
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Fetch tags
-        run: git fetch --tags --force
-      - name: Install Java 17
-        uses: actions/setup-java@v3
-        with:
-          java-version: 17
-          distribution: temurin
-      - name: Install mrcal deps
-        run: sudo apt-get update && sudo apt-get install -y libcholmod3 liblapack3 libsuitesparseconfig5
-      - name: Gradle Build
-        run: |
-          chmod +x gradlew
-          ./gradlew build -x check --max-workers 2
-      - name: Gradle Tests
-        run: ./gradlew testHeadless -i --max-workers 1 --stacktrace
-      - name: Gradle Coverage
-        run: ./gradlew jacocoTestReport --max-workers 1
-      - name: Publish Coverage Report
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./photon-server/build/reports/jacoco/test/jacocoTestReport.xml
-      - name: Publish Core Coverage Report
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./photon-core/build/reports/jacoco/test/jacocoTestReport.xml
-  build-offline-docs:
-    name: "Build Offline Docs"
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          repository: 'PhotonVision/photonvision-docs.git'
-          ref: master
-      - uses: actions/setup-python@v5
-        with:
-          python-version: '3.9'
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install sphinx sphinx_rtd_theme sphinx-tabs sphinxext-opengraph doc8
-          pip install -r requirements.txt
-      - name: Build the docs
-        run: |
-          make html
-      - uses: actions/upload-artifact@master
-        with:
-          name: built-docs
-          path: build/html
   build-photonlib-host:
     env:
       MACOSX_DEPLOYMENT_TARGET: 12
@@ -188,179 +79,3 @@ jobs:
         env:
           ARTIFACTORY_API_KEY: ${{ secrets.ARTIFACTORY_API_KEY }}
         if: github.event_name == 'push'
-  build-package:
-    needs: [build-client, build-gradle, build-offline-docs]
-
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - os: windows-latest
-            artifact-name: Win64
-            architecture: x64
-            arch-override: none
-          - os: macos-latest
-            artifact-name: macOS
-            architecture: x64
-            arch-override: none
-          - os: ubuntu-latest
-            artifact-name: Linux
-            architecture: x64
-            arch-override: none
-          - os: macos-latest
-            artifact-name: macOSArm
-            architecture: x64
-            arch-override: macarm64
-          - os: ubuntu-latest
-            artifact-name: LinuxArm32
-            architecture: x64
-            arch-override: linuxarm32
-          - os: ubuntu-latest
-            artifact-name: LinuxArm64
-            architecture: x64
-            arch-override: linuxarm64
-
-    runs-on: ${{ matrix.os }}
-    name: "Build fat JAR - ${{ matrix.artifact-name }}"
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Install Java 17
-        uses: actions/setup-java@v4
-        with:
-          java-version: 17
-          distribution: temurin
-      - run: |
-          rm -rf photon-server/src/main/resources/web/*
-          mkdir -p photon-server/src/main/resources/web/docs
-        if: ${{ (matrix.os) != 'windows-latest' }}
-      - run: |
-          del photon-server\src\main\resources\web\*.*
-          mkdir photon-server\src\main\resources\web\docs
-        if: ${{ (matrix.os) == 'windows-latest' }}
-      - uses: actions/download-artifact@v4
-        with:
-          name: built-client
-          path: photon-server/src/main/resources/web/
-      - uses: actions/download-artifact@v4
-        with:
-          name: built-docs
-          path: photon-server/src/main/resources/web/docs
-      - run: |
-          chmod +x gradlew
-          ./gradlew photon-server:shadowJar --max-workers 2 -PArchOverride=${{ matrix.arch-override }}
-        if: ${{ (matrix.arch-override != 'none') }}
-      - run: |
-          chmod +x gradlew
-          ./gradlew photon-server:shadowJar --max-workers 2
-        if: ${{ (matrix.arch-override == 'none') }}
-      - uses: actions/upload-artifact@v4
-        with:
-          name: jar-${{ matrix.artifact-name }}
-          path: photon-server/build/libs
-  build-image:
-    needs: [build-package]
-
-    if: ${{ github.event_name != 'pull_request' }}
-
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - os: ubuntu-latest
-            artifact-name: LinuxArm64
-            image_suffix: RaspberryPi
-            image_url: https://github.com/PhotonVision/photon-image-modifier/releases/download/v2024.0.4/photonvision_raspi.img.xz
-            cpu: cortex-a7
-            image_additional_mb: 0
-          - os: ubuntu-latest
-            artifact-name: LinuxArm64
-            image_suffix: limelight2
-            image_url: https://github.com/PhotonVision/photon-image-modifier/releases/download/v2024.0.4/photonvision_limelight.img.xz
-            cpu: cortex-a7
-            image_additional_mb: 0
-          - os: ubuntu-latest
-            artifact-name: LinuxArm64
-            image_suffix: limelight3
-            image_url: https://github.com/PhotonVision/photon-image-modifier/releases/download/v2024.0.5/photonvision_limelight3.img.xz
-            cpu: cortex-a7
-            image_additional_mb: 0
-          - os: ubuntu-latest
-            artifact-name: LinuxArm64
-            image_suffix: orangepi5
-            image_url: https://github.com/PhotonVision/photon-image-modifier/releases/download/v2024.0.6/photonvision_opi5.img.xz
-            cpu: cortex-a8
-            image_additional_mb: 4096
-          - os: ubuntu-latest
-            artifact-name: LinuxArm64
-            image_suffix: orangepi5plus
-            image_url: https://github.com/PhotonVision/photon-image-modifier/releases/download/v2024.0.6/photonvision_opi5plus.img.xz
-            cpu: cortex-a8
-            image_additional_mb: 4096
-
-    runs-on: ${{ matrix.os }}
-    name: "Build image - ${{ matrix.image_url }}"
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - uses: actions/download-artifact@v4
-        with:
-          name: jar-${{ matrix.artifact-name }}
-      - uses: pguyot/arm-runner-action@v2
-        name: Generate image
-        id: generate_image
-        with:
-          base_image: ${{ matrix.image_url }}
-          image_additional_mb: ${{ matrix.image_additional_mb }}
-          optimize_image: yes
-          cpu: ${{ matrix.cpu }}
-          # We do _not_ wanna copy photon into the image. Bind mount instead
-          bind_mount_repository: true
-          commands: |
-            chmod +x scripts/armrunner.sh
-            ./scripts/armrunner.sh
-      - name: Compress image
-        run: |
-          new_jar=$(realpath $(find . -name photonvision\*-linuxarm64.jar))
-          new_image_name=$(basename "${new_jar/.jar/_${{ matrix.image_suffix }}.img}")
-          mv ${{ steps.generate_image.outputs.image }} $new_image_name
-          sudo xz -T 0 -v $new_image_name
-      - uses: actions/upload-artifact@v4
-        name: Upload image
-        with:
-          name: image-${{ matrix.image_suffix }}
-          path: photonvision*.xz
-  release:
-    needs: [build-package, build-image]
-    runs-on: ubuntu-22.04
-    steps:
-      # Download literally every single artifact. This also downloads client and docs,
-      # but the filtering below won't pick these up (I hope)
-      - uses: actions/download-artifact@v4
-      - run: find
-      # Push to dev release
-      - uses: pyTooling/Actions/releaser@r0
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          tag: 'Dev'
-          rm: true
-          files: |
-            **/*.xz
-            **/*.jar
-            **/photonlib*.json
-        if: github.event_name == 'push'
-      # Upload all jars and xz archives
-      - uses: softprops/action-gh-release@v1
-        with:
-          files: |
-            **/*.xz
-            **/*.jar
-            **/photonlib*.json
-        if: startsWith(github.ref, 'refs/tags/v')
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/documentation.yml

@@ -1,92 +0,0 @@
-name: Documentation
-
-on:
-  push:
-    # For now, run on all commits to master
-    branches: [ master ]
-    # and also all tags starting with v
-    tags:
-      - 'v*'
-
-# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
-permissions:
-  contents: read
-  pages: write
-  id-token: write
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build-client:
-    name: "PhotonClient Build"
-    defaults:
-      run:
-        working-directory: photon-client
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: 18
-      - name: Install Dependencies
-        run: npm ci
-      - name: Build Production Client
-        run: npm run build-demo
-      - uses: actions/upload-artifact@v4
-        with:
-          name: built-client
-          path: photon-client/dist/
-
-  run_docs:
-    runs-on: "ubuntu-22.04"
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-    - name: Fetch tags
-      run: git fetch --tags --force
-    - name: Install Java 17
-      uses: actions/setup-java@v3
-      with:
-        java-version: 17
-        distribution: temurin
-
-    - name: Build javadocs/doxygen
-      run: |
-        chmod +x gradlew
-        ./gradlew docs:generateJavaDocs docs:doxygen
-
-    - uses: actions/upload-artifact@v4
-      with:
-        name: built-docs
-        path: docs/build/docs
-
-  release:
-    needs: [build-client, run_docs]
-
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-
-    runs-on: ubuntu-22.04
-    steps:
-
-      # Download literally every single artifact.
-      - uses: actions/download-artifact@v4
-
-      - run: find .
-
-      - name: Setup Pages
-        uses: actions/configure-pages@v4
-      - name: Upload artifact
-        uses: actions/upload-pages-artifact@v3
-        with:
-          # Upload entire repository
-          path: '.'
-      - name: Deploy to GitHub Pages
-        id: deployment
-        uses: actions/deploy-pages@v4

.github/workflows/lint-format.yml

@@ -1,88 +0,0 @@
-name: Lint and Format
-
-on:
-  push:
-    branches: [ master ]
-    tags:
-      - 'v*'
-  pull_request:
-    branches: [ master ]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  wpiformat:
-    name: "wpiformat"
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v3
-      - name: Fetch all history and metadata
-        run: |
-          git fetch --prune --unshallow
-          git checkout -b pr
-          git branch -f master origin/master
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.8
-      - name: Install wpiformat
-        run: pip3 install wpiformat
-      - name: Run
-        run: wpiformat
-      - name: Check output
-        run: git --no-pager diff --exit-code HEAD
-      - name: Generate diff
-        run: git diff HEAD > wpiformat-fixes.patch
-        if: ${{ failure() }}
-      - uses: actions/upload-artifact@v3
-        with:
-          name: wpiformat fixes
-          path: wpiformat-fixes.patch
-        if: ${{ failure() }}
-  javaformat:
-    name: "Java Formatting"
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - uses: actions/setup-java@v3
-        with:
-          java-version: 17
-          distribution: temurin
-      - run: |
-          chmod +x gradlew
-          ./gradlew spotlessCheck
-
-  client-lint-format:
-    name: "PhotonClient Lint and Formatting"
-    defaults:
-      run:
-        working-directory: photon-client
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v3
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: 18
-      - name: Install Dependencies
-        run: npm ci
-      - name: Check Linting
-        run: npm run lint-ci
-      - name: Check Formatting
-        run: npm run format-ci
-  server-index:
-    name: "Check server index.html not changed"
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v3
-      - name: Fetch all history and metadata
-        run: |
-          git fetch --prune --unshallow
-          git checkout -b pr
-          git branch -f master origin/master
-      - name: Check index.html not changed
-        run: git --no-pager diff --exit-code origin/master photon-server/src/main/resources/web/index.html

.github/workflows/python.yml

@@ -1,60 +0,0 @@
-name: Build and Distribute PhotonLibPy
-
-permissions:
-  id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
-
-on:
-  push:
-    branches: [ master ]
-    tags:
-      - 'v*'
-  pull_request:
-    branches: [ master ]
-
-jobs:
-  buildAndDeploy:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: 3.11
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install setuptools wheel pytest
-
-      - name: Build wheel
-        working-directory: ./photon-lib/py
-        run: |
-          python setup.py sdist bdist_wheel
-
-      - name: Run Unit Tests
-        working-directory: ./photon-lib/py
-        run: |
-            pip install --no-cache-dir dist/*.whl
-            pytest
-
-
-      - name: Upload artifacts
-        uses: actions/upload-artifact@master
-        with:
-          name: dist
-          path: ./photon-lib/py/dist/
-
-      - name: Publish package distributions to TestPyPI
-        # Only upload on tags
-        if: startsWith(github.ref, 'refs/tags/v')
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          packages_dir: ./photon-lib/py/dist/
-
-    permissions:
-      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing

build.gradle

@@ -13,8 +13,9 @@ allprojects {
     repositories {
         mavenCentral()
         mavenLocal()
-        maven { url = "https://maven.photonvision.org/repository/internal/" }
-        maven { url = "https://maven.photonvision.org/repository/snapshots/" }
+        maven { url = "https://maven.photonvision.org/releases" }
+        maven { url = "https://maven.photonvision.org/snapshots" }
+        maven { url = "https://jogamp.org/deployment/maven/" }
     }
     wpilibRepositories.addAllReleaseRepositories(it)
     wpilibRepositories.addAllDevelopmentRepositories(it)
@@ -30,7 +31,7 @@ ext {
     joglVersion = "2.4.0-rc-20200307"
     javalinVersion = "5.6.2"
     photonGlDriverLibVersion = "dev-v2023.1.0-9-g75fc678"
-    rknnVersion = "dev-v2024.0.0-30-g001b5ec"
+    rknnVersion = "dev-v2024.0.0-64-gc0836a6"
     frcYear = "2024"
     mrcalVersion = "dev-v2024.0.0-7-gc976aaa";
 
@@ -50,6 +51,10 @@ ext {
     println("Building for platform " + jniPlatform + " wpilib: " + wpilibNativeName)
     println("Using Wpilib: " + wpilibVersion)
     println("Using OpenCV: " + openCVversion)
+
+
+    photonMavenURL = 'https://maven.photonvision.org/' + (isDev ? 'snapshots' : 'releases');
+    println("Publishing Photonlib to " + photonMavenURL)
 }
 
 spotless {

photon-client/src/components/cameras/CameraCalibrationCard.vue

@@ -25,15 +25,10 @@ const getUniqueVideoFormatsByResolution = (): VideoFormat[] => {
 
       const calib = useCameraSettingsStore().getCalibrationCoeffs(format.resolution);
       if (calib !== undefined) {
-        // Is this the right formula for RMS error? who knows! not me!
-        const perViewSumSquareReprojectionError = calib.observations.flatMap((it) =>
-          it.reprojectionErrors.flatMap((it2) => [it2.x, it2.y])
-        );
         // For each error, square it, sum the squares, and divide by total points N
-        format.mean = Math.sqrt(
-          perViewSumSquareReprojectionError.map((it) => Math.pow(it, 2)).reduce((a, b) => a + b, 0) /
-            perViewSumSquareReprojectionError.length
-        );
+        if (calib.meanErrors.length)
+          format.mean = calib.meanErrors.reduce((a, b) => a + b, 0) / calib.meanErrors.length;
+        else format.mean = NaN;
 
         format.horizontalFOV =
           2 * Math.atan2(format.resolution.width / 2, calib.cameraIntrinsics.data[0]) * (180 / Math.PI);
@@ -109,7 +104,7 @@ const downloadCalibBoard = () => {
           const yPos = chessboardStartY + squareY * squareSizeIn.value;
 
           // Only draw the odd squares to create the chessboard pattern
-          if ((xPos + yPos + 0.25) % 2 === 0) {
+          if (squareY % 2 != squareX % 2) {
             doc.rect(xPos, yPos, squareSizeIn.value, squareSizeIn.value, "F");
           }
         }
@@ -263,7 +258,7 @@ const setSelectedVideoFormat = (format: VideoFormat) => {
               >
                 <td>{{ getResolutionString(value.resolution) }}</td>
                 <td>
-                  {{ value.mean !== undefined ? (isNaN(value.mean) ? "NaN" : value.mean.toFixed(2) + "px") : "-" }}
+                  {{ value.mean !== undefined ? (isNaN(value.mean) ? "Unknown" : value.mean.toFixed(2) + "px") : "-" }}
                 </td>
                 <td>{{ value.horizontalFOV !== undefined ? value.horizontalFOV.toFixed(2) + "°" : "-" }}</td>
                 <td>{{ value.verticalFOV !== undefined ? value.verticalFOV.toFixed(2) + "°" : "-" }}</td>

photon-client/src/components/cameras/CameraCalibrationInfoCard.vue

@@ -1,51 +1,19 @@
 <script setup lang="ts">
-import type { BoardObservation, CameraCalibrationResult, VideoFormat } from "@/types/SettingTypes";
+import type { CameraCalibrationResult, VideoFormat } from "@/types/SettingTypes";
 import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
 import { useStateStore } from "@/stores/StateStore";
-import { ref } from "vue";
-import loadingImage from "@/assets/images/loading.svg";
+import { computed, inject, ref } from "vue";
 import { getResolutionString, parseJsonFile } from "@/lib/PhotonUtils";
 
 const props = defineProps<{
   videoFormat: VideoFormat;
 }>();
 
-const getMeanFromView = (o: BoardObservation) => {
-  // Is this the right formula for RMS error? who knows! not me!
-  const perViewSumSquareReprojectionError = o.reprojectionErrors.flatMap((it2) => [it2.x, it2.y]);
-
-  // For each error, square it, sum the squares, and divide by total points N
-  return Math.sqrt(
-    perViewSumSquareReprojectionError.map((it) => Math.pow(it, 2)).reduce((a, b) => a + b, 0) /
-      perViewSumSquareReprojectionError.length
-  );
+const exportCalibration = ref();
+const openExportCalibrationPrompt = () => {
+  exportCalibration.value.click();
 };
 
-// Import and export functions
-const downloadCalibration = () => {
-  const calibData = useCameraSettingsStore().getCalibrationCoeffs(props.videoFormat.resolution);
-  if (calibData === undefined) {
-    useStateStore().showSnackbarMessage({
-      color: "error",
-      message:
-        "Calibration data isn't available for the requested resolution, please calibrate the requested resolution first"
-    });
-    return;
-  }
-
-  const camUniqueName = useCameraSettingsStore().currentCameraSettings.uniqueName;
-  const filename = `photon_calibration_${camUniqueName}_${calibData.resolution.width}x${calibData.resolution.height}.json`;
-  const fileData = JSON.stringify(calibData);
-
-  const element = document.createElement("a");
-  element.style.display = "none";
-  element.setAttribute("href", "data:text/plain;charset=utf-8," + encodeURIComponent(fileData));
-  element.setAttribute("download", filename);
-
-  document.body.appendChild(element);
-  element.click();
-  document.body.removeChild(element);
-};
 const importCalibrationFromPhotonJson = ref();
 const openUploadPhotonCalibJsonPrompt = () => {
   importCalibrationFromPhotonJson.value.click();
@@ -97,19 +65,28 @@ const importCalibration = async () => {
 };
 
 interface ObservationDetails {
-  snapshotSrc: any;
   mean: number;
   index: number;
 }
+
+const currentCalibrationCoeffs = computed<CameraCalibrationResult | undefined>(() =>
+  useCameraSettingsStore().getCalibrationCoeffs(props.videoFormat.resolution)
+);
+
 const getObservationDetails = (): ObservationDetails[] | undefined => {
-  return useCameraSettingsStore()
-    .getCalibrationCoeffs(props.videoFormat.resolution)
-    ?.observations.map((o, i) => ({
-      index: i,
-      mean: parseFloat(getMeanFromView(o).toFixed(2)),
-      snapshotSrc: o.includeObservationInCalibration ? "data:image/png;base64," + o.snapshotData.data : loadingImage
-    }));
+  const coefficients = currentCalibrationCoeffs.value;
+
+  return coefficients?.meanErrors.map((m, i) => ({
+    index: i,
+    mean: parseFloat(m.toFixed(2))
+  }));
 };
+
+const exportCalibrationURL = computed<string>(() =>
+  useCameraSettingsStore().getCalJSONUrl(inject("backendHost") as string, props.videoFormat.resolution)
+);
+const calibrationImageURL = (index: number) =>
+  useCameraSettingsStore().getCalImageUrl(inject<string>("backendHost") as string, props.videoFormat.resolution, index);
 </script>
 
 <template>
@@ -140,19 +117,22 @@ const getObservationDetails = (): ObservationDetails[] | undefined => {
         <v-btn
           color="secondary"
           class="mt-4"
-          :disabled="useCameraSettingsStore().getCalibrationCoeffs(props.videoFormat.resolution) === undefined"
+          :disabled="!currentCalibrationCoeffs"
           style="width: 100%"
-          @click="downloadCalibration"
+          @click="openExportCalibrationPrompt"
         >
           <v-icon left>mdi-export</v-icon>
           <span>Export</span>
         </v-btn>
+        <a
+          ref="exportCalibration"
+          style="color: black; text-decoration: none; display: none"
+          :href="exportCalibrationURL"
+          target="_blank"
+        />
       </v-col>
     </v-row>
-    <v-row
-      v-if="useCameraSettingsStore().getCalibrationCoeffs(props.videoFormat.resolution) !== undefined"
-      class="pt-2"
-    >
+    <v-row v-if="currentCalibrationCoeffs" class="pt-2">
       <v-card-subtitle>Calibration Details</v-card-subtitle>
       <v-simple-table dense style="width: 100%" class="pl-2 pr-2">
         <template #default>
@@ -231,7 +211,9 @@ const getObservationDetails = (): ObservationDetails[] | undefined => {
             </tr>
             <tr>
               <td>Horizontal FOV</td>
-              <td>{{ videoFormat.horizontalFOV !== undefined ? videoFormat.horizontalFOV.toFixed(2) + "°" : "-" }}</td>
+              <td>
+                {{ videoFormat.horizontalFOV !== undefined ? videoFormat.horizontalFOV.toFixed(2) + "°" : "-" }}
+              </td>
             </tr>
             <tr>
               <td>Vertical FOV</td>
@@ -242,11 +224,7 @@ const getObservationDetails = (): ObservationDetails[] | undefined => {
               <td>{{ videoFormat.diagonalFOV !== undefined ? videoFormat.diagonalFOV.toFixed(2) + "°" : "-" }}</td>
             </tr>
             <!-- Board warp, only shown for mrcal-calibrated cameras -->
-            <tr
-              v-if="
-                useCameraSettingsStore().getCalibrationCoeffs(props.videoFormat.resolution)?.calobjectWarp?.length === 2
-              "
-            >
+            <tr v-if="currentCalibrationCoeffs?.calobjectWarp?.length === 2">
               <td>Board warp, X/Y</td>
               <td>
                 {{
@@ -278,7 +256,7 @@ const getObservationDetails = (): ObservationDetails[] | undefined => {
         <template #expanded-item="{ headers, item }">
           <td :colspan="headers.length">
             <div style="display: flex; justify-content: center; width: 100%">
-              <img :src="item.snapshotSrc" alt="observation image" class="snapshot-preview pt-2 pb-2" />
+              <img :src="calibrationImageURL(item.index)" alt="observation image" class="snapshot-preview pt-2 pb-2" />
             </div>
           </td>
         </template>

photon-client/src/components/dashboard/tabs/ObjectDetectionTab.vue

@@ -11,6 +11,16 @@ const currentPipelineSettings = computed<ActivePipelineSettings>(
   () => useCameraSettingsStore().currentPipelineSettings
 );
 
+// TODO fix pv-range-slider so that store access doesn't need to be deferred
+const contourArea = computed<[number, number]>({
+  get: () => Object.values(useCameraSettingsStore().currentPipelineSettings.contourArea) as [number, number],
+  set: (v) => (useCameraSettingsStore().currentPipelineSettings.contourArea = v)
+});
+const contourRatio = computed<[number, number]>({
+  get: () => Object.values(useCameraSettingsStore().currentPipelineSettings.contourRatio) as [number, number],
+  set: (v) => (useCameraSettingsStore().currentPipelineSettings.contourRatio = v)
+});
+
 const interactiveCols = computed(() =>
   (getCurrentInstance()?.proxy.$vuetify.breakpoint.mdAndDown || false) &&
   (!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode)
@@ -32,5 +42,42 @@ const interactiveCols = computed(() =>
       :step="0.01"
       @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ confidence: value }, false)"
     />
+    <pv-range-slider
+      v-model="contourArea"
+      label="Area"
+      :min="0"
+      :max="100"
+      :slider-cols="interactiveCols"
+      :step="0.01"
+      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ contourArea: value }, false)"
+    />
+    <pv-range-slider
+      v-model="contourRatio"
+      label="Ratio (W/H)"
+      tooltip="Min and max ratio between the width and height of a contour's bounding rectangle"
+      :min="0"
+      :max="100"
+      :slider-cols="interactiveCols"
+      :step="0.01"
+      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ contourRatio: value }, false)"
+    />
+    <pv-select
+      v-model="useCameraSettingsStore().currentPipelineSettings.contourTargetOrientation"
+      label="Target Orientation"
+      tooltip="Used to determine how to calculate target landmarks, as well as aspect ratio"
+      :items="['Portrait', 'Landscape']"
+      :select-cols="interactiveCols"
+      @input="
+        (value) => useCameraSettingsStore().changeCurrentPipelineSetting({ contourTargetOrientation: value }, false)
+      "
+    />
+    <pv-select
+      v-model="currentPipelineSettings.contourSortMode"
+      label="Target Sort"
+      tooltip="Chooses the sorting mode used to determine the 'best' targets to provide to user code"
+      :select-cols="interactiveCols"
+      :items="['Largest', 'Smallest', 'Highest', 'Lowest', 'Rightmost', 'Leftmost', 'Centermost']"
+      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ contourSortMode: value }, false)"
+    />
   </div>
 </template>

photon-client/src/components/settings/MetricsCard.vue

@@ -27,42 +27,54 @@ const generalMetrics = computed<MetricItem[]>(() => [
     value: useSettingsStore().general.gpuAcceleration || "Unknown"
   }
 ]);
-const platformMetrics = computed<MetricItem[]>(() => [
-  {
-    header: "CPU Temp",
-    value: useSettingsStore().metrics.cpuTemp === undefined ? "Unknown" : `${useSettingsStore().metrics.cpuTemp}°C`
-  },
-  {
-    header: "CPU Usage",
-    value: useSettingsStore().metrics.cpuUtil === undefined ? "Unknown" : `${useSettingsStore().metrics.cpuUtil}%`
-  },
-  {
-    header: "CPU Memory Usage",
-    value:
-      useSettingsStore().metrics.ramUtil === undefined || useSettingsStore().metrics.cpuMem === undefined
-        ? "Unknown"
-        : `${useSettingsStore().metrics.ramUtil || "Unknown"}MB of ${useSettingsStore().metrics.cpuMem}MB`
-  },
-  {
-    header: "GPU Memory Usage",
-    value:
-      useSettingsStore().metrics.gpuMemUtil === undefined || useSettingsStore().metrics.gpuMem === undefined
-        ? "Unknown"
-        : `${useSettingsStore().metrics.gpuMemUtil}MB of ${useSettingsStore().metrics.gpuMem}MB`
-  },
-  {
-    header: "CPU Throttling",
-    value: useSettingsStore().metrics.cpuThr || "Unknown"
-  },
-  {
-    header: "CPU Uptime",
-    value: useSettingsStore().metrics.cpuUptime || "Unknown"
-  },
-  {
-    header: "Disk Usage",
-    value: useSettingsStore().metrics.diskUtilPct || "Unknown"
+
+const platformMetrics = computed<MetricItem[]>(() => {
+  const stats = [
+    {
+      header: "CPU Temp",
+      value: useSettingsStore().metrics.cpuTemp === undefined ? "Unknown" : `${useSettingsStore().metrics.cpuTemp}°C`
+    },
+    {
+      header: "CPU Usage",
+      value: useSettingsStore().metrics.cpuUtil === undefined ? "Unknown" : `${useSettingsStore().metrics.cpuUtil}%`
+    },
+    {
+      header: "CPU Memory Usage",
+      value:
+        useSettingsStore().metrics.ramUtil === undefined || useSettingsStore().metrics.cpuMem === undefined
+          ? "Unknown"
+          : `${useSettingsStore().metrics.ramUtil || "Unknown"}MB of ${useSettingsStore().metrics.cpuMem}MB`
+    },
+    {
+      header: "GPU Memory Usage",
+      value:
+        useSettingsStore().metrics.gpuMemUtil === undefined || useSettingsStore().metrics.gpuMem === undefined
+          ? "Unknown"
+          : `${useSettingsStore().metrics.gpuMemUtil}MB of ${useSettingsStore().metrics.gpuMem}MB`
+    },
+    {
+      header: "CPU Throttling",
+      value: useSettingsStore().metrics.cpuThr || "Unknown"
+    },
+    {
+      header: "CPU Uptime",
+      value: useSettingsStore().metrics.cpuUptime || "Unknown"
+    },
+    {
+      header: "Disk Usage",
+      value: useSettingsStore().metrics.diskUtilPct || "Unknown"
+    }
+  ];
+
+  if (useSettingsStore().metrics.npuUsage) {
+    stats.push({
+      header: "NPU Usage",
+      value: useSettingsStore().metrics.npuUsage || "Unknown"
+    });
   }
-]);
+
+  return stats;
+});
 
 const metricsLastFetched = ref("Never");
 const fetchMetrics = () => {

photon-client/src/stores/settings/CameraSettingsStore.ts

@@ -416,6 +416,23 @@ export const useCameraSettingsStore = defineStore("cameraSettings", {
       cameraIndex: number = useStateStore().currentCameraIndex
     ): CameraCalibrationResult | undefined {
       return this.cameras[cameraIndex].completeCalibrations.find((v) => resolutionsAreEqual(v.resolution, resolution));
+    },
+    getCalImageUrl(host: string, resolution: Resolution, idx: number, cameraIdx = useStateStore().currentCameraIndex) {
+      const url = new URL(`http://${host}/api/utils/getCalSnapshot`);
+      url.searchParams.set("width", Math.round(resolution.width).toFixed(0));
+      url.searchParams.set("height", Math.round(resolution.height).toFixed(0));
+      url.searchParams.set("snapshotIdx", Math.round(idx).toFixed(0));
+      url.searchParams.set("cameraIdx", Math.round(cameraIdx).toFixed(0));
+
+      return url.href;
+    },
+    getCalJSONUrl(host: string, resolution: Resolution, cameraIdx = useStateStore().currentCameraIndex) {
+      const url = new URL(`http://${host}/api/utils/getCalibrationJSON`);
+      url.searchParams.set("width", Math.round(resolution.width).toFixed(0));
+      url.searchParams.set("height", Math.round(resolution.height).toFixed(0));
+      url.searchParams.set("cameraIdx", Math.round(cameraIdx).toFixed(0));
+
+      return url.href;
     }
   }
 });

photon-client/src/stores/settings/GeneralSettingsStore.ts

@@ -59,7 +59,8 @@ export const useSettingsStore = defineStore("settings", {
       gpuMemUtil: undefined,
       cpuThr: undefined,
       cpuUptime: undefined,
-      diskUtilPct: undefined
+      diskUtilPct: undefined,
+      npuUsage: undefined
     },
     currentFieldLayout: {
       field: {
@@ -91,7 +92,8 @@ export const useSettingsStore = defineStore("settings", {
         gpuMemUtil: data.gpuMemUtil || undefined,
         cpuThr: data.cpuThr || undefined,
         cpuUptime: data.cpuUptime || undefined,
-        diskUtilPct: data.diskUtilPct || undefined
+        diskUtilPct: data.diskUtilPct || undefined,
+        npuUsage: data.npuUsage || undefined
       };
     },
     updateGeneralSettingsFromWebsocket(data: WebsocketSettingsUpdate) {

photon-client/src/types/SettingTypes.ts

@@ -20,6 +20,7 @@ export interface MetricData {
   cpuThr?: string;
   cpuUptime?: string;
   diskUtilPct?: string;
+  npuUsage?: string;
 }
 
 export enum NetworkConnectionType {
@@ -138,6 +139,9 @@ export interface CameraCalibrationResult {
   distCoeffs: JsonMatOfDouble;
   observations: BoardObservation[];
   calobjectWarp?: number[];
+  // We might have to omit observations for bandwith, so backend will send us this
+  numSnapshots: number;
+  meanErrors: number[];
 }
 
 export enum ValidQuirks {
@@ -255,7 +259,9 @@ export const PlaceholderCameraSettings: CameraSettings = {
           snapshotName: "img0.png",
           snapshotData: { rows: 480, cols: 640, type: CvType.CV_8U, data: "" }
         }
-      ]
+      ],
+      numSnapshots: 1,
+      meanErrors: [123.45]
     }
   ],
   pipelineNicknames: ["Placeholder Pipeline"],

photon-core/build.gradle

@@ -37,9 +37,8 @@ dependencies {
     implementation 'org.zeroturnaround:zt-zip:1.14'
 
     implementation "org.xerial:sqlite-jdbc:3.41.0.0"
-    def rknnjniversion = "dev-v2024.0.0-44-g8022c40"
-    implementation "org.photonvision:rknn_jni-jni:$rknnjniversion:linuxarm64"
-    implementation "org.photonvision:rknn_jni-java:$rknnjniversion"
+    implementation "org.photonvision:rknn_jni-jni:$rknnVersion:linuxarm64"
+    implementation "org.photonvision:rknn_jni-java:$rknnVersion"
     implementation "org.photonvision:photon-libcamera-gl-driver-jni:$photonGlDriverLibVersion:linuxarm64"
     implementation "org.photonvision:photon-libcamera-gl-driver-java:$photonGlDriverLibVersion"
 

photon-core/src/main/java/org/photonvision/common/configuration/NeuralNetworkModelManager.java

@@ -25,12 +25,14 @@ import java.nio.file.Paths;
 import java.util.List;
 import org.photonvision.common.logging.LogGroup;
 import org.photonvision.common.logging.Logger;
+import org.photonvision.rknn.RknnJNI;
 
 public class NeuralNetworkModelManager {
     private static NeuralNetworkModelManager INSTANCE;
     private static final Logger logger = new Logger(NeuralNetworkModelManager.class, LogGroup.Config);
 
     private final String MODEL_NAME = "note-640-640-yolov5s.rknn";
+    private final RknnJNI.ModelVersion modelVersion = RknnJNI.ModelVersion.YOLO_V5;
     private File defaultModelFile;
     private List<String> labels;
 
@@ -51,7 +53,7 @@ public class NeuralNetworkModelManager {
         this.defaultModelFile = new File(modelsFolder, MODEL_NAME);
         extractResource(modelResourcePath, defaultModelFile);
 
-        File labelsFile = new File(modelsFolder, "labels.txt");
+        File labelsFile = new File(modelsFolder, "labels_v5.txt");
         var labelResourcePath = "/models/" + labelsFile.getName();
         extractResource(labelResourcePath, labelsFile);
 
@@ -95,4 +97,8 @@ public class NeuralNetworkModelManager {
     public List<String> getLabels() {
         return labels;
     }
+
+    public RknnJNI.ModelVersion getModelVersion() {
+        return modelVersion;
+    }
 }

photon-core/src/main/java/org/photonvision/common/configuration/PhotonConfiguration.java

@@ -31,7 +31,7 @@ import org.photonvision.common.util.SerializationUtils;
 import org.photonvision.jni.RknnDetectorJNI;
 import org.photonvision.mrcal.MrCalJNILoader;
 import org.photonvision.raspi.LibCameraJNILoader;
-import org.photonvision.vision.calibration.CameraCalibrationCoefficients;
+import org.photonvision.vision.calibration.UICameraCalibrationCoefficients;
 import org.photonvision.vision.camera.QuirkyCamera;
 import org.photonvision.vision.processes.VisionModule;
 import org.photonvision.vision.processes.VisionModuleManager;
@@ -126,13 +126,6 @@ public class PhotonConfiguration {
 
         settingsSubmap.put("networkSettings", netConfigMap);
 
-        map.put(
-                "cameraSettings",
-                VisionModuleManager.getInstance().getModules().stream()
-                        .map(VisionModule::toUICameraConfig)
-                        .map(SerializationUtils::objectToHashMap)
-                        .collect(Collectors.toList()));
-
         var lightingConfig = new UILightingConfig();
         lightingConfig.brightness = hardwareSettings.ledBrightnessPercentage;
         lightingConfig.supported = !hardwareConfig.ledPins.isEmpty();
@@ -181,7 +174,7 @@ public class PhotonConfiguration {
         public HashMap<Integer, HashMap<String, Object>> videoFormatList;
         public int outputStreamPort;
         public int inputStreamPort;
-        public List<CameraCalibrationCoefficients> calibrations;
+        public List<UICameraCalibrationCoefficients> calibrations;
         public boolean isFovConfigurable = true;
         public QuirkyCamera cameraQuirks;
         public boolean isCSICamera;

photon-core/src/main/java/org/photonvision/common/dataflow/websocket/UIDataPublisher.java

@@ -26,6 +26,7 @@ import org.photonvision.common.logging.LogGroup;
 import org.photonvision.common.logging.Logger;
 import org.photonvision.common.util.SerializationUtils;
 import org.photonvision.vision.pipeline.result.CVPipelineResult;
+import org.photonvision.vision.pipeline.result.CalibrationPipelineResult;
 
 public class UIDataPublisher implements CVPipelineResultConsumer {
     private static final Logger logger = new Logger(UIDataPublisher.class, LogGroup.VisionModule);
@@ -41,16 +42,22 @@ public class UIDataPublisher implements CVPipelineResultConsumer {
     public void accept(CVPipelineResult result) {
         long now = System.currentTimeMillis();
 
-        // only update the UI at 15hz
+        // only update the UI at 10hz
         if (lastUIResultUpdateTime + 1000.0 / 10.0 > now) return;
 
         var dataMap = new HashMap<String, Object>();
         dataMap.put("fps", result.fps);
         dataMap.put("latency", result.getLatencyMillis());
         var uiTargets = new ArrayList<HashMap<String, Object>>(result.targets.size());
-        for (var t : result.targets) {
-            uiTargets.add(t.toHashMap());
+
+        // We don't actually need to send targets during calibration and it can take up a lot (up to
+        // 1.2Mbps for 60 snapshots) of target results with no pitch/yaw/etc set
+        if (!(result instanceof CalibrationPipelineResult)) {
+            for (var t : result.targets) {
+                uiTargets.add(t.toHashMap());
+            }
         }
+
         dataMap.put("targets", uiTargets);
         dataMap.put("classNames", result.objectDetectionClassNames);
 

photon-core/src/main/java/org/photonvision/common/hardware/metrics/MetricsManager.java

@@ -92,6 +92,10 @@ public class MetricsManager {
         return safeExecute(cmds.cpuThrottleReasonCmd);
     }
 
+    public String getNpuUsage() {
+        return safeExecute(cmds.npuUsageCommand);
+    }
+
     private String gpuMemSave = null;
 
     public String getGPUMemorySplit() {
@@ -128,6 +132,7 @@ public class MetricsManager {
         metrics.put("ramUtil", this.getUsedRam());
         metrics.put("gpuMemUtil", this.getMallocedMemory());
         metrics.put("diskUtilPct", this.getUsedDiskPct());
+        metrics.put("npuUsage", this.getNpuUsage());
 
         DataChangeService.getInstance().publishEvent(OutgoingUIEvent.wrappedOf("metrics", metrics));
     }

photon-core/src/main/java/org/photonvision/common/hardware/metrics/cmds/CmdBase.java

@@ -29,6 +29,8 @@ public class CmdBase {
     // GPU
     public String gpuMemoryCommand = "";
     public String gpuMemUsageCommand = "";
+    // NPU
+    public String npuUsageCommand = "";
     // RAM
     public String ramUsageCommand = "";
     // Disk

photon-core/src/main/java/org/photonvision/common/hardware/metrics/cmds/RK3588Cmds.java

@@ -44,5 +44,7 @@ public class RK3588Cmds extends LinuxCmds {
          */
         cpuTemperatureCommand =
                 "cat /sys/class/thermal/thermal_zone1/temp | awk '{printf \"%.1f\", $1/1000}'";
+
+        npuUsageCommand = "cat /sys/kernel/debug/rknpu/load | sed 's/NPU load://; s/^ *//; s/ *$//'";
     }
 }

photon-core/src/main/java/org/photonvision/jni/RknnDetectorJNI.java

@@ -65,12 +65,11 @@ public class RknnDetectorJNI extends PhotonJNICommon {
         long objPointer = -1;
         private List<String> labels;
         private final Object lock = new Object();
-
         private static final CopyOnWriteArrayList<Long> detectors = new CopyOnWriteArrayList<>();
 
-        public RknnObjectDetector(String modelPath, List<String> labels) {
+        public RknnObjectDetector(String modelPath, List<String> labels, RknnJNI.ModelVersion version) {
             synchronized (lock) {
-                objPointer = RknnJNI.create(modelPath, labels.size());
+                objPointer = RknnJNI.create(modelPath, labels.size(), version.ordinal(), -1);
                 detectors.add(objPointer);
                 System.out.println(
                         "Created " + objPointer + "! Detectors: " + Arrays.toString(detectors.toArray()));

photon-core/src/main/java/org/photonvision/vision/calibration/BoardObservation.java

@@ -24,7 +24,7 @@ import java.util.List;
 import org.opencv.core.Point;
 import org.opencv.core.Point3;
 
-public final class BoardObservation {
+public final class BoardObservation implements Cloneable {
     // Expected feature 3d location in the camera frame
     @JsonProperty("locationInObjectSpace")
     public List<Point3> locationInObjectSpace;
@@ -68,4 +68,33 @@ public final class BoardObservation {
         this.snapshotName = snapshotName;
         this.snapshotData = snapshotData;
     }
+
+    @Override
+    public String toString() {
+        return "BoardObservation [locationInObjectSpace="
+                + locationInObjectSpace
+                + ", locationInImageSpace="
+                + locationInImageSpace
+                + ", reprojectionErrors="
+                + reprojectionErrors
+                + ", optimisedCameraToObject="
+                + optimisedCameraToObject
+                + ", includeObservationInCalibration="
+                + includeObservationInCalibration
+                + ", snapshotName="
+                + snapshotName
+                + ", snapshotData="
+                + snapshotData
+                + "]";
+    }
+
+    @Override
+    public BoardObservation clone() {
+        try {
+            return (BoardObservation) super.clone();
+        } catch (CloneNotSupportedException e) {
+            System.err.println("Guhhh clone buh");
+            return null;
+        }
+    }
 }

photon-core/src/main/java/org/photonvision/vision/calibration/CameraCalibrationCoefficients.java

@@ -191,8 +191,8 @@ public class CameraCalibrationCoefficients implements Releasable {
                 + cameraIntrinsics
                 + ", distCoeffs="
                 + distCoeffs
-                + ", observations="
-                + observations
+                + ", observationslen="
+                + observations.size()
                 + ", calobjectWarp="
                 + Arrays.toString(calobjectWarp)
                 + ", intrinsicsArr="
@@ -201,4 +201,16 @@ public class CameraCalibrationCoefficients implements Releasable {
                 + Arrays.toString(distCoeffsArr)
                 + "]";
     }
+
+    public UICameraCalibrationCoefficients cloneWithoutObservations() {
+        return new UICameraCalibrationCoefficients(
+                resolution,
+                cameraIntrinsics,
+                distCoeffs,
+                calobjectWarp,
+                observations,
+                calobjectSize,
+                calobjectSpacing,
+                lensmodel);
+    }
 }

photon-core/src/main/java/org/photonvision/vision/calibration/JsonImageMat.java

@@ -76,4 +76,17 @@ public class JsonImageMat implements Releasable {
     public void release() {
         if (wrappedMat != null) wrappedMat.release();
     }
+
+    @Override
+    public String toString() {
+        return "JsonImageMat [rows="
+                + rows
+                + ", cols="
+                + cols
+                + ", type="
+                + type
+                + ", datalen="
+                + data.length()
+                + "]";
+    }
 }

photon-core/src/main/java/org/photonvision/vision/calibration/JsonMatOfDouble.java

@@ -40,7 +40,7 @@ public class JsonMatOfDouble implements Releasable {
     @JsonIgnore private Mat wrappedMat = null;
     @JsonIgnore private Matrix wpilibMat = null;
 
-    private MatOfDouble wrappedMatOfDouble;
+    @JsonIgnore private MatOfDouble wrappedMatOfDouble;
 
     public JsonMatOfDouble(int rows, int cols, double[] data) {
         this(rows, cols, CvType.CV_64FC1, data);

photon-core/src/main/java/org/photonvision/vision/calibration/UICameraCalibrationCoefficients.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.vision.calibration;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import org.opencv.core.Size;
+
+public class UICameraCalibrationCoefficients extends CameraCalibrationCoefficients {
+    public int numSnapshots;
+    public List<Double> meanErrors;
+
+    public UICameraCalibrationCoefficients(
+            Size resolution,
+            JsonMatOfDouble cameraIntrinsics,
+            JsonMatOfDouble distCoeffs,
+            double[] calobjectWarp,
+            List<BoardObservation> observations,
+            Size calobjectSize,
+            double calobjectSpacing,
+            CameraLensModel lensmodel) {
+        // yeet observations, keep all else
+        super(
+                resolution,
+                cameraIntrinsics,
+                distCoeffs,
+                calobjectWarp,
+                List.of(),
+                calobjectSize,
+                calobjectSpacing,
+                lensmodel);
+
+        this.numSnapshots = observations.size();
+        this.meanErrors =
+                observations.stream()
+                        .map(
+                                it2 ->
+                                        it2.reprojectionErrors.stream()
+                                                .mapToDouble(it -> Math.sqrt(it.x * it.x + it.y * it.y))
+                                                .average()
+                                                .orElse(0))
+                        .collect(Collectors.toList());
+    }
+}

photon-core/src/main/java/org/photonvision/vision/camera/USBCameraSource.java

@@ -395,6 +395,7 @@ public class USBCameraSource extends VisionSource {
                 // Sort by resolution
                 var sortedList =
                         videoModesList.stream()
+                                .distinct() // remove redundant video mode entries
                                 .sorted(((a, b) -> (b.width + b.height) - (a.width + a.height)))
                                 .collect(Collectors.toList());
                 Collections.reverse(sortedList);

photon-core/src/main/java/org/photonvision/vision/opencv/Contour.java

@@ -47,6 +47,16 @@ public class Contour implements Releasable {
         this.mat = mat;
     }
 
+    public Contour(Rect2d box) {
+        // no easy way to convert a Rect2d to Mat, diy it. Order is tl tr br bl
+        this.mat =
+                new MatOfPoint(
+                        box.tl(),
+                        new Point(box.x + box.width, box.y),
+                        box.br(),
+                        new Point(box.x, box.y + box.height));
+    }
+
     public MatOfPoint2f getMat2f() {
         if (mat2f == null) {
             mat2f = new MatOfPoint2f(mat.toArray());

photon-core/src/main/java/org/photonvision/vision/opencv/ContourSortMode.java

@@ -25,15 +25,15 @@ public enum ContourSortMode {
             Comparator.comparingDouble(PotentialTarget::getArea)
                     .reversed()), // reversed so that zero index has the largest size
     Smallest(Largest.getComparator().reversed()),
-    Highest(Comparator.comparingDouble(rect -> rect.getMinAreaRect().center.y)),
+    Highest(Comparator.comparingDouble(tgt -> tgt.getMinAreaRect().center.y)),
     Lowest(Highest.getComparator().reversed()),
-    Leftmost(Comparator.comparingDouble(target -> target.getMinAreaRect().center.x * -1)),
+    Leftmost(Comparator.comparingDouble(tgt -> tgt.getMinAreaRect().center.x * -1)),
     Rightmost(Leftmost.getComparator().reversed()),
     Centermost(
             Comparator.comparingDouble(
-                    rect ->
-                            (Math.pow(rect.getMinAreaRect().center.y, 2)
-                                    + Math.pow(rect.getMinAreaRect().center.x, 2))));
+                    tgt ->
+                            (Math.pow(tgt.getMinAreaRect().center.y, 2)
+                                    + Math.pow(tgt.getMinAreaRect().center.x, 2))));
 
     private final Comparator<PotentialTarget> m_comparator;
 

photon-core/src/main/java/org/photonvision/vision/pipe/impl/FilterObjectDetectionsPipe.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.vision.pipe.impl;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.photonvision.common.util.numbers.DoubleCouple;
+import org.photonvision.vision.frame.FrameStaticProperties;
+import org.photonvision.vision.pipe.CVPipe;
+
+public class FilterObjectDetectionsPipe
+        extends CVPipe<
+                List<NeuralNetworkPipeResult>,
+                List<NeuralNetworkPipeResult>,
+                FilterObjectDetectionsPipe.FilterContoursParams> {
+    List<NeuralNetworkPipeResult> m_filteredContours = new ArrayList<>();
+
+    @Override
+    protected List<NeuralNetworkPipeResult> process(List<NeuralNetworkPipeResult> in) {
+        m_filteredContours.clear();
+        for (var contour : in) {
+            filterContour(contour);
+        }
+
+        return m_filteredContours;
+    }
+
+    private void filterContour(NeuralNetworkPipeResult contour) {
+        var boc = contour.box;
+
+        // Area filtering
+        double areaPercentage = boc.area() / params.getFrameStaticProperties().imageArea * 100.0;
+        double minAreaPercentage = params.getArea().getFirst();
+        double maxAreaPercentage = params.getArea().getSecond();
+        if (areaPercentage < minAreaPercentage || areaPercentage > maxAreaPercentage) return;
+
+        // Aspect ratio filtering; much simpler since always axis-aligned
+        double aspectRatio = boc.width / boc.height;
+        if (aspectRatio < params.getRatio().getFirst() || aspectRatio > params.getRatio().getSecond())
+            return;
+
+        m_filteredContours.add(contour);
+    }
+
+    public static class FilterContoursParams {
+        private final DoubleCouple m_area;
+        private final DoubleCouple m_ratio;
+        private final FrameStaticProperties m_frameStaticProperties;
+        public final boolean isLandscape;
+
+        public FilterContoursParams(
+                DoubleCouple area,
+                DoubleCouple ratio,
+                FrameStaticProperties camProperties,
+                boolean isLandscape) {
+            this.m_area = area;
+            this.m_ratio = ratio;
+            this.m_frameStaticProperties = camProperties;
+            this.isLandscape = isLandscape;
+        }
+
+        public DoubleCouple getArea() {
+            return m_area;
+        }
+
+        public DoubleCouple getRatio() {
+            return m_ratio;
+        }
+
+        public FrameStaticProperties getFrameStaticProperties() {
+            return m_frameStaticProperties;
+        }
+    }
+}

photon-core/src/main/java/org/photonvision/vision/pipe/impl/RknnDetectionPipe.java

@@ -35,7 +35,8 @@ public class RknnDetectionPipe
         this.detector =
                 new RknnObjectDetector(
                         NeuralNetworkModelManager.getInstance().getDefaultRknnModel().getAbsolutePath(),
-                        NeuralNetworkModelManager.getInstance().getLabels());
+                        NeuralNetworkModelManager.getInstance().getLabels(),
+                        NeuralNetworkModelManager.getInstance().getModelVersion());
     }
 
     @Override

photon-core/src/main/java/org/photonvision/vision/pipe/impl/SortContoursPipe.java

@@ -42,6 +42,7 @@ public class SortContoursPipe
             if (params.getSortMode() != ContourSortMode.Centermost) {
                 m_sortedContours.sort(params.getSortMode().getComparator());
             } else {
+                // we need knowledge of camera properties to calculate this distance -- do it ourselves
                 m_sortedContours.sort(Comparator.comparingDouble(this::calcSquareCenterDistance));
             }
         }
@@ -50,10 +51,10 @@ public class SortContoursPipe
                 m_sortedContours.subList(0, Math.min(in.size(), params.getMaxTargets())));
     }
 
-    private double calcSquareCenterDistance(PotentialTarget rect) {
+    private double calcSquareCenterDistance(PotentialTarget tgt) {
         return Math.sqrt(
-                Math.pow(params.getCamProperties().centerX - rect.getMinAreaRect().center.x, 2)
-                        + Math.pow(params.getCamProperties().centerY - rect.getMinAreaRect().center.y, 2));
+                Math.pow(params.getCamProperties().centerX - tgt.getMinAreaRect().center.x, 2)
+                        + Math.pow(params.getCamProperties().centerY - tgt.getMinAreaRect().center.y, 2));
     }
 
     public static class SortContoursParams {

photon-core/src/main/java/org/photonvision/vision/pipeline/CVPipeline.java

@@ -26,6 +26,8 @@ import org.photonvision.vision.pipeline.result.CVPipelineResult;
 
 public abstract class CVPipeline<R extends CVPipelineResult, S extends CVPipelineSettings>
         implements Releasable {
+    static final int MAX_MULTI_TARGET_RESULTS = 10;
+
     protected S settings;
     protected FrameStaticProperties frameStaticProperties;
     protected QuirkyCamera cameraQuirks;

photon-core/src/main/java/org/photonvision/vision/pipeline/ColoredShapePipeline.java

@@ -109,7 +109,7 @@ public class ColoredShapePipeline
         SortContoursPipe.SortContoursParams sortContoursParams =
                 new SortContoursPipe.SortContoursParams(
                         settings.contourSortMode,
-                        settings.outputShowMultipleTargets ? 5 : 1,
+                        settings.outputShowMultipleTargets ? MAX_MULTI_TARGET_RESULTS : 1,
                         frameStaticProperties); // TODO don't hardcode?
         sortContoursPipe.setParams(sortContoursParams);
 

photon-core/src/main/java/org/photonvision/vision/pipeline/ObjectDetectionPipeline.java

@@ -17,21 +17,26 @@
 
 package org.photonvision.vision.pipeline;
 
-import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 import org.photonvision.vision.frame.Frame;
 import org.photonvision.vision.frame.FrameThresholdType;
+import org.photonvision.vision.opencv.DualOffsetValues;
 import org.photonvision.vision.pipe.CVPipe.CVPipeResult;
 import org.photonvision.vision.pipe.impl.*;
 import org.photonvision.vision.pipe.impl.RknnDetectionPipe.RknnDetectionPipeParams;
 import org.photonvision.vision.pipeline.result.CVPipelineResult;
+import org.photonvision.vision.target.PotentialTarget;
+import org.photonvision.vision.target.TargetOrientation;
 import org.photonvision.vision.target.TrackedTarget;
-import org.photonvision.vision.target.TrackedTarget.TargetCalculationParameters;
 
 public class ObjectDetectionPipeline
         extends CVPipeline<CVPipelineResult, ObjectDetectionPipelineSettings> {
     private final CalculateFPSPipe calculateFPSPipe = new CalculateFPSPipe();
     private final RknnDetectionPipe rknnPipe = new RknnDetectionPipe();
+    private final SortContoursPipe sortContoursPipe = new SortContoursPipe();
+    private final Collect2dTargetsPipe collect2dTargetsPipe = new Collect2dTargetsPipe();
+    private final FilterObjectDetectionsPipe filterContoursPipe = new FilterObjectDetectionsPipe();
 
     private static final FrameThresholdType PROCESSING_TYPE = FrameThresholdType.NONE;
 
@@ -52,6 +57,38 @@ public class ObjectDetectionPipeline
         params.confidence = settings.confidence;
         params.nms = settings.nms;
         rknnPipe.setParams(params);
+
+        DualOffsetValues dualOffsetValues =
+                new DualOffsetValues(
+                        settings.offsetDualPointA,
+                        settings.offsetDualPointAArea,
+                        settings.offsetDualPointB,
+                        settings.offsetDualPointBArea);
+
+        SortContoursPipe.SortContoursParams sortContoursParams =
+                new SortContoursPipe.SortContoursParams(
+                        settings.contourSortMode,
+                        settings.outputShowMultipleTargets ? MAX_MULTI_TARGET_RESULTS : 1,
+                        frameStaticProperties);
+        sortContoursPipe.setParams(sortContoursParams);
+
+        var filterContoursParams =
+                new FilterObjectDetectionsPipe.FilterContoursParams(
+                        settings.contourArea,
+                        settings.contourRatio,
+                        frameStaticProperties,
+                        settings.contourTargetOrientation == TargetOrientation.Landscape);
+        filterContoursPipe.setParams(filterContoursParams);
+
+        Collect2dTargetsPipe.Collect2dTargetsParams collect2dTargetsParams =
+                new Collect2dTargetsPipe.Collect2dTargetsParams(
+                        settings.offsetRobotOffsetMode,
+                        settings.offsetSinglePoint,
+                        dualOffsetValues,
+                        settings.contourTargetOffsetPointEdge,
+                        settings.contourTargetOrientation,
+                        frameStaticProperties);
+        collect2dTargetsPipe.setParams(collect2dTargetsParams);
     }
 
     @Override
@@ -60,31 +97,35 @@ public class ObjectDetectionPipeline
 
         // ***************** change based on backend ***********************
 
-        CVPipeResult<List<NeuralNetworkPipeResult>> ret = rknnPipe.run(input_frame.colorImage);
-        sumPipeNanosElapsed += ret.nanosElapsed;
+        CVPipeResult<List<NeuralNetworkPipeResult>> rknnResult = rknnPipe.run(input_frame.colorImage);
+        sumPipeNanosElapsed += rknnResult.nanosElapsed;
         List<NeuralNetworkPipeResult> targetList;
 
-        targetList = ret.output;
         var names = rknnPipe.getClassNames();
 
         input_frame.colorImage.getMat().copyTo(input_frame.processedImage.getMat());
 
         // ***************** change based on backend ***********************
 
-        List<TrackedTarget> targets = new ArrayList<>();
+        var filterContoursResult = filterContoursPipe.run(rknnResult.output);
+        sumPipeNanosElapsed += filterContoursResult.nanosElapsed;
 
-        for (var t : targetList) {
-            targets.add(
-                    new TrackedTarget(
-                            t,
-                            new TargetCalculationParameters(
-                                    false, null, null, null, null, frameStaticProperties)));
-        }
+        CVPipeResult<List<PotentialTarget>> sortContoursResult =
+                sortContoursPipe.run(
+                        filterContoursResult.output.stream()
+                                .map(shape -> new PotentialTarget(shape))
+                                .collect(Collectors.toList()));
+        sumPipeNanosElapsed += sortContoursResult.nanosElapsed;
+
+        CVPipeResult<List<TrackedTarget>> collect2dTargetsResult =
+                collect2dTargetsPipe.run(sortContoursResult.output);
+        sumPipeNanosElapsed += collect2dTargetsResult.nanosElapsed;
 
         var fpsResult = calculateFPSPipe.run(null);
         var fps = fpsResult.output;
 
-        return new CVPipelineResult(sumPipeNanosElapsed, fps, targets, input_frame, names);
+        return new CVPipelineResult(
+                sumPipeNanosElapsed, fps, collect2dTargetsResult.output, input_frame, names);
     }
 
     @Override

photon-core/src/main/java/org/photonvision/vision/pipeline/ReflectivePipeline.java

@@ -64,29 +64,6 @@ public class ReflectivePipeline extends CVPipeline<CVPipelineResult, ReflectiveP
                         settings.offsetDualPointB,
                         settings.offsetDualPointBArea);
 
-        // var rotateImageParams = new
-        // RotateImagePipe.RotateImageParams(settings.inputImageRotationMode);
-        // rotateImagePipe.setParams(rotateImageParams);
-
-        // if (cameraQuirks.hasQuirk(CameraQuirk.PiCam) && LibCameraJNI.isSupported()) {
-        //     LibCameraJNI.setThresholds(
-        //             settings.hsvHue.getFirst() / 180d,
-        //             settings.hsvSaturation.getFirst() / 255d,
-        //             settings.hsvValue.getFirst() / 255d,
-        //             settings.hsvHue.getSecond() / 180d,
-        //             settings.hsvSaturation.getSecond() / 255d,
-        //             settings.hsvValue.getSecond() / 255d);
-        //     //     LibCameraJNI.setInvertHue(settings.hueInverted);
-        //     LibCameraJNI.setRotation(settings.inputImageRotationMode.value);
-        //     //     LibCameraJNI.setShouldCopyColor(settings.inputShouldShow);
-        // } else {
-        //     var hsvParams =
-        //             new HSVPipe.HSVParams(
-        //                     settings.hsvHue, settings.hsvSaturation, settings.hsvValue,
-        // settings.hueInverted);
-        //     hsvPipe.setParams(hsvParams);
-        // }
-
         var findContoursParams = new FindContoursPipe.FindContoursParams();
         findContoursPipe.setParams(findContoursParams);
 
@@ -113,7 +90,7 @@ public class ReflectivePipeline extends CVPipeline<CVPipelineResult, ReflectiveP
         var sortContoursParams =
                 new SortContoursPipe.SortContoursParams(
                         settings.contourSortMode,
-                        settings.outputShowMultipleTargets ? 8 : 1, // TODO don't hardcode?
+                        settings.outputShowMultipleTargets ? MAX_MULTI_TARGET_RESULTS : 1,
                         frameStaticProperties);
         sortContoursPipe.setParams(sortContoursParams);
 

photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java

@@ -26,6 +26,7 @@ import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
 import org.opencv.core.Size;
 import org.photonvision.common.configuration.CameraConfiguration;
 import org.photonvision.common.configuration.ConfigManager;
@@ -536,7 +537,10 @@ public class VisionModule {
         ret.outputStreamPort = this.outputStreamPort;
         ret.inputStreamPort = this.inputStreamPort;
 
-        ret.calibrations = visionSource.getSettables().getConfiguration().calibrations;
+        ret.calibrations =
+                visionSource.getSettables().getConfiguration().calibrations.stream()
+                        .map(CameraCalibrationCoefficients::cloneWithoutObservations)
+                        .collect(Collectors.toList());
 
         ret.isFovConfigurable =
                 !(ConfigManager.getInstance().getConfig().getHardwareConfig().hasPresetFOV()

photon-core/src/main/java/org/photonvision/vision/processes/VisionRunner.java

@@ -98,8 +98,7 @@ public class VisionRunner {
                 var pipelineResult = pipeline.run(frame, cameraQuirks);
                 pipelineResultConsumer.accept(pipelineResult);
             } catch (Exception ex) {
-                logger.error("Exception on loop " + loopCount);
-                ex.printStackTrace();
+                logger.error("Exception on loop " + loopCount, ex);
             }
 
             loopCount++;

photon-core/src/main/java/org/photonvision/vision/target/PotentialTarget.java

@@ -21,7 +21,9 @@ import java.util.List;
 import org.opencv.core.RotatedRect;
 import org.photonvision.vision.opencv.CVShape;
 import org.photonvision.vision.opencv.Contour;
+import org.photonvision.vision.opencv.ContourShape;
 import org.photonvision.vision.opencv.Releasable;
+import org.photonvision.vision.pipe.impl.NeuralNetworkPipeResult;
 
 public class PotentialTarget implements Releasable {
 
@@ -29,6 +31,10 @@ public class PotentialTarget implements Releasable {
     public final List<Contour> m_subContours;
     public final CVShape shape;
 
+    // additional metadata about object detections we need to keep around
+    public final double confidence;
+    public final int clsId;
+
     public PotentialTarget(Contour inputContour) {
         this(inputContour, List.of());
     }
@@ -41,12 +47,26 @@ public class PotentialTarget implements Releasable {
         m_mainContour = inputContour;
         m_subContours = new ArrayList<>(subContours);
         this.shape = shape;
+        this.clsId = -1;
+        this.confidence = -1;
     }
 
     public PotentialTarget(Contour inputContour, CVShape shape) {
         this(inputContour, List.of(), shape);
     }
 
+    public PotentialTarget(NeuralNetworkPipeResult det) {
+        this.shape = new CVShape(new Contour(det.box), ContourShape.Quadrilateral);
+        this.m_mainContour = this.shape.getContour();
+        m_subContours = List.of();
+        this.clsId = det.classIdx;
+        this.confidence = det.confidence;
+    }
+
+    public PotentialTarget(CVShape cvShape) {
+        this(cvShape.getContour(), cvShape);
+    }
+
     public RotatedRect getMinAreaRect() {
         return m_mainContour.getMinAreaRect();
     }
@@ -61,7 +81,7 @@ public class PotentialTarget implements Releasable {
         for (var sc : m_subContours) {
             sc.release();
         }
-        m_subContours.clear();
+        if (!m_subContours.isEmpty()) m_subContours.clear();
         if (shape != null) shape.release();
     }
 }

photon-core/src/main/java/org/photonvision/vision/target/TrackedTarget.java

@@ -27,7 +27,6 @@ import org.opencv.core.Mat;
 import org.opencv.core.MatOfPoint;
 import org.opencv.core.MatOfPoint2f;
 import org.opencv.core.Point;
-import org.opencv.core.Rect2d;
 import org.opencv.core.RotatedRect;
 import org.photonvision.common.util.SerializationUtils;
 import org.photonvision.common.util.math.MathUtils;
@@ -39,7 +38,6 @@ import org.photonvision.vision.opencv.CVShape;
 import org.photonvision.vision.opencv.Contour;
 import org.photonvision.vision.opencv.DualOffsetValues;
 import org.photonvision.vision.opencv.Releasable;
-import org.photonvision.vision.pipe.impl.NeuralNetworkPipeResult;
 
 public class TrackedTarget implements Releasable {
     public final Contour m_mainContour;
@@ -76,6 +74,9 @@ public class TrackedTarget implements Releasable {
         this.m_subContours = origTarget.m_subContours;
         this.m_shape = shape;
         calculateValues(params);
+
+        this.m_classId = origTarget.clsId;
+        this.m_confidence = origTarget.confidence;
     }
 
     public TrackedTarget(
@@ -159,47 +160,6 @@ public class TrackedTarget implements Releasable {
         m_robotOffsetPoint = new Point();
     }
 
-    public TrackedTarget(
-            Rect2d box, int class_id, double confidence, TargetCalculationParameters params) {
-        m_targetOffsetPoint = new Point(box.x + box.width / 2.0, box.y + box.height / 2.0);
-        m_robotOffsetPoint = new Point();
-
-        var yawPitch =
-                TargetCalculations.calculateYawPitch(
-                        params.cameraCenterPoint.x,
-                        box.x + box.width / 2.0,
-                        params.horizontalFocalLength,
-                        params.cameraCenterPoint.y,
-                        box.y + box.height / 2.0,
-                        params.verticalFocalLength);
-        m_yaw = yawPitch.getFirst();
-        m_pitch = yawPitch.getSecond();
-
-        Point[] cornerPoints =
-                new Point[] {
-                    // Box.x/y is the top-left corner, not the center
-                    new Point(box.x, box.y), // tl
-                    new Point(box.x + box.width, box.y), // tr
-                    new Point(box.x + box.width, box.y + box.height), // br
-                    new Point(box.x, box.y + box.height), // bl
-                };
-
-        m_targetCorners = List.of(cornerPoints);
-        MatOfPoint contourMat = new MatOfPoint(cornerPoints);
-        m_approximateBoundingPolygon = new MatOfPoint2f(cornerPoints);
-
-        m_mainContour = new Contour(contourMat);
-        m_area = m_mainContour.getArea() / params.imageArea * 100;
-
-        m_classId = class_id;
-        m_confidence = confidence;
-    }
-
-    public TrackedTarget(
-            NeuralNetworkPipeResult t, TargetCalculationParameters targetCalculationParameters) {
-        this(t.box, t.classIdx, t.confidence, targetCalculationParameters);
-    }
-
     /**
      * @return Returns the confidence of the detection ranging from 0 - 1.
      */

photon-lib/py/photonlibpy/estimatedRobotPose.py

@@ -19,7 +19,7 @@ class EstimatedRobotPose:
     timestampSeconds: float
     """The estimated time the frame used to derive the robot pose was taken"""
 
-    targetsUsed: [PhotonTrackedTarget]
+    targetsUsed: list[PhotonTrackedTarget]
     """A list of the targets used to compute this pose"""
 
     strategy: "PoseStrategy"

photon-lib/py/photonlibpy/packet.py

@@ -4,7 +4,7 @@ import wpilib
 
 
 class Packet:
-    def __init__(self, data: list[int]):
+    def __init__(self, data: bytes):
         """
         * Constructs an empty packet.
         *
@@ -30,7 +30,7 @@ class Packet:
     matches the version of photonlib running in the robot code.
     """
 
-    def _getNextByte(self) -> int:
+    def _getNextByteAsInt(self) -> int:
         retVal = 0x00
 
         if not self.outOfBytes:
@@ -43,7 +43,7 @@ class Packet:
 
         return retVal
 
-    def getData(self) -> list[int]:
+    def getData(self) -> bytes:
         """
         * Returns the packet data.
         *
@@ -51,7 +51,7 @@ class Packet:
         """
         return self.packetData
 
-    def setData(self, data: list[int]):
+    def setData(self, data: bytes):
         """
         * Sets the packet data.
         *
@@ -65,7 +65,7 @@ class Packet:
         # Read ints in from the data buffer
         intList = []
         for _ in range(numBytes):
-            intList.append(self._getNextByte())
+            intList.append(self._getNextByteAsInt())
 
         # Interpret the bytes as a floating point number
         value = struct.unpack(unpackFormat, bytes(intList))[0]

photon-lib/py/photonlibpy/photonCamera.py

@@ -4,7 +4,7 @@ from wpilib import Timer
 import wpilib
 from photonlibpy.packet import Packet
 from photonlibpy.photonPipelineResult import PhotonPipelineResult
-from photonlibpy.version import PHOTONVISION_VERSION, PHOTONLIB_VERSION
+from photonlibpy.version import PHOTONVISION_VERSION, PHOTONLIB_VERSION  # type: ignore[import-untyped]
 
 
 class VisionLEDMode(Enum):
@@ -86,10 +86,11 @@ class PhotonCamera:
         if len(byteList) < 1:
             return retVal
         else:
-            retVal.populateFromPacket(Packet(byteList))
+            pkt = Packet(byteList)
+            retVal.populateFromPacket(pkt)
             # NT4 allows us to correct the timestamp based on when the message was sent
             retVal.setTimestampSeconds(
-                timestamp / 1e-6 - retVal.getLatencyMillis() / 1e-3
+                timestamp / 1e6 - retVal.getLatencyMillis() / 1e3
             )
             return retVal
 

photon-lib/py/photonlibpy/photonPipelineResult.py

@@ -38,3 +38,6 @@ class PhotonPipelineResult:
 
     def getTargets(self) -> list[PhotonTrackedTarget]:
         return self.targets
+
+    def hasTargets(self) -> bool:
+        return len(self.targets) > 0

photon-lib/py/photonlibpy/photonPoseEstimator.py

@@ -75,7 +75,7 @@ class PhotonPoseEstimator:
 
         self._multiTagFallbackStrategy = PoseStrategy.LOWEST_AMBIGUITY
         self._reportedErrors: set[int] = set()
-        self._poseCacheTimestampSeconds = -1
+        self._poseCacheTimestampSeconds = -1.0
         self._lastPose: Optional[Pose3d] = None
         self._referencePose: Optional[Pose3d] = None
 
@@ -143,7 +143,7 @@ class PhotonPoseEstimator:
         self._multiTagFallbackStrategy = strategy
 
     @property
-    def referencePose(self) -> Pose3d:
+    def referencePose(self) -> Optional[Pose3d]:
         """Return the reference position that is being used by the estimator.
 
         :returns: the referencePose
@@ -163,7 +163,7 @@ class PhotonPoseEstimator:
         self._referencePose = referencePose
 
     @property
-    def lastPose(self) -> Pose3d:
+    def lastPose(self) -> Optional[Pose3d]:
         return self._lastPose
 
     @lastPose.setter
@@ -178,10 +178,10 @@ class PhotonPoseEstimator:
         self._checkUpdate(self._lastPose, lastPose)
         self._lastPose = lastPose
 
-    def _invalidatePoseCache(self):
-        self._poseCacheTimestampSeconds = -1
+    def _invalidatePoseCache(self) -> None:
+        self._poseCacheTimestampSeconds = -1.0
 
-    def _checkUpdate(self, oldObj, newObj):
+    def _checkUpdate(self, oldObj, newObj) -> None:
         if oldObj != newObj and oldObj is not None and oldObj is not newObj:
             self._invalidatePoseCache()
 
@@ -204,27 +204,27 @@ class PhotonPoseEstimator:
         if not cameraResult:
             if not self._camera:
                 wpilib.reportError("[PhotonPoseEstimator] Missing camera!", False)
-                return
+                return None
             cameraResult = self._camera.getLatestResult()
 
         if cameraResult.timestampSec < 0:
-            return
+            return None
 
         # If the pose cache timestamp was set, and the result is from the same
         # timestamp, return an
         # empty result
         if (
-            self._poseCacheTimestampSeconds > 0
+            self._poseCacheTimestampSeconds > 0.0
             and abs(self._poseCacheTimestampSeconds - cameraResult.timestampSec) < 1e-6
         ):
-            return
+            return None
 
         # Remember the timestamp of the current result used
         self._poseCacheTimestampSeconds = cameraResult.timestampSec
 
         # If no targets seen, trivial case -- return empty result
         if not cameraResult.targets:
-            return
+            return None
 
         return self._update(cameraResult, self._primaryStrategy)
 
@@ -239,7 +239,7 @@ class PhotonPoseEstimator:
             wpilib.reportError(
                 "[PhotonPoseEstimator] Unknown Position Estimation Strategy!", False
             )
-            return
+            return None
 
         if not estimatedPose:
             self._lastPose = None
@@ -280,7 +280,7 @@ class PhotonPoseEstimator:
         """
         lowestAmbiguityTarget = None
 
-        lowestAmbiguityScore = 10
+        lowestAmbiguityScore = 10.0
 
         for target in result.targets:
             targetPoseAmbiguity = target.poseAmbiguity
@@ -293,7 +293,7 @@ class PhotonPoseEstimator:
         # Although there are confirmed to be targets, none of them may be fiducial
         # targets.
         if not lowestAmbiguityTarget:
-            return
+            return None
 
         targetFiducialId = lowestAmbiguityTarget.fiducialId
 
@@ -301,7 +301,7 @@ class PhotonPoseEstimator:
 
         if not targetPosition:
             self._reportFiducialPoseError(targetFiducialId)
-            return
+            return None
 
         return EstimatedRobotPose(
             targetPosition.transformBy(

photon-lib/src/main/java/org/photonvision/PhotonPoseEstimator.java

@@ -408,8 +408,8 @@ public class PhotonPoseEstimator {
                 return Optional.empty();
         }
 
-        if (estimatedPose.isEmpty()) {
-            lastPose = null;
+        if (estimatedPose.isPresent()) {
+            lastPose = estimatedPose.get().estimatedPose;
         }
 
         return estimatedPose;

photon-lib/src/main/native/cpp/photon/PhotonPoseEstimator.cpp

@@ -186,6 +186,9 @@ std::optional<EstimatedRobotPose> PhotonPoseEstimator::Update(
       ret = std::nullopt;
   }
 
+  if (ret) {
+    lastPose = ret.value().estimatedPose;
+  }
   return ret;
 }
 

photon-lib/src/main/native/include/photon/simulation/PhotonCameraSim.h

@@ -435,7 +435,7 @@ class PhotonCameraSim {
   double minTargetAreaPercent;
 
   frc::AprilTagFieldLayout tagLayout{
-      frc::LoadAprilTagLayoutField(frc::AprilTagField::k2023ChargedUp)};
+      frc::LoadAprilTagLayoutField(frc::AprilTagField::k2024Crescendo)};
 
   cs::CvSource videoSimRaw;
   cv::Mat videoSimFrameRaw{};

photon-server/src/main/java/org/photonvision/server/DataSocketHandler.java

@@ -350,8 +350,7 @@ public class DataSocketHandler {
         }
     }
 
-    private void sendMessage(Object message, WsContext user) throws JsonProcessingException {
-        ByteBuffer b = ByteBuffer.wrap(objectMapper.writeValueAsBytes(message));
+    private void sendMessage(ByteBuffer b, WsContext user) throws JsonProcessingException {
         if (user.session.isOpen()) {
             user.send(b);
         }
@@ -359,16 +358,18 @@ public class DataSocketHandler {
 
     public void broadcastMessage(Object message, WsContext userToSkip)
             throws JsonProcessingException {
+        ByteBuffer b = ByteBuffer.wrap(objectMapper.writeValueAsBytes(message));
+
         if (userToSkip == null) {
             for (WsContext user : users) {
-                sendMessage(message, user);
+                sendMessage(b, user);
             }
         } else {
             var skipUserPort = ((InetSocketAddress) userToSkip.session.getRemoteAddress()).getPort();
             for (WsContext user : users) {
                 var userPort = ((InetSocketAddress) user.session.getRemoteAddress()).getPort();
                 if (userPort != skipUserPort) {
-                    sendMessage(message, user);
+                    sendMessage(b, user);
                 }
             }
         }

photon-server/src/main/java/org/photonvision/server/RequestHandler.java

@@ -31,6 +31,9 @@ import java.util.HashMap;
 import java.util.Optional;
 import javax.imageio.ImageIO;
 import org.apache.commons.io.FileUtils;
+import org.opencv.core.MatOfByte;
+import org.opencv.core.MatOfInt;
+import org.opencv.imgcodecs.Imgcodecs;
 import org.photonvision.common.configuration.ConfigManager;
 import org.photonvision.common.configuration.NetworkConfig;
 import org.photonvision.common.dataflow.DataChangeDestination;
@@ -580,6 +583,77 @@ public class RequestHandler {
         ctx.status(204);
     }
 
+    public static void onCalibrationSnapshotRequest(Context ctx) {
+        logger.info(ctx.queryString().toString());
+
+        int idx = Integer.parseInt(ctx.queryParam("cameraIdx"));
+        var width = Integer.parseInt(ctx.queryParam("width"));
+        var height = Integer.parseInt(ctx.queryParam("height"));
+        var observationIdx = Integer.parseInt(ctx.queryParam("snapshotIdx"));
+
+        CameraCalibrationCoefficients calList =
+                VisionModuleManager.getInstance()
+                        .getModule(idx)
+                        .getStateAsCameraConfig()
+                        .calibrations
+                        .stream()
+                        .filter(
+                                it ->
+                                        Math.abs(it.resolution.width - width) < 1e-4
+                                                && Math.abs(it.resolution.height - height) < 1e-4)
+                        .findFirst()
+                        .orElse(null);
+
+        if (calList == null || calList.observations.size() < observationIdx) {
+            ctx.status(404);
+            return;
+        }
+
+        // encode as jpeg to save even more space. reduces size of a 1280p image from 300k to 25k
+        var jpegBytes = new MatOfByte();
+        Imgcodecs.imencode(
+                ".jpg",
+                calList.observations.get(observationIdx).snapshotData.getAsMat(),
+                jpegBytes,
+                new MatOfInt(Imgcodecs.IMWRITE_JPEG_QUALITY, 60));
+
+        ctx.result(jpegBytes.toArray());
+        jpegBytes.release();
+
+        ctx.status(200);
+    }
+
+    public static void onCalibrationExportRequest(Context ctx) {
+        logger.info(ctx.queryString().toString());
+
+        int idx = Integer.parseInt(ctx.queryParam("cameraIdx"));
+        var width = Integer.parseInt(ctx.queryParam("width"));
+        var height = Integer.parseInt(ctx.queryParam("height"));
+
+        var cc = VisionModuleManager.getInstance().getModule(idx).getStateAsCameraConfig();
+
+        CameraCalibrationCoefficients calList =
+                cc.calibrations.stream()
+                        .filter(
+                                it ->
+                                        Math.abs(it.resolution.width - width) < 1e-4
+                                                && Math.abs(it.resolution.height - height) < 1e-4)
+                        .findFirst()
+                        .orElse(null);
+
+        if (calList == null) {
+            ctx.status(404);
+            return;
+        }
+
+        var filename = "photon_calibration_" + cc.uniqueName + "_" + width + "x" + height + ".json";
+        ctx.contentType("application/zip");
+        ctx.header("Content-Disposition", "attachment; filename=\"" + filename + "\"");
+        ctx.json(calList);
+
+        ctx.status(200);
+    }
+
     public static void onImageSnapshotsRequest(Context ctx) {
         var snapshots = new ArrayList<HashMap<String, Object>>();
         var cameraDirs = ConfigManager.getInstance().getImageSavePath().toFile().listFiles();

photon-server/src/main/java/org/photonvision/server/Server.java

@@ -130,6 +130,8 @@ public class Server {
         app.post("/api/utils/restartDevice", RequestHandler::onDeviceRestartRequest);
         app.post("/api/utils/publishMetrics", RequestHandler::onMetricsPublishRequest);
         app.get("/api/utils/getImageSnapshots", RequestHandler::onImageSnapshotsRequest);
+        app.get("/api/utils/getCalSnapshot", RequestHandler::onCalibrationSnapshotRequest);
+        app.get("/api/utils/getCalibrationJSON", RequestHandler::onCalibrationExportRequest);
 
         // Calibration
         app.post("/api/calibration/end", RequestHandler::onCalibrationEndRequest);
@@ -138,7 +140,6 @@ public class Server {
         app.post("/api/calibration/importFromData", RequestHandler::onDataCalibrationImportRequest);
 
         app.start(port);
-        System.out.println("hi");
     }
 
     /**

photonlib-cpp-examples/aimattarget/src/main/include/Robot.h

@@ -39,8 +39,8 @@ class Robot : public frc::TimedRobot {
   void TeleopPeriodic() override;
 
  private:
-  // Change this to match the name of your camera
-  photon::PhotonCamera camera{"photonvision"};
+  // Change this to match the name of your camera as shown in the web UI
+  photon::PhotonCamera camera{"YOUR_CAMERA_NAME_HERE"};
   // PID constants should be tuned per robot
   frc::PIDController controller{.1, 0, 0};
 

photonlib-cpp-examples/build.gradle

@@ -6,7 +6,7 @@ allprojects {
     repositories {
         mavenCentral()
         mavenLocal()
-        maven { url = "https://maven.photonvision.org/repository/internal/" }
+        maven { url = "https://maven.photonvision.org/releases" }
     }
 }
 

photonlib-cpp-examples/swervedriveposeestsim/src/main/include/Constants.h

@@ -35,48 +35,48 @@
 
 namespace constants {
 namespace Vision {
-static constexpr std::string_view kCameraName{"YOUR CAMERA NAME"};
-static const frc::Transform3d kRobotToCam{
+inline constexpr std::string_view kCameraName{"YOUR CAMERA NAME"};
+inline const frc::Transform3d kRobotToCam{
     frc::Translation3d{0.5_m, 0.0_m, 0.5_m},
     frc::Rotation3d{0_rad, 0_rad, 0_rad}};
-static const frc::AprilTagFieldLayout kTagLayout{
-    frc::LoadAprilTagLayoutField(frc::AprilTagField::k2023ChargedUp)};
+inline const frc::AprilTagFieldLayout kTagLayout{
+    frc::LoadAprilTagLayoutField(frc::AprilTagField::k2024Crescendo)};
 
-static const Eigen::Matrix<double, 3, 1> kSingleTagStdDevs{4, 4, 8};
-static const Eigen::Matrix<double, 3, 1> kMultiTagStdDevs{0.5, 0.5, 1};
+inline const Eigen::Matrix<double, 3, 1> kSingleTagStdDevs{4, 4, 8};
+inline const Eigen::Matrix<double, 3, 1> kMultiTagStdDevs{0.5, 0.5, 1};
 }  // namespace Vision
 namespace Swerve {
 
-static constexpr units::meter_t kTrackWidth{18.5_in};
-static constexpr units::meter_t kTrackLength{18.5_in};
-static constexpr units::meter_t kRobotWidth{25_in + 3.25_in * 2};
-static constexpr units::meter_t kRobotLength{25_in + 3.25_in * 2};
-static constexpr units::meters_per_second_t kMaxLinearSpeed{15.5_fps};
-static constexpr units::radians_per_second_t kMaxAngularSpeed{720_deg_per_s};
-static constexpr units::meter_t kWheelDiameter{4_in};
-static constexpr units::meter_t kWheelCircumference{kWheelDiameter *
+inline constexpr units::meter_t kTrackWidth{18.5_in};
+inline constexpr units::meter_t kTrackLength{18.5_in};
+inline constexpr units::meter_t kRobotWidth{25_in + 3.25_in * 2};
+inline constexpr units::meter_t kRobotLength{25_in + 3.25_in * 2};
+inline constexpr units::meters_per_second_t kMaxLinearSpeed{15.5_fps};
+inline constexpr units::radians_per_second_t kMaxAngularSpeed{720_deg_per_s};
+inline constexpr units::meter_t kWheelDiameter{4_in};
+inline constexpr units::meter_t kWheelCircumference{kWheelDiameter *
                                                     std::numbers::pi};
 
-static constexpr double kDriveGearRatio = 6.75;
-static constexpr double kSteerGearRatio = 12.8;
+inline constexpr double kDriveGearRatio = 6.75;
+inline constexpr double kSteerGearRatio = 12.8;
 
-static constexpr units::meter_t kDriveDistPerPulse =
+inline constexpr units::meter_t kDriveDistPerPulse =
     kWheelCircumference / 1024.0 / kDriveGearRatio;
-static constexpr units::radian_t kSteerRadPerPulse =
+inline constexpr units::radian_t kSteerRadPerPulse =
     units::radian_t{2 * std::numbers::pi} / 1024.0;
 
-static constexpr double kDriveKP = 1.0;
-static constexpr double kDriveKI = 0.0;
-static constexpr double kDriveKD = 0.0;
+inline constexpr double kDriveKP = 1.0;
+inline constexpr double kDriveKI = 0.0;
+inline constexpr double kDriveKD = 0.0;
 
-static constexpr double kSteerKP = 20.0;
-static constexpr double kSteerKI = 0.0;
-static constexpr double kSteerKD = 0.25;
+inline constexpr double kSteerKP = 20.0;
+inline constexpr double kSteerKI = 0.0;
+inline constexpr double kSteerKD = 0.25;
 
-static const frc::SimpleMotorFeedforward<units::meters> kDriveFF{
+inline const frc::SimpleMotorFeedforward<units::meters> kDriveFF{
     0.25_V, 2.5_V / 1_mps, 0.3_V / 1_mps_sq};
 
-static const frc::SimpleMotorFeedforward<units::radians> kSteerFF{
+inline const frc::SimpleMotorFeedforward<units::radians> kSteerFF{
     0.5_V, 0.25_V / 1_rad_per_s, 0.01_V / 1_rad_per_s_sq};
 
 struct ModuleConstants {
@@ -106,13 +106,13 @@ struct ModuleConstants {
         centerOffset(frc::Translation2d{xOffset, yOffset}) {}
 };
 
-static const ModuleConstants FL_CONSTANTS{
+inline const ModuleConstants FL_CONSTANTS{
     1, 0, 0, 1, 1, 2, 3, 0, kTrackLength / 2, kTrackWidth / 2};
-static const ModuleConstants FR_CONSTANTS{
+inline const ModuleConstants FR_CONSTANTS{
     2, 2, 4, 5, 3, 6, 7, 0, kTrackLength / 2, -kTrackWidth / 2};
-static const ModuleConstants BL_CONSTANTS{
+inline const ModuleConstants BL_CONSTANTS{
     3, 4, 8, 9, 5, 10, 11, 0, -kTrackLength / 2, kTrackWidth / 2};
-static const ModuleConstants BR_CONSTANTS{
+inline const ModuleConstants BR_CONSTANTS{
     4, 6, 12, 13, 7, 14, 15, 0, -kTrackLength / 2, -kTrackWidth / 2};
 }  // namespace Swerve
 }  // namespace constants

photonlib-cpp-examples/swervedriveposeestsim/src/main/include/Vision.h

@@ -61,7 +61,7 @@ class Vision {
       cameraSim = std::make_shared<photon::PhotonCameraSim>(camera.get(),
                                                             *cameraProp.get());
 
-      visionSim->AddCamera(cameraSim.get(), robotToCam);
+      visionSim->AddCamera(cameraSim.get(), constants::Vision::kRobotToCam);
       cameraSim->EnableDrawWireframe(true);
     }
   }
@@ -138,12 +138,10 @@ class Vision {
   frc::Field2d& GetSimDebugField() { return visionSim->GetDebugField(); }
 
  private:
-  frc::Transform3d robotToCam{frc::Translation3d{0.5_m, 0.5_m, 0.5_m},
-                              frc::Rotation3d{}};
   photon::PhotonPoseEstimator photonEstimator{
-      LoadAprilTagLayoutField(frc::AprilTagField::k2023ChargedUp),
+      constants::Vision::kTagLayout,
       photon::PoseStrategy::MULTI_TAG_PNP_ON_COPROCESSOR,
-      photon::PhotonCamera{"photonvision"}, robotToCam};
+      photon::PhotonCamera{"photonvision"}, constants::Vision::kRobotToCam};
   std::shared_ptr<photon::PhotonCamera> camera{photonEstimator.GetCamera()};
   std::unique_ptr<photon::VisionSystemSim> visionSim;
   std::unique_ptr<photon::SimCameraProperties> cameraProp;

photonlib-java-examples/aimattarget/src/main/java/frc/robot/Robot.java

@@ -48,8 +48,8 @@ public class Robot extends TimedRobot {
     // How far from the target we want to be
     final double GOAL_RANGE_METERS = Units.feetToMeters(3);
 
-    // Change this to match the name of your camera
-    PhotonCamera camera = new PhotonCamera("photonvision");
+    // Change this to match the name of your camera as shown in the web UI
+    PhotonCamera camera = new PhotonCamera("YOUR_CAMERA_NAME_HERE");
 
     // PID constants should be tuned per robot
     final double LINEAR_P = 0.1;

photonlib-java-examples/build.gradle

@@ -8,7 +8,7 @@ allprojects {
     repositories {
         mavenCentral()
         mavenLocal()
-        maven { url = "https://maven.photonvision.org/repository/internal/" }
+        maven { url = "https://maven.photonvision.org/releases" }
     }
 }
 

scripts/install.sh

@@ -132,6 +132,10 @@ fi
 echo "Installing additional math packages"
 apt-get install --yes libcholmod3 liblapack3 libsuitesparseconfig5
 
+echo "Installing v4l-utils..."
+apt-get install --yes v4l-utils
+echo "v4l-utils installation complete."
+
 echo "Downloading latest stable release of PhotonVision..."
 mkdir -p /opt/photonvision
 cd /opt/photonvision

shared/javacommon.gradle

@@ -79,7 +79,7 @@ publishing {
 
     repositories {
         maven {
-            url ('https://maven.photonvision.org/repository/' + (isDev ? 'snapshots' : 'internal'))
+            url(photonMavenURL)
             credentials {
                 username 'ghactions'
                 password System.getenv("ARTIFACTORY_API_KEY")

shared/javacpp/publish.gradle

@@ -70,7 +70,7 @@ model {
 
         repositories {
             maven {
-                url ('https://maven.photonvision.org/repository/' + (isDev ? 'snapshots' : 'internal'))
+                url(photonMavenURL)
                 credentials {
                     username 'ghactions'
                     password System.getenv("ARTIFACTORY_API_KEY")