Update SimPhotonCamera.h (#785)

* Update libphotonlibcamera.so

* Bump Pi base image URL

.github/workflows/main.yml

@@ -22,26 +22,22 @@ jobs:
         working-directory: photon-client
 
     # The type of runner that the job will run on.
-    runs-on: ubuntu-latest
-
-    # Grab the docker container.
-    container:
-      image: docker://node:10
+    runs-on: ubuntu-22.04
 
     steps:
     # Checkout code.
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
 
     # Setup Node.js
     - name: Setup Node.js
-      uses: actions/setup-node@v1
+      uses: actions/setup-node@v3
       with:
-        node-version: 10
+        node-version: 16
 
     # Run npm
-    - run: |
-        npm ci
-        npm run build --if-present
+    - run: npm update -g npm
+    - run: npm ci
+    - run: npm run build --if-present
 
     # Upload client artifact.
     - uses: actions/upload-artifact@master
@@ -49,34 +45,80 @@ jobs:
         name: built-client
         path: photon-client/dist/
 
-  photon-build-all:
-    # The type of runner that the job will run on.
-    runs-on: ubuntu-latest
+  photon-build-examples:
+    runs-on: ubuntu-22.04
+    name: "Build Examples"
 
     steps:
       # Checkout code.
       - name: Checkout code
-        uses: actions/checkout@v1
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
 
       # Fetch tags.
       - name: Fetch tags
         run: git fetch --tags --force
 
-      # Install Java 11.
-      - name: Install Java 11
-        uses: actions/setup-java@v1
+      # Install Java 17.
+      - name: Install Java 17
+        uses: actions/setup-java@v3
         with:
-          java-version: 11
+          java-version: 17
+          distribution: temurin
 
-      # Run Gradle build.
+      # Need to publish to maven local first, so that C++ sim can pick it up
+      # Still haven't figure out how to make the vendordep file be copied before trying to build examples
+      - name: Publish photonlib to maven local
+        run: |
+          chmod +x gradlew
+          ./gradlew publishtomavenlocal -x check
+
+      - name: Build Java examples
+        working-directory: photonlib-java-examples
+        run: |
+          chmod +x gradlew
+          ./gradlew copyPhotonlib -x check
+          ./gradlew buildAllExamples -x check --max-workers 2
+
+      - name: Build C++ examples
+        working-directory: photonlib-cpp-examples
+        run: |
+          chmod +x gradlew
+          ./gradlew copyPhotonlib -x check
+          ./gradlew buildAllExamples -x check --max-workers 2
+
+  photon-build-all:
+    # The type of runner that the job will run on.
+    runs-on: ubuntu-22.04
+
+    steps:
+      # Checkout code.
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      # Fetch tags.
+      - name: Fetch tags
+        run: git fetch --tags --force
+
+      # Install Java 17.
+      - name: Install Java 17
+        uses: actions/setup-java@v3
+        with:
+          java-version: 17
+          distribution: temurin
+
+      # Run only build tasks, no checks??
       - name: Gradle Build
         run: |
           chmod +x gradlew
-          ./gradlew build -x check --max-workers 1
+          ./gradlew photon-server:build photon-lib:build -x check --max-workers 2
 
       # Run Gradle Tests.
       - name: Gradle Tests
-        run: ./gradlew testHeadless -i --max-workers 1
+        run: ./gradlew testHeadless -i --max-workers 1 --stacktrace
 
       # Generate Coverage Report.
       - name: Gradle Coverage
@@ -84,29 +126,29 @@ jobs:
 
       # Publish Coverage Report.
       - name: Publish Server Coverage Report
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v3
         with:
           file: ./photon-server/build/reports/jacoco/test/jacocoTestReport.xml
 
       - name: Publish Core Coverage Report
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v3
         with:
           file: ./photon-core/build/reports/jacoco/test/jacocoTestReport.xml
 
   photonserver-build-offline-docs:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     steps:
       # Checkout docs.
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           repository: 'PhotonVision/photonvision-docs.git'
           ref: master
 
       # Install Python.
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
-          python-version: '3.6'
+          python-version: '3.9'
 
       - name: Install dependencies
         run: |
@@ -114,10 +156,11 @@ jobs:
           pip install sphinx sphinx_rtd_theme sphinx-tabs sphinxext-opengraph doc8
           pip install -r requirements.txt
 
-      - name: Check the docs
-        run: |
-          make linkcheck
-          make lint
+      # Don't check the docs. If a PR was merged to the docs repo, it ought to pass CI. No need to re-check here.
+      # - name: Check the docs
+      #   run: |
+      #     make linkcheck
+      #     make lint
 
       - name: Build the docs
         run: |
@@ -131,24 +174,25 @@ jobs:
 
   photonserver-check-lint:
     # The type of runner that the job will run on.
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     steps:
       # Checkout code.
-      - uses: actions/checkout@v1
-
-      # Install Java 11.
-      - uses: actions/setup-java@v1
+      - uses: actions/checkout@v3
         with:
-          java-version: 11
+          fetch-depth: 0
+
+      # Install Java 17.
+      - uses: actions/setup-java@v3
+        with:
+          java-version: 17
+          distribution: temurin
 
       # Check server code with Spotless.
       - run: |
           chmod +x gradlew
           ./gradlew spotlessCheck
 
-
-
   #  Building photonlib
   photonlib-build-host:
     env:
@@ -157,22 +201,23 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - os: windows-latest
+          - os: windows-2022
             artifact-name: Win64
-          - os: macos-latest
+          - os: macos-11
             artifact-name: macOS
-          - os: ubuntu-latest
+          - os: ubuntu-22.04
             artifact-name: Linux
 
     runs-on: ${{ matrix.os }}
-    name: "Photonlib - Build - ${{ matrix.artifact-name }}"
+    name: "Photonlib - Build Host - ${{ matrix.artifact-name }}"
     steps:
-      - uses: actions/checkout@v2.3.4
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - uses: actions/setup-java@v1
+      - uses: actions/setup-java@v3
         with:
-          java-version: 11
+          java-version: 17
+          distribution: temurin
       - run: git fetch --tags --force
       - run: |
           chmod +x gradlew
@@ -188,27 +233,29 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - container: wpilib/roborio-cross-ubuntu:2022-18.04
+          - container: wpilib/roborio-cross-ubuntu:2023-22.04
             artifact-name: Athena
-          - container: wpilib/raspbian-cross-ubuntu:10-18.04
+          - container: wpilib/raspbian-cross-ubuntu:bullseye-22.04
             artifact-name: Raspbian
-          - container: wpilib/aarch64-cross-ubuntu:bionic-18.04
+          - container: wpilib/aarch64-cross-ubuntu:bullseye-22.04
             artifact-name: Aarch64
 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     container: ${{ matrix.container }}
-    name: "Photonlib - Build - ${{ matrix.artifact-name }}"
+    name: "Photonlib - Build Docker - ${{ matrix.artifact-name }}"
     steps:
-      - uses: actions/checkout@v2.3.4
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - uses: actions/setup-java@v1
-        with:
-          java-version: 11
-      - run: |
+      - name: Config Git
+        run: |
+          git config --global --add safe.directory /__w/photonvision/photonvision
+      - name: Build PhotonLib
+        run: |
           chmod +x gradlew
           ./gradlew photon-lib:build --max-workers 1
-      - run: |
+      - name: Publish
+        run: |
           chmod +x gradlew
           ./gradlew photon-lib:publish
         env:
@@ -217,16 +264,16 @@ jobs:
 
   photonlib-wpiformat:
     name: "wpiformat"
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Fetch all history and metadata
         run: |
           git fetch --prune --unshallow
           git checkout -b pr
           git branch -f master origin/master
       - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: 3.8
       - name: Install clang-format
@@ -243,40 +290,78 @@ jobs:
       - name: Generate diff
         run: git diff HEAD > wpiformat-fixes.patch
         if: ${{ failure() }}
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: wpiformat fixes
           path: wpiformat-fixes.patch
         if: ${{ failure() }}
 
   photon-build-package:
-    needs: [photonclient-build, photon-build-all, photonserver-build-offline-docs, photonlib-build-host, photonlib-build-docker]
+    needs: [photonclient-build, photon-build-all, photonserver-build-offline-docs]
+
+    strategy:
+        fail-fast: false
+        matrix:
+          include:
+            - os: windows-latest
+              artifact-name: Win64
+              architecture: x64
+              arch-override: none
+            - os: macos-latest
+              artifact-name: macOS
+              architecture: x64
+              arch-override: none
+            - os: ubuntu-latest
+              artifact-name: Linux
+              architecture: x64
+              arch-override: none
+            - os: macos-latest
+              artifact-name: macOSArm
+              architecture: x64
+              arch-override: macarm64
+            - os: ubuntu-latest
+              artifact-name: LinuxArm32
+              architecture: x64
+              arch-override: linuxarm32
+            - os: ubuntu-latest
+              artifact-name: LinuxArm64
+              architecture: x64
+              arch-override: linuxarm64
 
     # The type of runner that the job will run on.
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    name: "Build fat JAR - ${{ matrix.artifact-name }}"
 
     steps:
       # Checkout code.
-      - uses: actions/checkout@v1
-
-      # Install Java 11.
-      - uses: actions/setup-java@v1
+      - uses: actions/checkout@v3
         with:
-          java-version: 11
+          fetch-depth: 0
+
+      # Install Java 17.
+      - uses: actions/setup-java@v3
+        with:
+          java-version: 17
+          distribution: temurin
 
       # Clear any existing web resources.
       - run: |
           rm -rf photon-server/src/main/resources/web/*
           mkdir -p photon-server/src/main/resources/web/docs
+        if: ${{ (matrix.os) != 'windows-latest' }}
+      - run: |
+          del photon-server\src\main\resources\web\*.*
+          mkdir photon-server\src\main\resources\web\docs
+        if: ${{ (matrix.os) == 'windows-latest' }}
 
       # Download client artifact to resources folder.
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v3
         with:
           name: built-client
           path: photon-server/src/main/resources/web/
 
       # Download docs artifact to resources folder.
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v3
         with:
           name: built-docs
           path: photon-server/src/main/resources/web/docs
@@ -284,54 +369,89 @@ jobs:
       # Build fat jar for both pi and everything
       - run: |
           chmod +x gradlew
-          ./gradlew photon-server:shadowJar --max-workers 1
-          ./gradlew photon-server:shadowJar --max-workers 1 -Ppionly
-
-      # The image will only pull the Pi JAR in
-      - name: Generate image
-        if: github.event_name != 'pull_request'
-        run: |
-          chmod +x scripts/generatePiImage.sh
-          ./scripts/generatePiImage.sh
+          ./gradlew photon-server:shadowJar --max-workers 2 -PArchOverride=${{ matrix.arch-override }}
+        if: ${{ (matrix.arch-override != 'none') }}
+      - run: |
+          chmod +x gradlew
+          ./gradlew photon-server:shadowJar --max-workers 2
+        if: ${{ (matrix.arch-override == 'none') }}
 
       # Upload final fat jar as artifact.
-      - uses: actions/upload-artifact@master
+      - uses: actions/upload-artifact@v3
         with:
-          name: jar
+          name: jar-${{ matrix.artifact-name }}
           path: photon-server/build/libs
-      - uses: actions/upload-artifact@master
-        with:
-          name: image
-          path: photonvision*.zip
 
+
+  photon-image-generator:
+    needs: [photon-build-package]
+    if: ${{ github.event_name != 'pull_request' }}
+    strategy:
+        fail-fast: false
+        matrix:
+          include:
+            - os: ubuntu-latest
+              artifact-name: LinuxArm64
+              image_suffix: RaspberryPi
+              image_url: https://api.github.com/repos/photonvision/photon-pi-gen/releases/tags/v2023.1.3_arm64
+            - os: ubuntu-latest
+              artifact-name: LinuxArm64
+              image_suffix: limelight2
+              image_url: https://api.github.com/repos/photonvision/photon-pi-gen/releases/tags/v2023.2.2_limelight-arm64
+
+    runs-on: ${{ matrix.os }}
+    name: "Build image - ${{ matrix.image_url }}"
+
+    steps:
+      # Checkout code.
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: actions/download-artifact@v2
+        with:
+          name: jar-${{ matrix.artifact-name }}
+
+      - name: Generate image
+        run: |
+          chmod +x scripts/generatePiImage.sh
+          ./scripts/generatePiImage.sh ${{ matrix.image_url }} ${{ matrix.image_suffix }}
+
+      - uses: actions/upload-artifact@v3
+        name: Upload image
+        with:
+          name: image-${{ matrix.image_suffix }}
+          path: photonvision*.xz
+
+
+  photon-release:
+    needs: [photon-build-package, photon-image-generator]
+    runs-on: ubuntu-22.04
+    steps:
+      # Download literally every single artifact. This also downloads client and docs,
+      # but the filtering below won't pick these up (I hope)
+      - uses: actions/download-artifact@v2
+
+      - run: find
+
+      # Push to dev release
       - uses: pyTooling/Actions/releaser@r0
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           tag: 'Dev'
           rm: true
           files: |
-            photon-server/build/libs/*.jar
-            photonvision*.zip
+            **/*.xz
+            **/*.jar
         if: github.event_name == 'push'
 
-  photon-release:
-    if: startsWith(github.ref, 'refs/tags/v')
-    needs: [photon-build-package]
-    runs-on: ubuntu-latest
-    steps:
-      # This *should* pull in fat and pi-only jars
-      - uses: actions/download-artifact@v2
-        with:
-          name: jar
-
-      # And the image we made previously
-      - uses: actions/download-artifact@v2
-        with:
-          name: image
-
-      # All we've downloaded (ideally) is the fat jar, pi jar, and image. So just upload it all
+      # Upload all jars and xz archives
       - uses: softprops/action-gh-release@v1
         with:
-          files: '**/*'
+          files: |
+            **/*.xz
+            **/*.jar
+        if: startsWith(github.ref, 'refs/tags/v')
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -30,6 +30,7 @@ backend/settings/
 *.nar
 *.ear
 *.zip
+*.xz
 *.tar.gz
 *.rar
 
@@ -144,3 +145,11 @@ build
 photon-lib/src/main/java/org/photonvision/PhotonVersion.java
 /photonlib-java-examples/bin/
 photon-lib/src/generate/native/include/PhotonVersion.h
+.gitattributes
+lib/*
+photon-server/lib/libapriltag.so
+photon-server/bin/main/nativelibraries/apriltag/*
+photon-server/src/main/resources/nativelibraries/apriltag/*
+
+photonlib-java-examples/*/vendordeps/*
+photonlib-cpp-examples/*/vendordeps/*

.styleguide

@@ -11,8 +11,11 @@ cppSrcFileInclude {
 
 modifiableFileExclude {
   \.jpg$
+  \.jpeg$
   \.png$
+  \.gif$
   \.so$
+  \.dll$
 }
 
 includeProject {

LICENSE_MathUtils_orthogonalizeRotationMatrix.txt

@@ -0,0 +1,23 @@
+Copyright (c) 2022 Photon Vision. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+   * Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+   * Neither the name of FIRST, WPILib, nor the names of other WPILib
+     contributors may be used to endorse or promote products derived from
+     this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY FIRST AND OTHER WPILIB CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY NONINFRINGEMENT AND FITNESS FOR A PARTICULAR
+PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL FIRST OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -6,7 +6,7 @@ PhotonVision is the free, fast, and easy-to-use computer vision solution for the
 
 A copy of the latest Raspberry Pi image is available [here](https://github.com/PhotonVision/photon-pi-gen/releases). A copy of the latest standalone JAR is available [here](https://github.com/PhotonVision/photonvision/releases). If you are a Gloworm user you can find the latest Gloworm image [here](https://github.com/gloworm-vision/pi-gen/releases).
 
-If you are interested in contributing code or documentation to the project, please [read our getting started page for contributors](https://docs.photonvision.org/en/latest/docs/other/contributing/index.html) and **[join the Discord](https://discord.gg/wYxTwym) to introduce yourself!** We hope to provide a welcoming community to anyone who is interested in helping.
+If you are interested in contributing code or documentation to the project, please [read our getting started page for contributors](https://docs.photonvision.org/en/latest/docs/contributing/index.html) and **[join the Discord](https://discord.gg/wYxTwym) to introduce yourself!** We hope to provide a welcoming community to anyone who is interested in helping.
 
 ## Authors
 
@@ -18,10 +18,42 @@ If you are interested in contributing code or documentation to the project, plea
 
 Note that these are case sensitive!
 
-* `-Ppionly`: only builds for `linuxraspbian`, which reduces JAR size. The JAR name will have "-raspi" appended.
+* `-PArchOverride=foobar`: builds for a target system other than your current architecture. Valid overrides are winx32, winx64,
+macx64, macarm64, linuxx64, linuxarm64, linuxarm32, and linuxathena.
 - `-PtgtIp`: deploys (builds and copies the JAR) to the coprocessor at the specified IP
 - `-Pprofile`: enables JVM profiling
 
+## Building
+
+Gradle is used for all C++ and Java code, and NPM is used for the web UI. Instructions to compile PhotonVision yourself can be found [in our docs](https://docs.photonvision.org/en/latest/docs/contributing/photonvision/build-instructions.html?highlight=npm%20install#compiling-instructions).
+
+You can run one of the many built in examples straight from the command line, too! They contain a fully featured robot project, and some include simulation support. The projects can be found inside the `photonlib-java-examples` and `photonlib-cpp-examples` subdirectories, respectively. The projects currently available include:
+
+- photonlib-java-examples:
+  - aimandrange:simulateJava
+  - aimattarget:simulateJava
+  - getinrange:simulateJava
+  - simaimandrange:simulateJava
+  - simposeest:simulateJava
+- photonlib-cpp-examples:
+  - aimandrange:simulateNative
+  - getinrange:simulateNative
+
+To run them, use the commands listed below. Photonlib must first be published to your local maven repository, then the `copyPhotonlib` task will copy the generated vendordep json file into each example. After that, the simulateJava/simulateNative task can be used like a normal robot project. Robot simulation with attached debugger is technically possible by using simulateExternalJava and modifying the launch script it exports, though unsupported.
+
+```
+~/photonvision$ ./gradlew publishToMavenLocal
+
+~/photonvision$ cd photonlib-java-examples
+~/photonvision/photonlib-java-examples$ ./gradlew copyPhotonlib
+~/photonvision/photonlib-java-examples$ ./gradlew <example-name>:simulateJava
+
+~/photonvision$ cd photonlib-cpp-examples
+~/photonvision/photonlib-cpp-examples$ ./gradlew copyPhotonlib
+~/photonvision/photonlib-cpp-examples$ ./gradlew <example-name>:simulateNative
+```
+
+
 ## Acknowledgments
 PhotonVision was forked from [Chameleon Vision](https://github.com/Chameleon-Vision/chameleon-vision/). Thank you to everyone who worked on the original project.
 

build.gradle

@@ -4,14 +4,18 @@ plugins {
     id "com.github.node-gradle.node" version "3.1.1" apply false
     id "edu.wpi.first.GradleJni" version "1.0.0"
     id "edu.wpi.first.GradleVsCode" version "1.1.0"
-    id "edu.wpi.first.NativeUtils" version "2022.8.1" apply false
+    id "edu.wpi.first.NativeUtils" version "2023.11.1" apply false
     id "edu.wpi.first.wpilib.repositories.WPILibRepositoriesPlugin" version "2020.2"
     id "org.hidetake.ssh" version "2.10.1"
+    id 'edu.wpi.first.WpilibTools' version '1.0.0'
 }
 
+import org.gradle.api.internal.artifacts.dependencies.DefaultExternalModuleDependency;
+
 allprojects {
     repositories {
-        jcenter()
+        mavenCentral()
+        mavenLocal()
         maven { url = "https://maven.photonvision.org/repository/internal/" }
     }
     wpilibRepositories.addAllReleaseRepositories(it)
@@ -22,19 +26,24 @@ allprojects {
 apply from: "versioningHelper.gradle"
 
 ext {
-    wpilibVersion = "2022.1.1"
-    opencvVersion = "4.5.2-1"
+    wpilibVersion = "2023.2.1"
+    opencvVersion = "4.6.0-4"
     joglVersion = "2.4.0-rc-20200307"
     pubVersion = versionString
     isDev = pubVersion.startsWith("dev")
 
-
-    jniPlatforms = project.hasProperty('pionly') ? ['linuxraspbian']
-        : ['linuxaarch64bionic', 'linuxraspbian', 'linuxx86-64', 'osxx86-64', 'windowsx86-64']
-
-    println("Building for archs " + jniPlatforms)
+    // A list, for legacy reasons, with only the current platform contained
+    String nativeName = wpilibTools.platformMapper.currentPlatform.platformName;
+    if (nativeName == "linuxx64") nativeName = "linuxx86-64";
+    if (nativeName == "winx64") nativeName = "windowsx86-64";
+    if (nativeName == "macx64") nativeName = "osxx86-64";
+    if (nativeName == "macarm64") nativeName = "osxarm64";
+    jniPlatform = nativeName
+    println("Building for platform " + jniPlatform)
 }
 
+wpilibTools.deps.wpilibVersion = wpilibVersion
+
 spotless {
     java {
         toggleOffOn()

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

gradlew

@@ -1,7 +1,7 @@
-#!/usr/bin/env sh
+#!/bin/sh
 
 #
-# Copyright 2015 the original author or authors.
+# Copyright <20> 2015-2021 the original authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,68 +16,58 @@
 # limitations under the License.
 #
 
-##############################################################################
-##
-##  Gradle start up script for UN*X
-##
 ##############################################################################
 
 # Attempt to set APP_HOME
+
 # Resolve links: $0 may be a link
-PRG="$0"
-# Need this for relative symlinks.
-while [ -h "$PRG" ] ; do
-    ls=`ls -ld "$PRG"`
-    link=`expr "$ls" : '.*-> \(.*\)$'`
-    if expr "$link" : '/.*' > /dev/null; then
-        PRG="$link"
-    else
-        PRG=`dirname "$PRG"`"/$link"
-    fi
+app_path=$0
+
+# Need this for daisy-chained symlinks.
+while
+    APP_HOME=${app_path%"${app_path##*/}"}  # leaves a trailing /; empty if no leading path
+    [ -h "$app_path" ]
+do
+    ls=$( ls -ld "$app_path" )
+    link=${ls#*' -> '}
+    case $link in             #(
+      /*)   app_path=$link ;; #(
+      *)    app_path=$APP_HOME$link ;;
+    esac
 done
-SAVED="`pwd`"
-cd "`dirname \"$PRG\"`/" >/dev/null
-APP_HOME="`pwd -P`"
-cd "$SAVED" >/dev/null
+
+APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
 
 APP_NAME="Gradle"
-APP_BASE_NAME=`basename "$0"`
+APP_BASE_NAME=${0##*/}
 
 # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
 DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
-MAX_FD="maximum"
+MAX_FD=maximum
 
 warn () {
     echo "$*"
-}
+} >&2
 
 die () {
     echo
     echo "$*"
     echo
     exit 1
-}
+} >&2
 
 # OS specific support (must be 'true' or 'false').
 cygwin=false
 msys=false
 darwin=false
 nonstop=false
-case "`uname`" in
-  CYGWIN* )
-    cygwin=true
-    ;;
-  Darwin* )
-    darwin=true
-    ;;
-  MINGW* )
-    msys=true
-    ;;
-  NONSTOP* )
-    nonstop=true
-    ;;
+case "$( uname )" in                #(
+  CYGWIN* )         cygwin=true  ;; #(
+  Darwin* )         darwin=true  ;; #(
+  MSYS* | MINGW* )  msys=true    ;; #(
+  NONSTOP* )        nonstop=true ;;
 esac
 
 CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
@@ -87,9 +77,9 @@ CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
 if [ -n "$JAVA_HOME" ] ; then
     if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
         # IBM's JDK on AIX uses strange locations for the executables
-        JAVACMD="$JAVA_HOME/jre/sh/java"
+        JAVACMD=$JAVA_HOME/jre/sh/java
     else
-        JAVACMD="$JAVA_HOME/bin/java"
+        JAVACMD=$JAVA_HOME/bin/java
     fi
     if [ ! -x "$JAVACMD" ] ; then
         die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
@@ -98,7 +88,7 @@ Please set the JAVA_HOME variable in your environment to match the
 location of your Java installation."
     fi
 else
-    JAVACMD="java"
+    JAVACMD=java
     which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
 
 Please set the JAVA_HOME variable in your environment to match the
@@ -106,80 +96,95 @@ location of your Java installation."
 fi
 
 # Increase the maximum file descriptors if we can.
-if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
-    MAX_FD_LIMIT=`ulimit -H -n`
-    if [ $? -eq 0 ] ; then
-        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
-            MAX_FD="$MAX_FD_LIMIT"
-        fi
-        ulimit -n $MAX_FD
-        if [ $? -ne 0 ] ; then
-            warn "Could not set maximum file descriptor limit: $MAX_FD"
-        fi
-    else
-        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
-    fi
-fi
-
-# For Darwin, add options to specify how the application appears in the dock
-if $darwin; then
-    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
-fi
-
-# For Cygwin or MSYS, switch paths to Windows format before running java
-if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
-    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
-    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
-
-    JAVACMD=`cygpath --unix "$JAVACMD"`
-
-    # We build the pattern for arguments to be converted via cygpath
-    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
-    SEP=""
-    for dir in $ROOTDIRSRAW ; do
-        ROOTDIRS="$ROOTDIRS$SEP$dir"
-        SEP="|"
-    done
-    OURCYGPATTERN="(^($ROOTDIRS))"
-    # Add a user-defined pattern to the cygpath arguments
-    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
-        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
-    fi
-    # Now convert the arguments - kludge to limit ourselves to /bin/sh
-    i=0
-    for arg in "$@" ; do
-        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
-        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
-
-        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
-            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
-        else
-            eval `echo args$i`="\"$arg\""
-        fi
-        i=`expr $i + 1`
-    done
-    case $i in
-        0) set -- ;;
-        1) set -- "$args0" ;;
-        2) set -- "$args0" "$args1" ;;
-        3) set -- "$args0" "$args1" "$args2" ;;
-        4) set -- "$args0" "$args1" "$args2" "$args3" ;;
-        5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
-        6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
-        7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
-        8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
-        9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
+    case $MAX_FD in #(
+      max*)
+        MAX_FD=$( ulimit -H -n ) ||
+            warn "Could not query maximum file descriptor limit"
+    esac
+    case $MAX_FD in  #(
+      '' | soft) :;; #(
+      *)
+        ulimit -n "$MAX_FD" ||
+            warn "Could not set maximum file descriptor limit to $MAX_FD"
     esac
 fi
 
-# Escape application args
-save () {
-    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
-    echo " "
-}
-APP_ARGS=`save "$@"`
+# Collect all arguments for the java command, stacking in reverse order:
+#   * args from the command line
+#   * the main class name
+#   * -classpath
+#   * -D...appname settings
+#   * --module-path (only if needed)
+#   * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
 
-# Collect all arguments for the java command, following the shell quoting and substitution rules
-eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+# For Cygwin or MSYS, switch paths to Windows format before running java
+if "$cygwin" || "$msys" ; then
+    APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
+    CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
+
+    JAVACMD=$( cygpath --unix "$JAVACMD" )
+
+    # Now convert the arguments - kludge to limit ourselves to /bin/sh
+    for arg do
+        if
+            case $arg in                                #(
+              -*)   false ;;                            # don't mess with options #(
+              /?*)  t=${arg#/} t=/${t%%/*}              # looks like a POSIX filepath
+                    [ -e "$t" ] ;;                      #(
+              *)    false ;;
+            esac
+        then
+            arg=$( cygpath --path --ignore --mixed "$arg" )
+        fi
+        # Roll the args list around exactly as many times as the number of
+        # args, so each arg winds up back in the position where it started, but
+        # possibly modified.
+        #
+        # NB: a `for` loop captures its iteration list before it begins, so
+        # changing the positional parameters here affects neither the number of
+        # iterations, nor the values presented in `arg`.
+        shift                   # remove old arg
+        set -- "$@" "$arg"      # push replacement arg
+    done
+fi
+
+# Collect all arguments for the java command;
+#   * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
+#     shell script including quotes and variable substitutions, so put them in
+#     double quotes to make sure that they get re-expanded; and
+#   * put everything else in single quotes, so that it's not re-expanded.
+
+set -- \
+        "-Dorg.gradle.appname=$APP_BASE_NAME" \
+        -classpath "$CLASSPATH" \
+        org.gradle.wrapper.GradleWrapperMain \
+        "$@"
+
+# Use "xargs" to parse quoted args.
+#
+# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
+#
+# In Bash we could simply go:
+#
+#   readarray ARGS < <( xargs -n1 <<<"$var" ) &&
+#   set -- "${ARGS[@]}" "$@"
+#
+# but POSIX shell has neither arrays nor command substitution, so instead we
+# post-process each arg (as a line of input to sed) to backslash-escape any
+# character that might be a shell metacharacter, then use eval to reverse
+# that process (while maintaining the separation between arguments), and wrap
+# the whole thing up as a single "set" statement.
+#
+# This will of course break if any of these variables contains a newline or
+# an unmatched quote.
+#
+
+eval "set -- $(
+        printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
+        xargs -n1 |
+        sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
+        tr '\n' ' '
+    )" '"$@"'
 
 exec "$JAVACMD" "$@"

gradlew.bat

@@ -1,89 +1,89 @@
-@rem
-@rem Copyright 2015 the original author or authors.
-@rem
-@rem Licensed under the Apache License, Version 2.0 (the "License");
-@rem you may not use this file except in compliance with the License.
-@rem You may obtain a copy of the License at
-@rem
-@rem      https://www.apache.org/licenses/LICENSE-2.0
-@rem
-@rem Unless required by applicable law or agreed to in writing, software
-@rem distributed under the License is distributed on an "AS IS" BASIS,
-@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-@rem See the License for the specific language governing permissions and
-@rem limitations under the License.
-@rem
-
-@if "%DEBUG%" == "" @echo off
-@rem ##########################################################################
-@rem
-@rem  Gradle startup script for Windows
-@rem
-@rem ##########################################################################
-
-@rem Set local scope for the variables with windows NT shell
-if "%OS%"=="Windows_NT" setlocal
-
-set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
-set APP_BASE_NAME=%~n0
-set APP_HOME=%DIRNAME%
-
-@rem Resolve any "." and ".." in APP_HOME to make it shorter.
-for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
-
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
-
-@rem Find java.exe
-if defined JAVA_HOME goto findJavaFromJavaHome
-
-set JAVA_EXE=java.exe
-%JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto execute
-
-echo.
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:findJavaFromJavaHome
-set JAVA_HOME=%JAVA_HOME:"=%
-set JAVA_EXE=%JAVA_HOME%/bin/java.exe
-
-if exist "%JAVA_EXE%" goto execute
-
-echo.
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:execute
-@rem Setup the command line
-
-set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
-
-
-@rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
-
-:end
-@rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
-
-:fail
-rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
-rem the _cmd.exe /c_ return code!
-if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
-exit /b 1
-
-:mainEnd
-if "%OS%"=="Windows_NT" endlocal
-
-:omega
+@rem
+@rem Copyright 2015 the original author or authors.
+@rem
+@rem Licensed under the Apache License, Version 2.0 (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+@rem
+@rem      https://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+@rem
+
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Resolve any "." and ".." in APP_HOME to make it shorter.
+for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto execute
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto execute
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

photon-client/package.json

@@ -16,9 +16,10 @@
     "jspdf": "^2.4.0",
     "material-design-icons-iconfont": "^5.0.1",
     "msgpack5": "^4.2.1",
+    "three-full": "^28.0.2",
     "vue": "^2.6.12",
     "vue-axios": "^2.1.5",
-    "vue-native-websocket": "git+https://github.com/PhotonVision/vue-native-websocket.git#7a32791",
+    "vue-native-websocket": "git+https://git@github.com/PhotonVision/vue-native-websocket.git#5189f29",
     "vue-router": "^3.4.3",
     "vuetify": "^2.3.10",
     "vuex": "^3.5.1"

photon-client/public/thinclient.html

@@ -0,0 +1,317 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+    <title>ThinClient</title>
+
+    <style>
+        * {
+            margin: 0;
+            padding: 0;
+        }
+        .imgbox {
+            display: grid;
+            height: 100%;
+            width: 100%;
+        }
+        .center-fit {
+
+            width: 90vw;
+            margin: auto;
+        }
+    </style>
+
+</head>
+
+<body>
+    <hr>
+        <div class="imgbox">
+            <img id="streamImg" class="center-fit" src=''>
+        </div>
+    <hr>
+
+    <form id="frm1">
+        Host <input type="text" id="host" value="photonvision.local"><br>
+        Port <input type="text" id="port" value="1181"><br>
+    </form>
+
+    <button>Start Stream</button>
+
+    <script type="module">
+        class WebsocketVideoStream{
+
+            constructor(drawDiv, streamPort, host) {
+
+                this.drawDiv = drawDiv;
+                this.image = document.getElementById(this.drawDiv);
+                this.streamPort = streamPort;
+                this.newStreamPortReq = null;
+                this.serverAddr = "ws://" + host + "/websocket_cameras";
+                this.dispNoStream();
+                this.ws_connect();
+                this.imgData = null;
+                this.imgDataTime = -1;
+                this.imgObjURL = null;
+                this.frameRxCount = 0;
+
+                //Display state machine
+                this.DSM_DISCONNECTED = "DISCONNECTED";
+                this.DSM_WAIT_FOR_VALID_PORT = "WAIT_FOR_VALID_PORT";
+                this.DSM_SUBSCRIBE = "SUBSCRIBE";
+                this.DSM_WAIT_FOR_FIRST_FRAME = "WAIT_FOR_FIRST_FRAME";
+                this.DSM_SHOWING = "SHOWING";
+                this.DSM_RESTART_UNSUBSCRIBE = "UNSUBSCRIBE";
+                this.DSM_RESTART_WAIT = "WAIT_BEFORE_SUBSCRIBE";
+
+                this.dsm_cur_state = this.DSM_DISCONNECTED;
+                this.dsm_prev_state = this.DSM_DISCONNECTED;
+                this.dsm_restart_start_time = window.performance.now();
+
+                requestAnimationFrame(()=>this.animationLoop());
+            }
+
+            dispImageData(){
+                //From https://stackoverflow.com/questions/67507616/set-image-src-from-image-blob/67507685#67507685
+                if(this.imgObjURL != null){
+                    URL.revokeObjectURL(this.imgObjURL)
+                }
+                this.imgObjURL = URL.createObjectURL(this.imgData);
+
+                //Update the image with the new mimetype and image
+                this.image.src = this.imgObjURL;
+            }
+
+            dispNoStream() {
+                this.image.src = "loading.gif";
+            }
+
+            animationLoop(){
+                // Update time metrics
+                var now = window.performance.now();
+                var timeInState  = now - this.dsm_restart_start_time;
+
+                // Save previous state
+                this.dsm_prev_state = this.dsm_cur_state;
+
+                // Evaluate state transitions
+                if(this.serverConnectionActive == false){
+                    //Any state - if the server connection goes false, always transition to disconnected
+                    this.dsm_cur_state = this.DSM_DISCONNECTED;
+                } else {
+                    //Conditional transitions
+                    switch(this.dsm_cur_state) {
+                        case this.DSM_DISCONNECTED:
+                            //Immediately transition to waiting for the first frame
+                            this.dsm_cur_state = this.DSM_WAIT_FOR_VALID_PORT;
+                            break;
+                        case this.DSM_WAIT_FOR_VALID_PORT:
+                            // Wait until the user has configured a valid port
+                            if(this.streamPort > 0){
+                                this.dsm_cur_state = this.DSM_SUBSCRIBE;
+                            } else {
+                                this.dsm_cur_state = this.DSM_WAIT_FOR_VALID_PORT;
+                            }
+                            break;
+                        case this.DSM_SUBSCRIBE:
+                            // Immediately transition after subscriptions is sent
+                            this.dsm_cur_state = this.DSM_WAIT_FOR_FIRST_FRAME;
+                            break;
+                        case this.DSM_WAIT_FOR_FIRST_FRAME:
+                            if(this.imgData != null){
+                                //we got some image data, start showing it
+                                this.dsm_cur_state = this.DSM_SHOWING;
+                            } else if (this.newStreamPortReq != null){
+                                //Stream port requested changed, unsubscribe and restart
+                                this.dsm_cur_state = this.DSM_RESTART_UNSUBSCRIBE;
+                            } else {
+                                this.dsm_cur_state = this.DSM_WAIT_FOR_FIRST_FRAME;
+                            }
+                            break;
+                        case this.DSM_SHOWING:
+                            if((now - this.imgDataTime) > 2500){
+                                //timeout, begin the restart sequence
+                                this.dsm_cur_state = this.DSM_RESTART_UNSUBSCRIBE;
+                            } else if (this.newStreamPortReq != null){
+                                //Stream port requested changed, unsubscribe and restart
+                                this.dsm_cur_state = this.DSM_RESTART_UNSUBSCRIBE;
+                            }  else {
+                                //stay in this state.
+                                this.dsm_cur_state = this.DSM_SHOWING;
+                            }
+                            break;
+                        case this.DSM_RESTART_UNSUBSCRIBE:
+                            //Only should spend one loop in Unsubscribe, immediately transition
+                            this.dsm_cur_state = this.DSM_RESTART_WAIT;
+                            break;
+                        case this.DSM_RESTART_WAIT:
+                            if (timeInState > 250) {
+                                //we've waited long enough, go to try to re-subscribe
+                                this.dsm_cur_state = this.DSM_WAIT_FOR_VALID_PORT;
+                            } else {
+                                //stay in this state.
+                                this.dsm_cur_state = this.DSM_RESTART_WAIT;
+                            }
+                            break;
+                        default:
+                            // Shouldn't get here, default back to init
+                            this.dsm_cur_state = this.DSM_DISCONNECTED;
+                    }
+                }
+
+                //take current-state or state-transition actions
+
+                if(this.dsm_cur_state != this.dsm_prev_state){
+                    //Any state transition
+                    console.log("State Change: " + this.dsm_prev_state + " -> " + this.dsm_cur_state);
+                }
+
+                if(this.dsm_cur_state == this.DSM_SHOWING){
+                    // Currently in SHOWING
+                    this.dispImageData();
+                }
+
+                if(this.dsm_cur_state != this.DSM_SHOWING && this.dsm_prev_state == this.DSM_SHOWING ){
+                    //Any transition out of showing - no stream
+                    this.dispNoStream();
+                }
+
+                if(this.dsm_cur_state == this.DSM_RESTART_UNSUBSCRIBE){
+                    // Currently in UNSUBSCRIBE, do the unsubscribe actions
+                    this.stopStream();
+                    this.dsm_restart_start_time = now;
+                }
+
+                if(this.dsm_cur_state == this.DSM_SUBSCRIBE){
+                    // Currently in SUBSCRIBE, do the subscribe actions
+                    this.startStream();
+                    this.dsm_restart_start_time = now;
+                }
+
+                if(this.dsm_cur_state == this.DSM_WAIT_FOR_VALID_PORT){
+                    // Currently waiting for a vaild port to be requested
+                    if(this.newStreamPortReq != null){
+                        this.streamPort = this.newStreamPortReq;
+                        this.newStreamPortReq = null;
+                    }
+                }
+
+                requestAnimationFrame(()=>this.animationLoop());
+            }
+
+            startStream() {
+                console.log("Subscribing to port " + this.streamPort);
+                this.imgData = null;
+                this.ws.send(JSON.stringify({"cmd": "subscribe", "port":this.streamPort}));
+            }
+
+            stopStream() {
+                console.log("Unsubscribing");
+                this.ws.send(JSON.stringify({"cmd": "unsubscribe"}));
+                this.imgData = null;
+            }
+
+            setPort(streamPort){
+                console.log("Port set to " + streamPort);
+                this.newStreamPortReq = streamPort;
+            }
+
+            ws_onOpen() {
+                // Set the flag allowing general server communication
+                this.serverConnectionActive = true;
+                console.log("Connected!");
+            }
+
+            ws_onClose(e) {
+                //Clear flags to stop server communication
+                this.ws = null;
+                this.serverConnectionActive = false;
+
+                console.log('Camera Socket is closed. Reconnect will be attempted in 0.5 second.', e.reason);
+                setTimeout(this.ws_connect.bind(this), 500);
+
+                if(!e.wasClean){
+                    console.error('Socket encountered error!');
+                }
+
+            }
+
+            ws_onError(e){
+                e; //prevent unused failure
+                this.ws.close();
+            }
+
+            ws_onMessage(e){
+                if(typeof e.data === 'string'){
+                    //string data from host
+                    //TODO - anything to recieve info here? Maybe "avaialble streams?"
+                } else {
+                    if(e.data.size > 0){
+                        //binary data - a frame
+                        this.imgData = e.data;
+                        this.imgDataTime = window.performance.now();
+                        this.frameRxCount++;
+                    } else {
+                        //TODO - server is sending empty frames?
+                    }
+
+                }
+
+            }
+
+            ws_connect() {
+                this.serverConnectionActive = false;
+                this.ws = new WebSocket(this.serverAddr);
+                this.ws.binaryType = "blob";
+                this.ws.onopen = this.ws_onOpen.bind(this);
+                this.ws.onmessage = this.ws_onMessage.bind(this);
+                this.ws.onclose = this.ws_onClose.bind(this);
+                this.ws.onerror = this.ws_onError.bind(this);
+                console.log("Connecting to server " + this.serverAddr);
+            }
+
+            ws_close(){
+                this.ws.close();
+            }
+
+        }
+
+        var stream = null;
+
+        function streamStartRequest() {
+            var host = document.getElementById("host").value + ":5800";
+            var port = document.getElementById("port").value;
+            if(stream == null){
+                stream = new WebsocketVideoStream("streamImg",port,host);
+            } else {
+                stream.setPort(port);
+            }
+
+        }
+
+        // Attach listener
+        document.querySelector('button').addEventListener('click', streamStartRequest);
+
+        // Deal with URLParams, validating inputs
+        const queryString = window.location.search;
+        const urlParams = new URLSearchParams(queryString);
+        const port_in = urlParams.get('port')
+        const host_in = urlParams.get('host')
+        if(port_in != ""){
+            document.getElementById("port").value = port_in;
+        }
+
+        if(host_in != ""){
+            document.getElementById("host").value = host_in;
+        }
+
+        if(port_in != "" && host_in != ""){
+            streamStartRequest(); //we got valid inputs, auto-start the stream
+        }
+
+
+    </script>
+
+</body>
+
+
+</html>

photon-client/src/App.vue

@@ -1,35 +1,17 @@
 <template>
   <v-app>
     <!-- Although most of the app runs with the "light" theme, the navigation drawer needs to have white text and icons so it uses the dark theme-->
-    <v-navigation-drawer
-      dark
-      app
-      permanent
-      :mini-variant="compact"
-      color="primary"
-    >
+    <v-navigation-drawer dark app permanent :mini-variant="compact" color="primary">
       <v-list>
         <!-- List item for the heading; note that there are some tricks in setting padding and image width make things look right -->
         <v-list-item :class="compact ? 'pr-0 pl-0' : ''">
           <v-list-item-icon class="mr-0">
-            <img
-              v-if="!compact"
-              class="logo"
-              src="./assets/logoLarge.png"
-            >
-            <img
-              v-else
-              class="logo"
-              src="./assets/logoSmall.png"
-            >
+            <img v-if="!compact" class="logo" src="./assets/logoLarge.png">
+            <img v-else class="logo" src="./assets/logoSmall.png">
           </v-list-item-icon>
         </v-list-item>
 
-        <v-list-item
-          link
-          to="dashboard"
-          @click="rollbackPipelineIndex()"
-        >
+        <v-list-item link to="dashboard" @click="rollbackPipelineIndex()">
           <v-list-item-icon>
             <v-icon>mdi-view-dashboard</v-icon>
           </v-list-item-icon>
@@ -37,12 +19,7 @@
             <v-list-item-title>Dashboard</v-list-item-title>
           </v-list-item-content>
         </v-list-item>
-        <v-list-item
-          ref="camerasTabOpener"
-          link
-          to="cameras"
-          @click="switchToDriverMode()"
-        >
+        <v-list-item ref="camerasTabOpener" link to="cameras" @click="switchToDriverMode()">
           <v-list-item-icon>
             <v-icon>mdi-camera</v-icon>
           </v-list-item-icon>
@@ -50,11 +27,7 @@
             <v-list-item-title>Cameras</v-list-item-title>
           </v-list-item-content>
         </v-list-item>
-        <v-list-item
-          link
-          to="settings"
-          @click="switchToSettingsTab()"
-        >
+        <v-list-item link to="settings" @click="switchToSettingsTab()">
           <v-list-item-icon>
             <v-icon>mdi-settings</v-icon>
           </v-list-item-icon>
@@ -62,10 +35,7 @@
             <v-list-item-title>Settings</v-list-item-title>
           </v-list-item-content>
         </v-list-item>
-        <v-list-item
-          link
-          to="docs"
-        >
+        <v-list-item link to="docs">
           <v-list-item-icon>
             <v-icon>mdi-bookshelf</v-icon>
           </v-list-item-icon>
@@ -73,11 +43,7 @@
             <v-list-item-title>Documentation</v-list-item-title>
           </v-list-item-content>
         </v-list-item>
-        <v-list-item
-          v-if="this.$vuetify.breakpoint.mdAndUp"
-          link
-          @click.stop="toggleCompactMode"
-        >
+        <v-list-item v-if="this.$vuetify.breakpoint.mdAndUp" link @click.stop="toggleCompactMode">
           <v-list-item-icon>
             <v-icon v-if="compact">
               mdi-chevron-right
@@ -97,44 +63,24 @@
               <v-icon v-if="$store.state.settings.networkSettings.runNTServer">
                 mdi-server
               </v-icon>
-              <img
-                v-else-if="$store.state.ntConnectionInfo.connected"
-                src="@/assets/robot.svg"
-                alt=""
-              >
-              <img
-                v-else
-                class="pulse"
-                style="border-radius: 100%"
-                src="@/assets/robot-off.svg"
-                alt=""
-              >
+              <img v-else-if="$store.state.ntConnectionInfo.connected" src="@/assets/robot.svg" alt="">
+              <img v-else class="pulse" style="border-radius: 100%" src="@/assets/robot-off.svg" alt="">
             </v-list-item-icon>
             <v-list-item-content>
-              <v-list-item-title
-                v-if="$store.state.settings.networkSettings.runNTServer"
-                class="text-wrap"
-              >
-                NetworkTables server running for {{ $store.state.ntConnectionInfo.clients ? $store.state.ntConnectionInfo.clients : 'zero' }} clients!
+              <v-list-item-title v-if="$store.state.settings.networkSettings.runNTServer" class="text-wrap">
+                NetworkTables server running for {{ $store.state.ntConnectionInfo.clients ?
+                    $store.state.ntConnectionInfo.clients : 'zero'
+                }} clients!
               </v-list-item-title>
-              <v-list-item-title
-                v-else-if="$store.state.ntConnectionInfo.connected && $store.state.backendConnected"
-                class="text-wrap"
-              >
+              <v-list-item-title v-else-if="$store.state.ntConnectionInfo.connected && $store.state.backendConnected"
+                class="text-wrap">
                 Robot connected! {{ $store.state.ntConnectionInfo.address }}
               </v-list-item-title>
-              <v-list-item-title
-                v-else
-                class="text-wrap"
-              >
+              <v-list-item-title v-else class="text-wrap">
                 Not connected to robot!
               </v-list-item-title>
-              <router-link
-                v-if="!$store.state.settings.networkSettings.runNTServer"
-                to="settings"
-                class="accent--text"
-                @click="switchToSettingsTab"
-              >
+              <router-link v-if="!$store.state.settings.networkSettings.runNTServer" to="settings" class="accent--text"
+                @click="switchToSettingsTab">
                 Team number is {{ $store.state.settings.networkSettings.teamNumber }}
               </router-link>
             </v-list-item-content>
@@ -145,11 +91,7 @@
               <v-icon v-if="$store.state.backendConnected">
                 mdi-wifi
               </v-icon>
-              <v-icon
-                v-else
-                class="pulse"
-                style="border-radius: 100%;"
-              >
+              <v-icon v-else class="pulse" style="border-radius: 100%;">
                 mdi-wifi-off
               </v-icon>
             </v-list-item-icon>
@@ -163,10 +105,7 @@
       </v-list>
     </v-navigation-drawer>
     <v-main>
-      <v-container
-        fluid
-        fill-height
-      >
+      <v-container fluid fill-height>
         <v-layout>
           <v-flex>
             <router-view @switch-to-cameras="switchToDriverMode" />
@@ -175,33 +114,16 @@
       </v-container>
     </v-main>
 
-    <v-dialog
-      v-model="$store.state.logsOverlay"
-      width="1500"
-      dark
-    >
+    <v-dialog v-model="$store.state.logsOverlay" width="1500" dark>
       <logs />
     </v-dialog>
-    <v-dialog
-      v-model="needsTeamNumberSet"
-      width="500"
-      dark
-      persistent
-    >
-      <v-card
-        dark
-        color="primary"
-        flat
-      >
+    <v-dialog v-model="needsTeamNumberSet" width="500" dark persistent>
+      <v-card dark color="primary" flat>
         <v-card-title>No team number set!</v-card-title>
         <v-card-text>
           PhotonVision cannot connect to your robot! Please
-          <router-link
-            to="settings"
-            class="accent--text"
-            @click="switchToSettingsTab"
-          >
-            vist the settings tab
+          <router-link to="settings" class="accent--text" @click="switchToSettingsTab">
+            visit the settings tab
           </router-link>
           and set your team number.
         </v-card-text>
@@ -212,141 +134,143 @@
 
 <script>
 import Logs from "./views/LogsView"
-// import {mapState} from "vuex";
+import { ReconnectingWebsocket } from "./plugins/ReconnectingWebsocket.js"
 
 export default {
-        name: 'App',
-        components: {
-            Logs
-        },
-        data: () => ({
-            // Used so that we can switch back to the previously selected pipeline after camera calibration
-            previouslySelectedIndices: [],
-            timer: undefined,
-            teamNumberDialog: true
-        }),
-        computed: {
-            needsTeamNumberSet: {
-              get() {
-                return this.$store.state.settings.networkSettings.teamNumber < 1
-                    && this.teamNumberDialog && this.$store.state.backendConnected
-                    && !this.$route.name.toLowerCase().includes("settings");
-              }
-            },
-            compact: {
-                get() {
-                    if (this.$store.state.compactMode === undefined) {
-                        return this.$vuetify.breakpoint.smAndDown;
-                    } else {
-                        return this.$store.state.compactMode || this.$vuetify.breakpoint.smAndDown;
-                    }
-                },
-                set(value) {
-                    // compactMode is the user's preference for compact mode; it overrides screen size
-                    this.$store.commit("compactMode", value);
-                    localStorage.setItem("compactMode", value);
-                },
-            },
-        },
-        created() {
-            document.addEventListener("keydown", e => {
-                switch (e.key) {
-                    case "`":
-                        this.$store.state.logsOverlay = !this.$store.state.logsOverlay;
-                        break;
-                    case "z":
-                        if (e.ctrlKey && this.$store.getters.canUndo) {
-                            this.$store.dispatch('undo', {vm: this});
-                        }
-                        break;
-                    case "y":
-                        if (e.ctrlKey && this.$store.getters.canRedo) {
-                            this.$store.dispatch('redo', {vm: this});
-                        }
-                        break;
-
-                }
-            });
-
-            this.$options.sockets.onmessage = (data) => {
-                try {
-                    let message = this.$msgPack.decode(data.data);
-                    for (let prop in message) {
-                        if (message.hasOwnProperty(prop)) {
-                            this.handleMessage(prop, message[prop]);
-                        }
-                    }
-                } catch (error) {
-                    console.error('error: ' + JSON.stringify(data.data) + " , " + error);
-                }
-            };
-            this.$options.sockets.onopen = () => {
-                this.$store.commit("backendConnected", true)
-                this.$store.state.connectedCallbacks.forEach(it => it())
-            };
-
-            let closed = () => {
-              this.$store.commit("backendConnected", false)
-            };
-            this.$options.sockets.onclose = closed;
-            this.$options.sockets.onerror = closed;
-
-            this.$connect();
-        },
-        methods: {
-            handleMessage(key, value) {
-                if (key === "logMessage") {
-                    this.logMessage(value["logMessage"], value["logLevel"]);
-                } else if(key === "log"){
-                    this.logMessage(value["logMessage"]["logMessage"], value["logMessage"]["logLevel"]);
-                } else if (key === "updatePipelineResult") {
-                    this.$store.commit('mutatePipelineResults', value)
-                } else if (this.$store.state.hasOwnProperty(key)) {
-                    this.$store.commit(key, value);
-                } else if (this.$store.getters.currentPipelineSettings.hasOwnProperty(key)) {
-                    this.$store.commit('mutatePipeline', {[key]: value});
-                } else if (this.$store.state.settings.hasOwnProperty(key)) {
-                    this.$store.commit('mutateSettings', {[key]: value});
-                } else {
-                    switch (key) {
-                        default: {
-                            console.error("Unknown message from backend: " + value);
-                        }
-                    }
-                }
-            },
-            toggleCompactMode() {
-                this.compact = !this.compact;
-            },
-            // eslint-disable-next-line no-unused-vars
-            logMessage(message, levelInt) {
-                this.$store.commit('logString', {
-                    ['level']: levelInt,
-                    ['message']: message
-                })
-            },
-            switchToDriverMode() {
-                if (!this.previouslySelectedIndices) this.previouslySelectedIndices = [];
-
-                for (const [i, cameraSettings] of this.$store.state.cameraSettings.entries()) {
-                  this.previouslySelectedIndices[i] = cameraSettings.currentPipelineIndex;
-                  this.handleInputWithIndex('currentPipeline', -1, i);
-                }
-            },
-            rollbackPipelineIndex()
-            {
-              if (this.previouslySelectedIndices !== null) {
-                for (const [i] of this.$store.state.cameraSettings.entries()) {
-                  this.handleInputWithIndex('currentPipeline', this.previouslySelectedIndices[i] || 0, i);
-                }
-              }
-              this.previouslySelectedIndices = null;
-            },
-            switchToSettingsTab() {
-                this.axios.post('http://' + this.$address + '/api/sendMetrics', {})
-            }
+  name: 'App',
+  components: {
+    Logs
+  },
+  data: () => ({
+    // Used so that we can switch back to the previously selected pipeline after camera calibration
+    previouslySelectedIndices: [],
+    timer: undefined,
+    teamNumberDialog: true,
+    websocket: null,
+  }),
+  computed: {
+    needsTeamNumberSet: {
+      get() {
+        return this.$store.state.settings.networkSettings.teamNumber < 1
+          && this.teamNumberDialog && this.$store.state.backendConnected
+          && !this.$route.name.toLowerCase().includes("settings");
+      }
+    },
+    compact: {
+      get() {
+        if (this.$store.state.compactMode === undefined) {
+          return this.$vuetify.breakpoint.smAndDown;
+        } else {
+          return this.$store.state.compactMode || this.$vuetify.breakpoint.smAndDown;
         }
-    };
+      },
+      set(value) {
+        // compactMode is the user's preference for compact mode; it overrides screen size
+        this.$store.commit("compactMode", value);
+        localStorage.setItem("compactMode", value);
+      },
+    },
+  },
+  created() {
+    document.addEventListener("keydown", e => {
+      switch (e.key) {
+        case "`":
+          this.$store.state.logsOverlay = !this.$store.state.logsOverlay;
+          break;
+        case "z":
+          if (e.ctrlKey && this.$store.getters.canUndo) {
+            this.$store.dispatch('undo', { vm: this });
+          }
+          break;
+        case "y":
+          if (e.ctrlKey && this.$store.getters.canRedo) {
+            this.$store.dispatch('redo', { vm: this });
+          }
+          break;
+
+      }
+    });
+
+    const wsDataURL = 'ws://' + this.$address + '/websocket_data';
+    this.websocket = new ReconnectingWebsocket(
+      wsDataURL,
+
+      // On data in
+      (event) => {
+        try {
+          let message = this.$msgPack.decode(event.data);
+          for (let prop in message) {
+            if (message.hasOwnProperty(prop)) {
+              this.handleMessage(prop, message[prop]);
+            }
+          }
+        } catch (error) {
+          console.log(event)
+          console.error('error: ' + JSON.stringify(event.data) + " , " + error);
+        }
+      },
+
+      // on connect
+      (event) => {
+        event; this.$store.commit("backendConnected", true);
+        this.$store.state.connectedCallbacks.forEach(it => it());
+       },
+
+      // on disconnect
+      (event) => { event; this.$store.commit("backendConnected", false) }
+    );
+
+    this.$store.commit("websocket", this.websocket);
+  },
+  methods: {
+    handleMessage(key, value) {
+      if (key === "logMessage") {
+        this.logMessage(value["logMessage"], value["logLevel"]);
+      } else if (key === "log") {
+        this.logMessage(value["logMessage"]["logMessage"], value["logMessage"]["logLevel"]);
+      } else if (key === "updatePipelineResult") {
+        this.$store.commit('mutatePipelineResults', value)
+      } else if (this.$store.state.hasOwnProperty(key)) {
+        this.$store.commit(key, value);
+      } else if (this.$store.getters.currentPipelineSettings.hasOwnProperty(key)) {
+        this.$store.commit('mutatePipeline', { [key]: value });
+      } else if (this.$store.state.settings.hasOwnProperty(key)) {
+        this.$store.commit('mutateSettings', { [key]: value });
+      } else {
+        console.error("Unknown message from backend: " + value);
+      }
+    },
+    toggleCompactMode() {
+      this.compact = !this.compact;
+    },
+    // eslint-disable-next-line no-unused-vars
+    logMessage(message, levelInt) {
+      this.$store.commit('logString', {
+        ['level']: levelInt,
+        ['message']: message
+      })
+    },
+    switchToDriverMode() {
+      if (!this.previouslySelectedIndices) this.previouslySelectedIndices = [];
+
+      for (const [i, cameraSettings] of this.$store.state.cameraSettings.entries()) {
+        this.previouslySelectedIndices[i] = cameraSettings.currentPipelineIndex;
+        this.handleInputWithIndex('currentPipeline', -1, i);
+      }
+    },
+    rollbackPipelineIndex() {
+      if (this.previouslySelectedIndices !== null) {
+        for (const [i] of this.$store.state.cameraSettings.entries()) {
+          this.handleInputWithIndex('currentPipeline', this.previouslySelectedIndices[i] || 0, i);
+        }
+      }
+      this.previouslySelectedIndices = null;
+    },
+    switchToSettingsTab() {
+      this.axios.post('http://' + this.$address + '/api/sendMetrics', {})
+    }
+  }
+};
 </script>
 
 <style lang="sass">
@@ -354,76 +278,77 @@ export default {
 </style>
 
 <style>
-    .pulse {
-        animation: pulse-animation 2s infinite;
-    }
+.pulse {
+  animation: pulse-animation 2s infinite;
+}
 
-    @keyframes pulse-animation {
-        0% {
-            box-shadow: 0 0 0 0px rgba(0, 0, 0, 0.2);
-            background-color: rgba(0, 0, 0, 0.2);
-        }
-        100% {
-            box-shadow: 0 0 0 20px rgba(0, 0, 0, 0);
-            background-color: rgba(0, 0, 0, 0);
-        }
-    }
+@keyframes pulse-animation {
+  0% {
+    box-shadow: 0 0 0 0px rgba(0, 0, 0, 0.2);
+    background-color: rgba(0, 0, 0, 0.2);
+  }
 
-    .logo {
-        width: 100%;
-        height: 70px;
-        object-fit: contain;
-    }
+  100% {
+    box-shadow: 0 0 0 20px rgba(0, 0, 0, 0);
+    background-color: rgba(0, 0, 0, 0);
+  }
+}
 
-    ::-webkit-scrollbar {
-        width: 0.5em;
-        border-radius: 5px;
-    }
+.logo {
+  width: 100%;
+  height: 70px;
+  object-fit: contain;
+}
 
-    ::-webkit-scrollbar-track {
-        -webkit-box-shadow: inset 0 0 6px rgba(0, 0, 0, 0.3);
-        border-radius: 10px;
-    }
+::-webkit-scrollbar {
+  width: 0.5em;
+  border-radius: 5px;
+}
+
+::-webkit-scrollbar-track {
+  -webkit-box-shadow: inset 0 0 6px rgba(0, 0, 0, 0.3);
+  border-radius: 10px;
+}
 
 
-    ::-webkit-scrollbar-thumb {
-        background-color: #ffd843;
-        border-radius: 10px;
-    }
+::-webkit-scrollbar-thumb {
+  background-color: #ffd843;
+  border-radius: 10px;
+}
 
-    .container {
-        background-color: #232c37;
-        padding: 0 !important;
-    }
+.container {
+  background-color: #232c37;
+  padding: 0 !important;
+}
 
-    #title {
-        color: #ffd843;
-    }
+#title {
+  color: #ffd843;
+}
 </style>
 
 <style>
-  /* Hacks */
+/* Hacks */
 
-  .v-divider {
-    border-color: white !important;
-  }
+.v-divider {
+  border-color: white !important;
+}
 
-  .v-input {
-      font-size: 1rem !important;
-  }
+.v-input {
+  font-size: 1rem !important;
+}
 
-  /* This is unfortunately the only way to override table background color */
-  .theme--dark.v-data-table > .v-data-table__wrapper > table > tbody > tr:hover:not(.v-data-table__expanded__content):not(.v-data-table__empty-wrapper) {
-    background: #005281 !important;
-  }
+/* This is unfortunately the only way to override table background color */
+.theme--dark.v-data-table>.v-data-table__wrapper>table>tbody>tr:hover:not(.v-data-table__expanded__content):not(.v-data-table__empty-wrapper) {
+  background: #005281 !important;
+}
 </style>
 
 <style lang="scss">
-    @import '~vuetify/src/styles/settings/_variables';
+@import '~vuetify/src/styles/settings/_variables';
 
-    @media #{map-get($display-breakpoints, 'md-and-down')} {
-        html {
-            font-size: 14px !important;
-        }
-    }
+@media #{map-get($display-breakpoints, 'md-and-down')} {
+  html {
+    font-size: 14px !important;
+  }
+}
 </style>

photon-client/src/components/common/cv-image.vue

@@ -4,16 +4,17 @@
     crossOrigin="anonymous"
     :style="styleObject"
     :src="src"
-    alt=""
-    @click="e => $emit('click', e)"
-  >
+    :alt="alt"
+    @click="clickHandler"
+    @error="loadErrHandler"
+  />
 </template>
 
 <script>
     export default {
         name: "CvImage",
         // eslint-disable-next-line vue/require-prop-types
-        props: ['address', 'scale', 'maxHeight', 'maxHeightMd', 'maxHeightLg', 'maxHeightXl', 'colorPicking', 'id', 'disconnected'],
+        props: ['address', 'scale', 'maxHeight', 'maxHeightMd', 'maxHeightLg', 'maxHeightXl', 'colorPicking', 'id', 'disconnected', 'alt'],
         data() {
             return {
                 seed: 1.0,
@@ -26,13 +27,14 @@
                       "border-radius": "3px",
                       "display": "block",
                       "object-fit": "contain",
+                      "background-size:": "contain",
                       "object-position": "50% 50%",
                       "max-width": "100%",
                       "margin-left": "auto",
                       "margin-right": "auto",
                       "max-height": this.maxHeight,
                       height: `${this.scale}%`,
-                      cursor: (this.colorPicking ? `url(${require("../../assets/eyedropper.svg")}),` : "") + "default",
+                      cursor: (this.colorPicking ? `url(${require("../../assets/eyedropper.svg")}),` : "pointer") + "default",
                     };
 
                     if (this.$vuetify.breakpoint.xl) {
@@ -48,7 +50,14 @@
             },
             src: {
               get() {
-                return this.disconnected ? require("../../assets/noStream.jpg") : this.address + "?" + this.seed // This prevents caching
+                var port = this.getCurPort();
+                if(port <= 0){
+                  //Invalid port, keep it spinny
+                  return require("../../assets/loading.gif");
+                } else {
+                  //Valid port, connect
+                  return this.getSrcURLFromPort(port);
+                }
               },
             },
         },
@@ -56,6 +65,43 @@
             this.reload(); // Force reload image on creation
         },
         methods: {
+            getCurPort(){
+              var port = -1;
+              if(this.disconnected){
+                //Disconnected, port is unknown.
+                port = -1;
+              } else {
+                //Connected - get the port
+                if(this.id == 'raw-stream'){
+                  port = this.$store.state.cameraSettings[this.$store.state.currentCameraIndex].inputStreamPort
+                } else {
+                  port = this.$store.state.cameraSettings[this.$store.state.currentCameraIndex].outputStreamPort
+                }
+              }
+              return port;
+            },
+            getSrcURLFromPort(port){
+              return "http://" + location.hostname + ":" + port + "/stream.mjpg" + "?" + this.seed;
+            },
+            loadErrHandler(event) {
+                console.log(event);
+                console.log("Error loading image, attempting to do it again...");
+                this.reload();
+            },
+            clickHandler(event) {
+              if(this.colorPicking){
+                this.$emit('click', event);
+              } else {
+                var port = this.getCurPort();
+                if(port <= 0){
+                  console.log("No valid port, ignoring click.");
+                } else {
+                  //Valid port, connect
+                  window.open(this.getSrcURLFromPort(port), '_blank');
+                }
+              }
+
+            },
             reload() {
                 this.seed = new Date().getTime();
             }

photon-client/src/components/common/cv-input.vue

@@ -26,7 +26,7 @@
     </v-row>
   </div>
 </template>
-s
+
 <script>
     import TooltippedLabel from "./cv-tooltipped-label";
 

photon-client/src/components/common/cv-select.vue

@@ -37,7 +37,7 @@ import TooltippedLabel from "./cv-tooltipped-label";
             TooltippedLabel,
         },
       // eslint-disable-next-line vue/require-prop-types
-        props: ['list', 'name', 'value', 'disabled', 'selectCols', 'rules', 'tooltip'],
+        props: ['list', 'name', 'value', 'disabled', 'filteredIndices', 'selectCols', 'rules', 'tooltip'],
         computed: {
             localValue: {
                 get() {
@@ -50,6 +50,7 @@ import TooltippedLabel from "./cv-tooltipped-label";
             indexList() {
                 let list = [];
                 for (let i = 0; i < this.list.length; i++) {
+                    if (this.filteredIndices instanceof Set && this.filteredIndices.has(i)) continue;
                     list.push({
                         name: this.list[i],
                         index: i

photon-client/src/components/pipeline/3D/MiniMap.vue

@@ -1,154 +1,268 @@
 <template>
-  <div>
+  <div
+    id="MapContainer"
+    style="flex-grow:1"
+  >
     <v-row>
       <v-col
         align="center"
         cols="12"
       >
         <span class="white--text">Target Location</span>
+      </v-col>
+    </v-row>
+    <v-row>
+      <v-col
+        align="center"
+        cols="12"
+        align-self="stretch"
+      >
         <canvas
           id="canvasId"
-          class="mt-2"
-          width="800"
-          height="800"
+          style="width:100%;height:100%"
         />
       </v-col>
+      <v-row>
+        <v-col>
+          <v-btn
+            class="ml-10"
+            color="secondary"
+            @click="resetCamFirstPerson"
+          >
+            First Person
+          </v-btn>
+        </v-col>
+        <v-col>
+          <v-btn
+            class="ml-10"
+            color="secondary"
+            @click="resetCamThirdPerson"
+          >
+            Third Person
+          </v-btn>
+        </v-col>
+      </v-row>
     </v-row>
   </div>
 </template>
 
 <script>
-    import theme from "../../../theme";
 
-    export default {
-        name: "MiniMap",
-        props: {
-          // eslint-disable-next-line vue/require-default-prop
-          targets: Array,
-          // eslint-disable-next-line vue/require-default-prop
-          horizontalFOV: Number
-        },
-        data() {
-            return {
-                ctx: undefined,
-                canvas: undefined,
-                x: 0,
-                y: 0,
-                targetWidth: 40,
-                targetHeight: 6
-            }
-        },
-        computed: {
-            hLen: {
-                get() {
-                    return Math.tan(this.horizontalFOV / 2 * Math.PI / 180) * 150;
-                }
-            }
-        },
-        watch: {
-            targets: {
-                deep: true,
-                handler() {
-                    this.draw();
-                }
-            },
-            horizontalFOV() {
-                this.draw();
-            }
-        },
-        mounted: function () {
-            const canvas = document.getElementById("canvasId"); // getting the canvas element
-            const ctx = canvas.getContext("2d"); // getting the canvas context
-            this.canvas = canvas; // setting the canvas as a vue variable
-            this.ctx = ctx; // setting the canvas context as a vue variable
-            this.grad = this.ctx.createLinearGradient(400, 800, 400, 600);
-            this.grad.addColorStop(0, "rgb(119,119,119)");
-            this.grad.addColorStop(0.05, "rgba(14,92,22,0.96)");
-            this.grad.addColorStop(0.8, 'rgba(43,43,43,0.48)');
+import {
+  ArrowHelper,
+  BoxGeometry,
+  ConeGeometry,
+  Mesh,
+  MeshNormalMaterial,
+  PerspectiveCamera,
+  Quaternion,
+  Scene,
+  TrackballControls,
+  Vector3,
+    Color,
+  WebGLRenderer
+} from "three-full";
 
-            // setting canvas context values for drawing
+export default {
+  name: "MiniMap",
+  props: {
+    // eslint-disable-next-line vue/require-default-prop
+    targets: Array,
+    // eslint-disable-next-line vue/require-default-prop
+    horizontalFOV: Number
+  },
+  data() {
+    return {
+      scene: undefined,
+      cubes: [],
 
-
-            this.ctx.font = "26px Arial";
-            this.ctx.strokeStyle = "whitesmoke";
-            this.ctx.lineWidth = 2;
-
-            this.$nextTick(function () {
-                this.drawPlayer();
-            });
-        },
-        methods: {
-            draw() {
-                this.clearBoard();
-                this.drawPlayer();
-                for (let index in this.targets) {
-                    this.drawTarget(index, this.targets[index].pose);
-                }
-            },
-            drawTarget(index, target) {
-                // first save the untranslated/unrotated context
-                let x = 800 - (160 * target.x); // getting meters as pixels
-                let y = 400 - (160 * target.y);
-                this.ctx.save();
-                this.ctx.beginPath();
-                // move the rotation point to the center of the rect
-                this.ctx.translate(y + this.targetWidth / 2, x + this.targetHeight / 2); // wpi lib makes x forward and back and y left to right
-                // rotate the rect
-                this.ctx.rotate(target.rot * -1 * Math.PI / 180.0);
-
-                // draw the rect on the transformed context
-                // Note: after transforming [0,0] is visually [x,y]
-                //       so the rect needs to be offset accordingly when drawn
-                this.ctx.rect(-this.targetWidth / 2, -this.targetHeight / 2, this.targetWidth, this.targetHeight);
-
-                this.ctx.fillStyle = theme.accent;
-                this.ctx.fill();
-
-                // restore the context to its untranslated/unrotated state
-                this.ctx.restore();
-                this.ctx.fillStyle = "whitesmoke";
-                this.ctx.beginPath();
-                this.ctx.arc(y + this.targetWidth / 2, x + this.targetHeight / 2, 3, 0, 2 * Math.PI, true);
-                this.ctx.fill();
-                this.ctx.fillText(index, y - 30, x - 5);
-
-            },
-            drawPlayer() {
-                this.ctx.beginPath();
-                this.ctx.moveTo(400, 820);
-                this.ctx.lineTo(400 + this.hLen, 650);
-                this.ctx.lineTo(400 - this.hLen, 650);
-                this.ctx.closePath();
-                this.ctx.fillStyle = this.grad;
-                this.ctx.fill();
-                this.ctx.beginPath();
-                this.ctx.moveTo(400, 820);
-                this.ctx.lineTo(400 + this.hLen, 650);
-                this.ctx.stroke();
-                this.ctx.moveTo(400, 820);
-                this.ctx.lineTo(400 - this.hLen, 650);
-                this.ctx.stroke();
-
-            },
-            clearBoard() {
-                this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height); // clearing the canvas
-            }
-        }
     }
+  },
+  watch: {
+    targets: {
+      deep: true,
+      handler() {
+        this.drawTargets();
+      }
+    },
+  },
+
+  mounted() {
+    const scene = new Scene();
+    this.scene = scene;
+    const camera = new PerspectiveCamera(75, 800 / 800, 0.1, 1000);
+    this.camera = camera;
+
+    const canvas = document.getElementById("canvasId"); // getting the canvas element
+    this.canvas = canvas;
+    const renderer = new WebGLRenderer({"canvas": canvas});
+    this.renderer = renderer;
+    scene.background = new Color(0xa9a9a9)
+
+    //Set up resize handlers
+    this.onWindowResize();
+    window.addEventListener( 'resize', this.onWindowResize, false );
+
+    //Add the reference frame cues
+    this.refFrameCues = []
+    // coordinate system
+    this.refFrameCues.push(new ArrowHelper(new Vector3(1, 0, 0).normalize(), new Vector3(0, 0, 0),
+        1, // length
+        0xff0000,
+        0.1,
+        0.1,
+    ))
+    this.refFrameCues.push(new ArrowHelper(new Vector3(0, 1, 0).normalize(), new Vector3(0, 0, 0),
+        1, // length
+        0x00ff00,
+        0.1,
+        0.1,
+    ))
+    this.refFrameCues.push(new ArrowHelper(new Vector3(0, 0, 1).normalize(), new Vector3(0, 0, 0),
+        1, // length
+        0x0000ff,
+        0.1,
+        0.1,
+    ))
+
+    //something that looks vaguely like a camera
+    const camSize = 0.2;
+    const camBodyGeometry = new BoxGeometry(camSize, camSize, camSize);
+    const camLensGeometry = new ConeGeometry(camSize*0.4, camSize*0.8, 30);
+    const camMaterial = new MeshNormalMaterial();
+    const camBody = new Mesh(camBodyGeometry, camMaterial);
+    const camLens = new Mesh(camLensGeometry, camMaterial);
+    camBody.position.set(0,0,0);
+    camLens.rotateZ(Math.PI / 2);
+    camLens.position.set(camSize*0.8,0,0);
+    this.refFrameCues.push(camBody)
+    this.refFrameCues.push(camLens)
+
+    var controls = new TrackballControls(
+        camera,
+        renderer.domElement
+    );
+    controls.rotateSpeed = 1.0;
+    controls.zoomSpeed = 1.2;
+    controls.panSpeed = 0.8;
+    controls.noZoom = false;
+    controls.noPan = false;
+    controls.staticMoving = true;
+    controls.dynamicDampingFactor = 0.3;
+    controls.keys = [65, 83, 68];
+    this.controls = controls;
+
+    this.scene.add(...this.refFrameCues)
+    this.resetCamFirstPerson();
+
+    controls.update();
+
+    function animate() {
+      requestAnimationFrame(animate);
+
+      controls.update();
+      renderer.render(scene, camera);
+
+      //camera.updateMatrixWorld();
+      //console.log("================")
+      //console.log(camera.position);
+      //console.log(camera.rotation);
+      //console.log(camera.up);
+
+    }
+
+    this.drawTargets()
+
+    animate();
+  },
+  methods: {
+    drawTargets() {
+      this.scene.remove(...this.cubes)
+      this.cubes = []
+
+      for (const target of this.targets) {
+        const geometry = new BoxGeometry(0.3 / 5, 0.2, 0.2);
+        const material = new MeshNormalMaterial();
+        let quat = (new Quaternion(
+            target.pose.qx,
+            target.pose.qy,
+            target.pose.qz,
+            target.pose.qw,
+        ))
+        const cube = new Mesh(geometry, material);
+        cube.position.set(target.pose.x, target.pose.y, target.pose.z)
+        cube.rotation.setFromQuaternion(quat);
+        this.cubes.push(cube)
+
+        let arrow = (new ArrowHelper(new Vector3(1, 0, 0).normalize(), new Vector3(0, 0, 0),
+            1, // length
+            0xff0000,
+            0.1,
+            0.1,
+        ));
+        arrow.rotation.setFromQuaternion(quat)
+        arrow.rotateZ(-Math.PI / 2)
+        arrow.position.set(target.pose.x, target.pose.y, target.pose.z)
+        this.cubes.push(arrow);
+
+        arrow = (new ArrowHelper(new Vector3(1, 0, 0).normalize(), new Vector3(0, 0, 0),
+            1, // length
+            0x00ff00,
+            0.1,
+            0.1,
+        ));
+        arrow.rotation.setFromQuaternion(quat)
+        // arrow.rotateX(Math.PI / 2)
+        arrow.position.set(target.pose.x, target.pose.y, target.pose.z)
+        this.cubes.push(arrow);
+        arrow = (new ArrowHelper(new Vector3(1, 0, 0).normalize(), new Vector3(0, 0, 0),
+            1, // length
+            0x0000ff,
+            0.1,
+            0.1,
+        ));
+        arrow.setRotationFromQuaternion(quat)
+        arrow.rotateX(Math.PI / 2)
+        arrow.position.set(target.pose.x, target.pose.y, target.pose.z)
+        this.cubes.push(arrow);
+      }
+      if(this.cubes.length > 0)
+        this.scene.add(...this.cubes);
+    },
+
+    onWindowResize() {
+      var container = document.getElementById("MapContainer")
+      if(container){
+        this.canvas.width = container.clientWidth * 0.95;
+        this.canvas.height = container.clientWidth * 0.85;
+        this.camera.aspect = this.canvas.width / this.canvas.height;
+        this.camera.updateProjectionMatrix();
+        this.renderer.setSize( this.canvas.width, this.canvas.height );
+      }
+    },
+    resetCamThirdPerson(){
+      //Sets camera to third person position
+      this.controls.reset();
+      this.camera.position.set(-1.39,-1.09,1.17);
+      this.camera.up.set(0,0,1);
+      this.controls.target.set(4.0,0.0,0.0);
+      this.controls.update();
+      this.scene.add(...this.refFrameCues)
+    },
+    resetCamFirstPerson(){
+      //Sets camera to first person position
+      this.controls.reset();
+      this.camera.position.set(-0.1,0,0);
+      this.camera.up.set(0,0,1);
+      this.controls.target.set(0.0,0.0,0.0);
+      this.controls.update();
+      this.scene.remove(...this.refFrameCues)
+    },
+  }
+
+
+}
 </script>
 
 <style scoped>
-    #canvasId {
-        width: 400px;
-        height: 400px;
-        background-color: #232C37;
-        border-radius: 5px;
-        border: 2px solid grey;
-        box-shadow: 0 0 5px 1px;
-    }
-
-    th {
-        width: 80px;
-        text-align: center;
-    }
 </style>

photon-client/src/components/pipeline/CameraAndPipelineSelect.vue

@@ -153,7 +153,7 @@
           v-model="currentPipelineType"
           name="Type"
           tooltip="Changes the pipeline type, which changes the type of processing that will happen on input frames"
-          :list="['Reflective Tape', 'Colored Shape']"
+          :list="['Reflective Tape', 'Colored Shape', 'AprilTag']"
           @input="e => showTypeDialog(e)"
         />
       </v-col>
@@ -257,7 +257,7 @@ export default {
   },
   data: () => {
     return {
-      re: RegExp("^[A-Za-z0-9 \\-)(]*[A-Za-z0-9][A-Za-z0-9 \\-)(.]*$"),
+      re: RegExp("^[A-Za-z0-9_ \\-)(]*[A-Za-z0-9][A-Za-z0-9_ \\-)(.]*$"),
       isCameraNameEdit: false,
       newCameraName: "",
       cameraNameError: "",

photon-client/src/main.js

@@ -15,16 +15,15 @@ if (process.env.NODE_ENV === "production") {
     Vue.prototype.$address = location.hostname + ":5800";
 }
 
-const wsURL = '//' + Vue.prototype.$address + '/websocket';
+// const wsDataURL = '//' + Vue.prototype.$address + '/websocket_data';
+// import VueNativeSock from 'vue-native-websocket';
+// Vue.use(VueNativeSock, wsDataURL, {
+//     reconnection: true,
+//     reconnectionDelay: 100,
+//     connectManually: true,
+//     format: "arraybuffer",
+// });
 
-import VueNativeSock from 'vue-native-websocket';
-
-Vue.use(VueNativeSock, wsURL, {
-    reconnection: true,
-    reconnectionDelay: 100,
-    connectManually: true,
-    format: "arraybuffer",
-});
 Vue.use(VueAxios, axios);
 Vue.prototype.$msgPack = msgPack(true);
 

photon-client/src/mixins/global/dataHandleMixin.js

@@ -2,14 +2,14 @@ export const dataHandleMixin = {
     methods: {
         handleInput(key, value) {
             let msg = this.$msgPack.encode({[key]: value});
-            this.$socket.send(msg);
+            this.$store.state.websocket.ws.send(msg);
         },
         handleInputWithIndex(key, value, cameraIndex = this.$store.getters.currentCameraIndex) {
             let msg = this.$msgPack.encode({
                 [key]: value,
                 ["cameraIndex"]: cameraIndex,
             });
-            this.$socket.send(msg);
+            this.$store.state.websocket.ws.send(msg);
         },
         handleData(val) {
             this.handleInput(val, this[val]);
@@ -22,7 +22,7 @@ export const dataHandleMixin = {
                     ["cameraIndex"]: this.$store.getters.currentCameraIndex
                 }
             });
-            this.$socket.send(msg);
+            this.$store.state.websocket.ws.send(msg);
             this.$emit('update')
         },
         handlePipelineUpdate(key, val) {
@@ -32,7 +32,7 @@ export const dataHandleMixin = {
                     ["cameraIndex"]: this.$store.getters.currentCameraIndex
                 }
             });
-            this.$socket.send(msg);
+            this.$store.state.websocket.ws.send(msg);
             this.$emit('update')
         },
         handleTruthyPipelineData(val) {
@@ -42,7 +42,7 @@ export const dataHandleMixin = {
                     ["cameraIndex"]: this.$store.getters.currentCameraIndex
                 }
             });
-            this.$socket.send(msg);
+            this.$store.state.websocket.ws.send(msg);
             this.$emit('update')
         },
         rollback(val, e) {

photon-client/src/plugins/ColorPicker.js

@@ -5,7 +5,7 @@ function initColorPicker() {
     if (!canvas)
         canvas = document.createElement('canvas');
 
-    image = document.querySelector('#normal-stream');
+    image = document.querySelector('#raw-stream');
     if (image !== null) {
       canvas.width = image.width;
       canvas.height = image.height;

photon-client/src/plugins/ReconnectingWebsocket.js

@@ -0,0 +1,74 @@
+/**
+ * Auto-reconnecting Websocket, a stripped down version of the NT4 client from
+ * https://raw.githubusercontent.com/wpilibsuite/NetworkTablesClients/2f8d378ac08d5ca703d590cfb019fc4af062db89/nt4/js/src/nt4.js
+ */
+export class ReconnectingWebsocket {
+    constructor(serverAddr,
+        onDataIn_in,
+        onConnect_in,
+        onDisconnect_in) {
+
+        this.onDataIn = onDataIn_in;
+        this.onConnect = onConnect_in;
+        this.onDisconnect = onDisconnect_in;
+
+        // WS Connection State (with defaults)
+        this.serverAddr = serverAddr;
+        this.serverConnectionActive = false;
+
+        //Trigger the websocket to connect automatically
+        this.ws_connect();
+    }
+
+    //////////////////////////////////////////////////////////////
+    // Websocket connection Maintenance
+
+    ws_onOpen() {
+        // Set the flag allowing general server communication
+        this.serverConnectionActive = true;
+
+        console.log("[WebSocket] Connected!");
+
+        // User connection-opened hook
+        this.onConnect();
+    }
+
+    ws_onClose(e) {
+        //Clear flags to stop server communication
+        this.ws = null;
+        this.serverConnectionActive = false;
+
+        // User connection-closed hook
+        this.onDisconnect();
+
+        console.log('[WebSocket] Socket is closed. Reconnect will be attempted in 0.5 second.', e.reason);
+        setTimeout(this.ws_connect.bind(this), 500);
+
+        if (!e.wasClean) {
+            console.error('Socket encountered error!');
+        }
+
+    }
+
+    ws_onError(e) {
+        console.log("[WebSocket] Websocket error - " + e.toString());
+        this.ws.close();
+    }
+
+    ws_onMessage(e) {
+       this.onDataIn(e);
+    }
+
+    ws_connect() {
+        this.ws = new WebSocket(this.serverAddr);
+        this.ws.binaryType = "arraybuffer";
+        this.ws.onopen = this.ws_onOpen.bind(this);
+        this.ws.onmessage = this.ws_onMessage.bind(this);
+        this.ws.onclose = this.ws_onClose.bind(this);
+        this.ws.onerror = this.ws_onError.bind(this);
+
+        console.log("[WebSocket] Starting...");
+    }
+}
+
+export default { ReconnectingWebsocket }

photon-client/src/plugins/WebsocketVideoStream.js

@@ -0,0 +1,359 @@
+// Circular buffer storage. Externally-apparent 'length' increases indefinitely
+// while any items with indexes below length-n will be forgotten (undefined
+// will be returned if you try to get them, trying to set is an exception).
+// n represents the initial length of the array, not a maximum
+class StatsHistoryBuffer{
+    constructor (){
+        this.windowLen = 10;
+        this._array= new Array(this.windowLen);
+        this.headPtr = 0;
+        this.frameCount = 0;
+        this.bitAvgAccum = 0;
+
+        //calculated vals
+        this.bitRate_Mbps = 0;
+        this.framerate_fps = 0;
+    }
+
+    putAndPop(v){
+        this.headPtr++;
+        var idx = (this.headPtr)%this._array.length;
+        var poppedVal = this._array[idx];
+        this._array[idx] = v;
+        return poppedVal;
+    }
+
+    addSample(time, frameSize_bits, dispFrame_count) {
+        var oldVal = this.putAndPop([time, frameSize_bits, dispFrame_count]);
+
+        this.bitAvgAccum += frameSize_bits;
+
+        if(oldVal !=null){
+            var oldTime = oldVal[0];
+            var oldFrameSize = oldVal[1];
+            var oldFrameCount = oldVal[2];
+
+            var deltaTime_s = (time - oldTime);
+
+            this.bitAvgAccum -= oldFrameSize;
+
+            //bitrate - total bits transferred over the time period, divided by the period length
+            // converted to mbps
+            this.bitRate_Mbps = ( this.bitAvgAccum / deltaTime_s ) * (1.0/1048576.0);
+
+            //framerate - total frames displayed over the time period, divided by the period length
+            this.framerate_fps = (dispFrame_count - oldFrameCount) / deltaTime_s;
+        }
+
+
+    }
+
+    getText(){
+        return "Streaming @ " + this.framerate_fps.toFixed(1) + "FPS  " + this.bitRate_Mbps.toFixed(1) + "Mbps";
+    }
+
+}
+
+
+export class WebsocketVideoStream{
+
+    constructor(drawDiv, streamPort, host) {
+        console.log("host " + host + " port " + streamPort)
+
+        this.drawDiv = drawDiv;
+        this.image = document.getElementById(this.drawDiv);
+        this.streamPort = streamPort;
+        this.newStreamPortReq = null;
+        this.serverAddr = "ws://" + host + "/websocket_cameras";
+        this.imgData = null;
+        this.imgDataTime = -1;
+        this.prevImgDataTime = -1;
+        this.imgObjURL = null;
+        this.frameRxCount = 0;
+        this.dispFrameCount = 0;
+        this.stats = null;
+
+        //Set up div for stream stats info provided for users
+        this.statsTextDiv = this.image.parentNode.appendChild(document.createElement("div"));
+
+        //Centered over the image
+        this.statsTextDiv.style.position = "absolute";
+        this.statsTextDiv.style.left = "50%";
+        this.statsTextDiv.style.top = "50%";
+        this.statsTextDiv.style.transform = "translate(-50%, -50%)";
+
+        // Big enough for a line or two of text, with centered text
+        this.statsTextDiv.style.padding = "0.5em"
+        this.statsTextDiv.style.overflow = "hidden";
+        this.statsTextDiv.style.textAlign = "center";
+        this.statsTextDiv.style.verticalAlign = "middle";
+
+        // Styled to be black with grey text
+        this.statsTextDiv.style.backgroundColor = "black";
+        this.statsTextDiv.style.color = "#9E9E9E";
+        this.statsTextDiv.style.borderRadius = "3px";
+
+        //Default no text
+        this.statsTextDiv.innerHTML = "";
+
+        // Only show on mouseover, with opacity fade-in/fade-out
+        this.statsTextDiv.style.opacity = "0.0";
+        this.statsTextDiv.style.transition = "opacity 0.25s ease 0.25s";
+        this.statsTextDiv.style.transitionDelay = "opacity 0.5s";
+        this.image.addEventListener('mouseover', () => {this.statsTextDiv.style.opacity = "0.6";});
+        this.statsTextDiv.addEventListener('mouseover', () => {this.statsTextDiv.style.opacity = "0.6";});
+        this.image.addEventListener('mouseout', () => {this.statsTextDiv.style.opacity = "0.0";});
+
+        //Display state machine descriptions
+        this.DSM_DISCONNECTED = "Disconnected";
+        this.DSM_WAIT_FOR_VALID_PORT = "Waiting for valid port ID";
+        this.DSM_SUBSCRIBE = "Subscribing";
+        this.DSM_WAIT_FOR_FIRST_FRAME = "Waiting for frame data";
+        this.DSM_SHOWING = "Showing Frames";
+        this.DSM_RESTART_UNSUBSCRIBE = "Unsubscribing";
+        this.DSM_RESTART_WAIT = "Waiting before resubscribe";
+
+        this.dsm_cur_state = this.DSM_DISCONNECTED;
+        this.dsm_prev_state = this.DSM_DISCONNECTED;
+        this.dsm_restart_start_time = window.performance.now();
+
+        this.dispNoStream();
+        this.ws_connect();
+
+        requestAnimationFrame(()=>this.animationLoop());
+
+    }
+
+    dispImageData(){
+        if(this.prevImgDataTime != this.imgDataTime){
+            //From https://stackoverflow.com/questions/67507616/set-image-src-from-image-blob/67507685#67507685
+            //Ensure uniqueness by making the new one before revoking the old one.
+            var oldURL = this.imgObjURL
+            this.imgObjURL = URL.createObjectURL(this.imgData);
+            if(oldURL != null){
+                URL.revokeObjectURL(oldURL)
+            }
+
+            //Update the image with the new mimetype and image
+            this.image.src = this.imgObjURL;
+
+            this.dispFrameCount++;
+            this.prevImgDataTime = this.imgDataTime;
+        } // else no new image, don't update anything
+    }
+
+    dispNoStream() {
+        this.image.src = require("../assets/loading.gif");
+    }
+
+    animationLoop(){
+        // Update time metrics
+        var curTime_s = window.performance.now() / 1000.0;
+        var timeInState  = curTime_s - this.dsm_restart_start_time;
+
+        // Save previous state
+        this.dsm_prev_state = this.dsm_cur_state;
+
+        // Evaluate state transitions
+        if(this.serverConnectionActive == false){
+            //Any state - if the server connection goes false, always transition to disconnected
+            this.dsm_cur_state = this.DSM_DISCONNECTED;
+        } else {
+            //Conditional transitions
+            switch(this.dsm_cur_state) {
+                case this.DSM_DISCONNECTED:
+                    //Immediately transition to waiting for the first frame
+                    this.dsm_cur_state = this.DSM_WAIT_FOR_VALID_PORT;
+                    break;
+                case this.DSM_WAIT_FOR_VALID_PORT:
+                    // Wait until the user has configured a valid port
+                    if(this.streamPort > 0){
+                        this.dsm_cur_state = this.DSM_SUBSCRIBE;
+                    } else {
+                        this.dsm_cur_state = this.DSM_WAIT_FOR_VALID_PORT;
+                    }
+                    break;
+                case this.DSM_SUBSCRIBE:
+                    // Immediately transition after subscriptions is sent
+                    this.dsm_cur_state = this.DSM_WAIT_FOR_FIRST_FRAME;
+                    break;
+                case this.DSM_WAIT_FOR_FIRST_FRAME:
+                    if(this.imgData != null){
+                        //we got some image data, start showing it
+                        this.dsm_cur_state = this.DSM_SHOWING;
+                    } else if (this.newStreamPortReq != null){
+                        //Stream port requested changed, unsubscribe and restart
+                        this.dsm_cur_state = this.DSM_RESTART_UNSUBSCRIBE;
+                    } else {
+                        this.dsm_cur_state = this.DSM_WAIT_FOR_FIRST_FRAME;
+                    }
+                    break;
+                case this.DSM_SHOWING:
+                    if((curTime_s - this.imgDataTime) > 2.5){
+                        //timeout, begin the restart sequence
+                        this.dsm_cur_state = this.DSM_RESTART_UNSUBSCRIBE;
+                    } else if (this.newStreamPortReq != null){
+                        //Stream port requested changed, unsubscribe and restart
+                        this.dsm_cur_state = this.DSM_RESTART_UNSUBSCRIBE;
+                    }  else {
+                        //stay in this state.
+                        this.dsm_cur_state = this.DSM_SHOWING;
+                    }
+                    break;
+                case this.DSM_RESTART_UNSUBSCRIBE:
+                    //Only should spend one loop in Unsubscribe, immediately transition
+                    this.dsm_cur_state = this.DSM_RESTART_WAIT;
+                    break;
+                case this.DSM_RESTART_WAIT:
+                    if (timeInState > 0.25) {
+                        //we've waited long enough, go to try to re-subscribe
+                        this.dsm_cur_state = this.DSM_WAIT_FOR_VALID_PORT;
+                    } else {
+                        //stay in this state.
+                        this.dsm_cur_state = this.DSM_RESTART_WAIT;
+                    }
+                    break;
+                default:
+                    // Shouldn't get here, default back to init
+                    this.dsm_cur_state = this.DSM_DISCONNECTED;
+              }
+        }
+
+        //take current-state or state-transition actions
+
+        if(this.dsm_cur_state != this.dsm_prev_state){
+            //Any state transition
+            console.log("State Change: " + this.dsm_prev_state + " -> " + this.dsm_cur_state);
+        }
+
+        if(this.dsm_cur_state == this.DSM_SHOWING){
+            // Currently in SHOWING
+            // Show image and update status text
+            this.dispImageData();
+            this.statsTextDiv.innerHTML = this.stats.getText();
+        } else {
+            //Just show the state for debug
+            this.statsTextDiv.innerHTML = this.dsm_cur_state;
+        }
+
+        if(this.dsm_cur_state != this.DSM_SHOWING && this.dsm_prev_state == this.DSM_SHOWING ){
+            //Any transition out of showing - no stream
+            this.dispNoStream();
+        }
+
+        if(this.dsm_cur_state == this.DSM_RESTART_UNSUBSCRIBE){
+            // Currently in UNSUBSCRIBE, do the unsubscribe actions
+            this.stopStream();
+            this.dsm_restart_start_time = curTime_s;
+        }
+
+        if(this.dsm_cur_state == this.DSM_SUBSCRIBE){
+            // Currently in SUBSCRIBE, do the subscribe actions
+            this.startStream();
+            this.dsm_restart_start_time = curTime_s;
+        }
+
+        if(this.dsm_cur_state == this.DSM_WAIT_FOR_VALID_PORT){
+            // Currently waiting for a vaild port to be requested
+            if(this.newStreamPortReq != null){
+                this.streamPort = this.newStreamPortReq;
+                this.newStreamPortReq = null;
+            }
+        }
+
+        //Update status text
+
+        requestAnimationFrame(()=>this.animationLoop());
+    }
+
+    startStream() {
+        console.log("Subscribing to port " + this.streamPort);
+        this.imgData = null;
+        this.ws.send(JSON.stringify({"cmd": "subscribe", "port":this.streamPort}));
+    }
+
+    stopStream() {
+        console.log("Unsubscribing");
+        this.ws.send(JSON.stringify({"cmd": "unsubscribe"}));
+        this.imgData = null;
+    }
+
+    setPort(streamPort){
+        console.log("Port set to " + streamPort);
+        this.newStreamPortReq = streamPort;
+    }
+
+    ws_onOpen() {
+        // Set the flag allowing general server communication
+        this.serverConnectionActive = true;
+        console.log("Camera Websockets Connected!");
+
+        // New websocket connection, reset stats
+        this.frameRxCount = 0;
+        this.dispFrameCount = 0;
+        this.stats = new StatsHistoryBuffer();
+    }
+
+    ws_onClose(e) {
+        //Clear flags to stop server communication
+        this.ws = null;
+        this.serverConnectionActive = false;
+
+        console.log('Camera Socket is closed. Reconnect will be attempted in 0.5 second.', e.reason);
+        setTimeout(this.ws_connect.bind(this), 500);
+
+        if(!e.wasClean){
+            console.error('Socket encountered error!');
+        }
+
+    }
+
+    ws_onError(e){
+        e; //prevent unused failure
+        this.ws.close();
+    }
+
+    ws_onMessage(e){
+        //console.log("Got message from " + this.serverAddr)
+        var msgTime_s = window.performance.now() / 1000.0;
+        if(typeof e.data === 'string'){
+            //string data from host
+            //TODO - anything to receive info here? Maybe "available streams?"
+        } else {
+            if(e.data.size > 0){
+                //binary data - a frame!
+                //Save frame data for display in the next animation thread
+                this.imgData = e.data;
+                this.imgDataTime = msgTime_s;
+
+                //Count the incoming frame
+                this.frameRxCount++;
+
+                //keep the stats up to date
+                this.stats.addSample(msgTime_s,this.imgData.size * 8,this.dispFrameCount);
+            } else {
+                console.log("WS Stream Error: Server sent empty frame!");
+            }
+        }
+
+    }
+
+    ws_connect() {
+        this.serverConnectionActive = false;
+        this.ws = new WebSocket(this.serverAddr);
+        this.ws.binaryType = "blob";
+        this.ws.onopen = this.ws_onOpen.bind(this);
+        this.ws.onmessage = this.ws_onMessage.bind(this);
+        this.ws.onclose = this.ws_onClose.bind(this);
+        this.ws.onerror = this.ws_onError.bind(this);
+        console.log("Connecting to server " + this.serverAddr);
+    }
+
+    ws_close(){
+        this.ws.close();
+    }
+
+}
+
+
+export default {WebsocketVideoStream}

photon-client/src/plugins/objectDiff.js

@@ -1,121 +0,0 @@
-//https://gomakethings.com/getting-the-differences-between-two-objects-with-vanilla-js/
-export const diff = function (obj1, obj2) {
-
-    // Make sure an object to compare is provided
-    if (!obj2 || Object.prototype.toString.call(obj2) !== '[object Object]') {
-        return obj1;
-    }
-
-    //
-    // Variables
-    //
-
-    let diffs = {};
-    let key;
-
-
-    //
-    // Methods
-    //
-
-    /**
-     * Check if two arrays are equal
-     * @param  {Array}   arr1 The first array
-     * @param  {Array}   arr2 The second array
-     * @return {Boolean}      If true, both arrays are equal
-     */
-    const arraysMatch = function (arr1, arr2) {
-
-        // Check if the arrays are the same length
-        if (arr1.length !== arr2.length) return false;
-
-        // Check if all items exist and are in the same order
-        for (let i = 0; i < arr1.length; i++) {
-            if (arr1[i] !== arr2[i]) return false;
-        }
-
-        // Otherwise, return true
-        return true;
-
-    };
-
-    /**
-     * Compare two items and push non-matches to object
-     * @param  {*}      item1 The first item
-     * @param  {*}      item2 The second item
-     * @param  {String} key   The key in our object
-     */
-    const compare = function (item1, item2, key) {
-
-        // Get the object type
-        let type1 = Object.prototype.toString.call(item1);
-        let type2 = Object.prototype.toString.call(item2);
-
-        // If type2 is undefined it has been removed
-        if (type2 === '[object Undefined]') {
-            diffs[key] = null;
-            return;
-        }
-
-        // If items are different types
-        if (type1 !== type2) {
-            diffs[key] = item2;
-            return;
-        }
-
-        // If an object, compare recursively
-        if (type1 === '[object Object]') {
-            let objDiff = diff(item1, item2);
-            if (Object.keys(objDiff).length > 1) {
-                diffs[key] = objDiff;
-            }
-            return;
-        }
-
-        // If an array, compare
-        if (type1 === '[object Array]') {
-            if (!arraysMatch(item1, item2)) {
-                diffs[key] = item2;
-            }
-            return;
-        }
-
-        // Else if it's a function, convert to a string and compare
-        // Otherwise, just compare
-        if (type1 === '[object Function]') {
-            if (item1.toString() !== item2.toString()) {
-                diffs[key] = item2;
-            }
-        } else {
-            if (item1 !== item2) {
-                diffs[key] = item2;
-            }
-        }
-
-    };
-
-
-    //
-    // Compare our objects
-    //
-
-    // Loop through the first object
-    for (key in obj1) {
-        if (obj1.hasOwnProperty(key)) {
-            compare(obj1[key], obj2[key], key);
-        }
-    }
-
-    // Loop through the second object and find missing items
-    for (key in obj2) {
-        if (obj2.hasOwnProperty(key)) {
-            if (!obj1[key] && obj1[key] !== obj2[key] ) {
-                diffs[key] = obj2[key];
-            }
-        }
-    }
-
-    // Return the object of differences
-    return diffs;
-
-};

photon-client/src/store/index.js

@@ -15,6 +15,7 @@ export default new Vuex.Store({
     },
     state: {
         backendConnected: false,
+        websocket: null,
         ntConnectionInfo: {
             connected: false,
             address: "",
@@ -35,8 +36,8 @@ export default new Vuex.Store({
                 tiltDegrees: 0.0,
                 currentPipelineIndex: 0,
                 pipelineNicknames: ["Unknown"],
-                outputStreamPort: 1181,
-                inputStreamPort: 1182,
+                outputStreamPort: 0,
+                inputStreamPort: 0,
                 nickname: "Unknown",
                 videoFormatList: [
                     {
@@ -51,12 +52,13 @@ export default new Vuex.Store({
                 isFovConfigurable: true,
                 calibrated: false,
                 currentPipelineSettings: {
-                    pipelineType: 2, // One of "calib", "driver", "reflective", "shape"
+                    pipelineType: 5, // One of "calib", "driver", "reflective", "shape", "AprilTag"
                     // 2 is reflective
 
                     // Settings that apply to all pipeline types
                     cameraExposure: 1,
                     cameraBrightness: 2,
+                    cameraAutoExposure: false,
                     cameraRedGain: 3,
                     cameraBlueGain: 4,
                     inputImageRotationMode: 0,
@@ -88,7 +90,16 @@ export default new Vuex.Store({
 
                     cornerDetectionAccuracyPercentage: 10,
 
-                    // Settings that apply to shape
+                    // Settings that apply to AprilTag
+                    tagFamily: 1,
+                    decimate: 1.0,
+                    blur: 0.0,
+                    threads: 1,
+                    debug: false,
+                    refineEdges: true,
+                    numIterations: 1,
+                    decisionMargin: 0,
+                    hammingDist: 0,
                 }
             }
         ],
@@ -102,9 +113,18 @@ export default new Vuex.Store({
                     skew: 0,
                     area: 0,
                     // 3D only
-                    pose: {x: 0, y: 0, rot: 0},
-                }]
-            },
+                    pose: {x: 1, y: 1, z: 0, qw: 1, qx: 0, qy: 0, qz: 0},
+                },
+            {
+                // Available in both 2D and 3D
+                pitch: 0,
+                yaw: 0,
+                skew: 0,
+                area: 0,
+                // 3D only
+                pose: {x: 2, y: 3, z: 0, qw: 1, qx: 0, qy: 0, qz: 0},
+            }]
+        },
         settings: {
             general: {
                 version: "Unknown",
@@ -150,6 +170,7 @@ export default new Vuex.Store({
     },
     mutations: {
         compactMode: set('compactMode'),
+        websocket: set('websocket'),
         cameraSettings: set('cameraSettings'),
         currentCameraIndex: set('currentCameraIndex'),
         selectedOutputs: set('selectedOutputs'),

photon-client/src/views/CamerasView.vue

@@ -31,14 +31,6 @@
               :label-cols="$vuetify.breakpoint.mdAndUp ? undefined : 7"
             />
             <br>
-            <CVnumberinput
-              v-model="cameraSettings.tiltDegrees"
-              name="Camera pitch"
-              tooltip="How many degrees above the horizontal the physical camera is tilted"
-              :step="0.01"
-              :label-cols="$vuetify.breakpoint.mdAndUp ? undefined : 7"
-            />
-            <br>
             <v-btn
               style="margin-top:10px"
               small
@@ -80,6 +72,14 @@
                     :disabled="isCalibrating"
                     tooltip="Resolution to calibrate at (you will have to calibrate every resolution you use 3D mode on)"
                   />
+                  <CVselect
+                    v-model="streamingFrameDivisor"
+                    name="Decimation"
+                    tooltip="Resolution to which camera frames are downscaled for detection. Calibration still uses full-res"
+                    :list="calibrationDivisors"
+                    select-cols="7"
+                    @rollback="e => rollback('streamingFrameDivisor', e)"
+                  />
                   <CVselect
                     v-model="boardType"
                     name="Board Type"
@@ -146,6 +146,24 @@
                             text="Standard Deviation"
                           />
                         </th>
+                        <th class="text-center">
+                          <tooltipped-label
+                            tooltip="Estimated Horizontal FOV, in degrees"
+                            text="Horizontal FOV"
+                          />
+                        </th>
+                        <th class="text-center">
+                          <tooltipped-label
+                            tooltip="Estimated Vertical FOV, in degrees"
+                            text="Vertical FOV"
+                          />
+                        </th>
+                        <th class="text-center">
+                          <tooltipped-label
+                            tooltip="Estimated Diagonal FOV, in degrees"
+                            text="Diagonal FOV"
+                          />
+                        </th>
                       </tr>
                     </thead>
                     <tbody>
@@ -158,6 +176,9 @@
                           {{ isCalibrated(value) ? value.mean.toFixed(2) + "px" : "—" }}
                         </td>
                         <td> {{ isCalibrated(value) ? value.standardDeviation.toFixed(2) + "px" : "—" }} </td>
+                        <td> {{ isCalibrated(value) ? value.horizontalFOV.toFixed(2) + "°" : "—" }} </td>
+                        <td> {{ isCalibrated(value) ? value.verticalFOV.toFixed(2) + "°" : "—" }} </td>
+                        <td> {{ isCalibrated(value) ? value.diagonalFOV.toFixed(2) + "°" : "—" }} </td>
                       </tr>
                     </tbody>
                   </v-simple-table>
@@ -181,10 +202,13 @@
               >
                 <CVslider
                   v-model="$store.getters.currentPipelineSettings.cameraExposure"
+                  :disabled="$store.getters.currentPipelineSettings.cameraAutoExposure"
                   name="Exposure"
                   :min="0"
                   :max="100"
                   slider-cols="8"
+                  step="0.1"
+                  tooltip="Directly controls how much light is allowed to fall onto the sensor, which affects apparent brightness"
                   @input="e => handlePipelineUpdate('cameraExposure', e)"
                 />
                 <CVslider
@@ -195,6 +219,24 @@
                   slider-cols="8"
                   @input="e => handlePipelineUpdate('cameraBrightness', e)"
                 />
+                <CVswitch
+                  v-model="$store.getters.currentPipelineSettings.cameraAutoExposure"
+                  class="pt-2"
+                  name="Auto Exposure"
+                  tooltip="Enables or Disables camera automatic adjustment for current lighting conditions"
+                  @input="e => handlePipelineUpdate('cameraAutoExposure', e)"
+                />
+                <CVslider
+                  v-if="cameraGain >= 0"
+                  v-model="cameraGain"
+                  name="Camera Gain"
+                  min="0"
+                  max="100"
+                  tooltip="Controls camera gain, similar to brightness"
+                  :slider-cols="largeBox"
+                  @input="handlePipelineData('cameraGain')"
+                  @rollback="e => rollback('cameraGain', e)"
+                />
                 <CVslider
                   v-if="$store.getters.currentPipelineSettings.cameraRedGain !== -1"
                   v-model="$store.getters.currentPipelineSettings.cameraRedGain"
@@ -257,6 +299,19 @@
                   Download Target
                 </v-btn>
               </v-col>
+              <v-col>
+                <v-btn
+                  color="secondary"
+                  small
+                  style="width: 100%;"
+                  @click="$refs.importCalibrationFromCalibdb.click()"
+                >
+                  <v-icon left>
+                    mdi-upload
+                  </v-icon>
+                  Import From CalibDB
+                </v-btn>
+              </v-col>
             </v-row>
           </div>
         </v-card>
@@ -268,7 +323,8 @@
       >
         <template>
           <CVimage
-            :address="$store.getters.streamAddress[1]"
+            :id="cameras-cal"
+            :idx=1
             :disconnected="!$store.state.backendConnected"
             scale="100"
             style="border-radius: 5px;"
@@ -332,6 +388,20 @@
         </template>
       </v-col>
     </v-row>
+
+    <!-- Special hidden upload input that gets 'clicked' when the user imports settings -->
+    <input
+      ref="importCalibrationFromCalibdb"
+      type="file"
+      accept=".json"
+      style="display: none;"
+      @change="readImportedCalibration"
+    />
+
+    <v-snackbar v-model="uploadSnack" top :color="uploadSnackData.color" timeout="-1">
+      <span>{{ uploadSnackData.text }}</span>
+    </v-snackbar>
+
   </div>
 </template>
 
@@ -339,6 +409,7 @@
 import CVselect from '../components/common/cv-select';
 import CVnumberinput from '../components/common/cv-number-input';
 import CVslider from '../components/common/cv-slider';
+import CVswitch from '../components/common/cv-switch';
 import CVimage from "../components/common/cv-image";
 import TooltippedLabel from "../components/common/cv-tooltipped-label";
 import jsPDF from "jspdf";
@@ -351,6 +422,7 @@ export default {
         CVselect,
         CVnumberinput,
         CVslider,
+        CVswitch,
         CVimage
     },
     data() {
@@ -360,6 +432,12 @@ export default {
             calibrationFailed: false,
             filteredVideomodeIndex: 0,
             settingsValid: true,
+            unfilteredStreamDivisors: [1, 2, 4],
+            uploadSnackData: {
+              color: "success",
+              text: "",
+            },
+            uploadSnack: false,
         }
     },
     computed: {
@@ -384,6 +462,31 @@ export default {
             }
         },
 
+        cameraGain: {
+          get() {
+              return parseInt(this.$store.getters.currentPipelineSettings.cameraGain)
+          },
+          set(val) {
+              this.$store.commit("mutatePipeline", {"cameraGain": parseInt(val)});
+          }
+        },
+
+        calibrationDivisors: {
+          get() {
+            return this.unfilteredStreamDivisors.filter(item => {
+              var res = this.stringResolutionList[this.selectedFilteredResIndex].split(" X ").map(it => parseInt(it));
+              console.log(res);
+              console.log(item);
+              // Realistically, we need more than 320x240, but lower than this is
+              // basically unusable. For now, don't allow decimations that take us
+              // below that
+              const ret = ((res[0] / item) >= 300 && (res[1] / item) >= 220) || (item === 1);
+              console.log(ret);
+              return ret;
+            })
+          }
+        },
+
         // Makes sure there's only one entry per resolution
         filteredResolutionList: {
             get() {
@@ -396,6 +499,9 @@ export default {
                         if (calib != null) {
                             it['standardDeviation'] = calib.standardDeviation;
                             it['mean'] = calib.perViewErrors.reduce((a, b) => a + b) / calib.perViewErrors.length;
+                            it['horizontalFOV'] = 2 * Math.atan2(it.width/2,calib.intrinsics[0]) * (180/Math.PI);
+                            it['verticalFOV'] = 2 * Math.atan2(it.height/2,calib.intrinsics[4]) * (180/Math.PI);
+                            it['diagonalFOV'] = 2 * Math.atan2(Math.sqrt(it.width**2 + (it.height/(calib.intrinsics[4]/calib.intrinsics[0]))**2)/2,calib.intrinsics[0]) * (180/Math.PI);
                         }
                         filtered.push(it);
                     }
@@ -404,13 +510,11 @@ export default {
                 return filtered
             }
         },
-
         stringResolutionList: {
             get() {
                 return this.filteredResolutionList.map(res => `${res['width']} X ${res['height']}`);
             }
         },
-
         cameraSettings: {
             get() {
                 return this.$store.getters.currentCameraSettings;
@@ -420,6 +524,16 @@ export default {
             }
         },
 
+        streamingFrameDivisor: {
+            get() {
+                return this.$store.getters.currentPipelineSettings.streamingFrameDivisor;
+            },
+            set(val) {
+                this.$store.commit("mutatePipeline", {"streamingFrameDivisor": val});
+                this.handlePipelineUpdate("streamingFrameDivisor", val);
+            }
+        },
+
         boardType: {
             get() {
                 return this.calibrationData.boardType
@@ -489,6 +603,57 @@ export default {
         },
     },
     methods: {
+
+    readImportedCalibration(event) {
+      // let formData = new FormData();
+      // formData.append("zipData", event.target.files[0]);
+      const filename = event.target.files[0].name;
+
+      event.target.files[0].text().then(fileText => {
+        const data = {
+          "cameraIndex": this.$store.getters.currentCameraIndex,
+          "payload": fileText,
+          "filename": filename,
+        };
+
+        this.axios
+          .post("http://" + this.$address + "/api/calibration/import", data, {
+            headers: { "Content-Type": "text/plain" },
+          })
+          .then(() => {
+            this.uploadSnackData = {
+              color: "success",
+              text:
+                "Calibration imported successfully! PhotonVision will restart in the background...",
+            };
+            this.uploadSnack = true;
+          })
+          .catch((err) => {
+            if (err.response) {
+              this.uploadSnackData = {
+                color: "error",
+                text:
+                  "Error while uploading calibration file! Could not process provided file.",
+              };
+            } else if (err.request) {
+              this.uploadSnackData = {
+                color: "error",
+                text:
+                  "Error while uploading calibration file! No respond to upload attempt.",
+              };
+            } else {
+              this.uploadSnackData = {
+                color: "error",
+                text: "Error while uploading calibration file!",
+              };
+            }
+            this.uploadSnack = true;
+          });
+
+        })
+
+    },
+
         closeDialog() {
             this.snack = false;
             this.calibrationInProgress = false;
@@ -601,8 +766,7 @@ export default {
             this.axios.post("http://" + this.$address + "/api/settings/camera", {
                 "settings": this.cameraSettings,
                 "index": this.$store.state.currentCameraIndex
-            }).then(
-                function (response) {
+            }).then(response => {
                     if (response.status === 200) {
                         this.$store.state.saveBar = true;
                     }
@@ -623,14 +787,15 @@ export default {
             if (this.isCalibrating === true) {
                 data['takeCalibrationSnapshot'] = true
             } else {
+                // This store prevents an edge case of a user not selecting a different resolution, which causes the set logic to not be called
+                this.$store.commit('mutateCalibrationState', {['videoModeIndex']: this.filteredResolutionList[this.selectedFilteredResIndex].index});
                 const calData = this.calibrationData;
                 calData.isCalibrating = true;
                 data['startPnpCalibration'] = calData;
-
                 console.log("starting calibration with index " + calData.videoModeIndex);
             }
-
-            this.$socket.send(this.$msgPack.encode(data));
+            this.$store.commit('currentPipelineIndex', -2);
+            this.$store.state.websocket.ws.send(this.$msgPack.encode(data));
         },
         sendCalibrationFinish() {
             console.log("finishing calibration for index " + this.$store.getters.currentCameraIndex);

photon-client/src/views/LogsView.vue

@@ -12,12 +12,21 @@
         color="secondary"
         style="margin-left: auto;"
         depressed
-        @click="download('photonlog.log', rawLogs.map(it => it.message).join('\n'))"
+        @click="$refs.exportLogFile.click()"
       >
         <v-icon left>
           mdi-download
         </v-icon>
         Download Log
+
+        <!-- Special hidden link that gets 'clicked' when the user exports journalctl logs -->
+        <a
+          ref="exportLogFile"
+          style="color: black; text-decoration: none; display: none"
+          :href="'http://' + this.$address + '/api/settings/photonvision-journalctl.txt'"
+          download="photonvision-journalctl.txt"
+        />
+
       </v-btn>
     </v-card-title>
     <div class="pr-6 pl-6">

photon-client/src/views/PipelineView.vue

@@ -1,75 +1,75 @@
 <template>
   <div>
     <v-container
-      class="pa-3"
-      fluid
+        class="pa-3"
+        fluid
     >
       <v-row
-        no-gutters
-        align="center"
-        justify="center"
+          no-gutters
+          align="center"
+          justify="center"
       >
         <v-col
-          cols="12"
-          :class="['pb-3 ', 'pr-lg-3']"
-          lg="8"
-          align-self="stretch"
+            cols="12"
+            :class="['pb-3 ', 'pr-lg-3']"
+            lg="8"
+            align-self="stretch"
         >
           <v-card
-            color="primary"
-            height="100%"
-            style="display: flex; flex-direction: column"
-            dark
+              color="primary"
+              height="100%"
+              style="display: flex; flex-direction: column"
+              dark
           >
             <v-card-title
-              class="pb-0 mb-0 pl-4 pt-1"
-              style="height: 15%; min-height: 50px;"
+                class="pb-0 mb-0 pl-4 pt-1"
+                style="height: 15%; min-height: 50px;"
             >
               Cameras
               <v-chip
-                :class="fpsTooLow ? 'ml-2 mt-1' : 'mt-2'"
-                x-small
-                label
-                :color="fpsTooLow ? 'error' : 'transparent'"
-                :text-color="fpsTooLow ? 'white' : 'grey'"
+                  :class="fpsTooLow ? 'ml-2 mt-1' : 'mt-2'"
+                  x-small
+                  label
+                  :color="fpsTooLow ? 'error' : 'transparent'"
+                  :text-color="fpsTooLow ? 'white' : 'grey'"
               >
-                <span class="pr-1">{{ Math.round($store.state.pipelineResults.fps) }}&nbsp;FPS &ndash;</span>
-                <span v-if="!fpsTooLow">{{ Math.min(Math.round($store.state.pipelineResults.latency), 100) }} ms latency</span>
-                <span v-else-if="!$store.getters.currentPipelineSettings.inputShouldShow">HSV thresholds are too broad; narrow them for better performance</span>
-                <span v-else>stop viewing the color stream for better performance</span>
+                <span class="pr-1">Processing @ {{ Math.round($store.state.pipelineResults.fps) }}&nbsp;FPS &ndash;</span>
+                <span v-if="fpsTooLow && !$store.getters.currentPipelineSettings.inputShouldShow && $store.getters.pipelineType == 2">HSV thresholds are too broad; narrow them for better performance</span>
+                <span v-else-if="fpsTooLow && getters.currentCameraSettings.inputShouldShow">stop viewing the raw stream for better performance</span>
+                <span v-else>{{ Math.min(Math.round($store.state.pipelineResults.latency), 9999) }} ms latency</span>
               </v-chip>
               <v-switch
-                v-model="driverMode"
-                label="Driver Mode"
-                style="margin-left: auto;"
-                color="accent"
+                  v-model="driverMode"
+                  label="Driver Mode"
+                  style="margin-left: auto;"
+                  color="accent"
               />
             </v-card-title>
             <v-row
-              align="center"
+                align="center"
             >
               <v-col
-                v-for="idx in (selectedOutputs instanceof Array ? selectedOutputs : [selectedOutputs])"
-                :key="idx"
-                cols="12"
-                :md="selectedOutputs.length === 1 ? 12 : Math.floor(12 / selectedOutputs.length)"
-                class="pb-0 pt-0"
-                style="height: 100%;"
+                  v-for="idx in (selectedOutputs instanceof Array ? selectedOutputs : [selectedOutputs])"
+                  :key="idx"
+                  cols="12"
+                  :md="selectedOutputs.length === 1 ? 12 : Math.floor(12 / selectedOutputs.length)"
+                  class="pb-0 pt-0"
+                  style="height: 100%;"
               >
                 <div style="position: relative; width: 100%; height: 100%;">
                   <cv-image
-                    :id="idx === 0 ? 'normal-stream' : ''"
-                    ref="streams"
-                    :address="$store.getters.streamAddress[idx]"
-                    :disconnected="!$store.state.backendConnected"
-                    scale="100"
-                    :max-height="$store.getters.isDriverMode ? '40vh' : '300px'"
-                    :max-height-md="$store.getters.isDriverMode ? '50vh' : '380px'"
-                    :max-height-lg="$store.getters.isDriverMode ? '55vh' : '390px'"
-                    :max-height-xl="$store.getters.isDriverMode ? '60vh' : '450px'"
-                    :alt="'Stream' + idx"
-                    :color-picking="$store.state.colorPicking && idx === 0"
-                    @click="onImageClick"
+                      :id="idx === 0 ? 'raw-stream' : 'processed-stream'"
+                      ref="streams"
+                      :idx=idx
+                      :disconnected="!$store.state.backendConnected"
+                      scale="100"
+                      :max-height="$store.getters.isDriverMode ? '40vh' : '300px'"
+                      :max-height-md="$store.getters.isDriverMode ? '50vh' : '380px'"
+                      :max-height-lg="$store.getters.isDriverMode ? '55vh' : '390px'"
+                      :max-height-xl="$store.getters.isDriverMode ? '60vh' : '450px'"
+                      :alt="idx === 0 ? 'Raw stream' : 'Processed stream'"
+                      :color-picking="$store.state.colorPicking && idx === 0"
+                      @click="onImageClick"
                   />
                 </div>
               </v-col>
@@ -77,44 +77,44 @@
           </v-card>
         </v-col>
         <v-col
-          cols="12"
-          class="pb-3"
-          lg="4"
-          align-self="stretch"
+            cols="12"
+            class="pb-3"
+            lg="4"
+            align-self="stretch"
         >
           <v-card
-            color="primary"
+              color="primary"
           >
-            <camera-and-pipeline-select @camera-name-changed="reloadStreams" />
+            <camera-and-pipeline-select />
           </v-card>
           <v-card
-            :disabled="$store.getters.isDriverMode || $store.state.colorPicking"
-            class="mt-3"
-            color="primary"
+              :disabled="$store.getters.isDriverMode || $store.state.colorPicking"
+              class="mt-3"
+              color="primary"
           >
             <v-row
-              align="center"
-              class="pl-3 pr-3"
+                align="center"
+                class="pl-3 pr-3"
             >
               <v-col lg="12">
                 <p style="color: white;">
                   Processing mode:
                 </p>
                 <v-btn-toggle
-                  v-model="processingMode"
-                  mandatory
-                  dark
-                  class="fill"
+                    v-model="processingMode"
+                    mandatory
+                    dark
+                    class="fill"
                 >
                   <v-btn
-                    color="secondary"
+                      color="secondary"
                   >
                     <v-icon>mdi-crop-square</v-icon>
                     <span>2D</span>
                   </v-btn>
                   <v-btn
-                    color="secondary"
-                    @click="on3DClick"
+                      color="secondary"
+                      @click="on3DClick"
                   >
                     <v-icon>mdi-cube-outline</v-icon>
                     <span>3D</span>
@@ -126,25 +126,25 @@
                   Stream display:
                 </p>
                 <v-btn-toggle
-                  v-model="selectedOutputs"
-                  :multiple="$vuetify.breakpoint.mdAndUp"
-                  mandatory
-                  dark
-                  class="fill"
+                    v-model="selectedOutputs"
+                    :multiple="$vuetify.breakpoint.mdAndUp"
+                    mandatory
+                    dark
+                    class="fill"
                 >
                   <v-btn
-                    color="secondary"
-                    class="fill"
+                      color="secondary"
+                      class="fill"
                   >
-                    <v-icon>mdi-palette</v-icon>
-                    <span>Normal</span>
+                    <v-icon>mdi-import</v-icon>
+                    <span>Raw</span>
                   </v-btn>
                   <v-btn
-                    color="secondary"
-                    class="fill"
+                      color="secondary"
+                      class="fill"
                   >
-                    <v-icon>mdi-compare</v-icon>
-                    <span>Threshold</span>
+                    <v-icon>mdi-export</v-icon>
+                    <span>Processed</span>
                   </v-btn>
                 </v-btn-toggle>
               </v-col>
@@ -154,29 +154,29 @@
       </v-row>
       <v-row no-gutters>
         <v-col
-          v-for="(tabs, idx) in tabGroups"
-          :key="idx"
-          :cols="Math.floor(12 / tabGroups.length)"
-          :class="idx !== tabGroups.length - 1 ? 'pr-3' : ''"
-          align-self="stretch"
+            v-for="(tabs, idx) in tabGroups"
+            :key="idx"
+            :cols="Math.floor(12 / tabGroups.length)"
+            :class="idx !== tabGroups.length - 1 ? 'pr-3' : ''"
+            align-self="stretch"
         >
           <v-card
-            color="primary"
-            height="100%"
-            class="pr-4 pl-4"
+              color="primary"
+              height="100%"
+              class="pr-4 pl-4"
           >
             <v-tabs
-              v-if="!$store.getters.isDriverMode"
-              v-model="selectedTabs[idx]"
-              grow
-              background-color="primary"
-              dark
-              height="48"
-              slider-color="accent"
+                v-if="!$store.getters.isDriverMode"
+                v-model="selectedTabs[idx]"
+                grow
+                background-color="primary"
+                dark
+                height="48"
+                slider-color="accent"
             >
               <v-tab
-                v-for="(tab, i) in tabs.filter(it => it.name !== '3D' || $store.getters.currentPipelineSettings.solvePNPEnabled)"
-                :key="i"
+                  v-for="(tab, i) in tabs"
+                  :key="i"
               >
                 {{ tab.name }}
               </v-tab>
@@ -184,10 +184,10 @@
             <div class="pl-4 pr-4 pt-2">
               <keep-alive>
                 <component
-                  :is="(tabs[selectedTabs[idx]] || tabs[0]).component"
-                  :ref="(tabs[selectedTabs[idx]] || tabs[0]).name"
-                  v-model="$store.getters.pipeline"
-                  @update="$emit('save')"
+                    :is="(tabs[selectedTabs[idx]] || tabs[0]).component"
+                    :ref="(tabs[selectedTabs[idx]] || tabs[0]).name"
+                    v-model="$store.getters.pipeline"
+                    @update="$emit('save')"
                 />
               </keep-alive>
             </div>
@@ -197,18 +197,18 @@
     </v-container>
 
     <v-snackbar
-      v-model="showNTWarning"
-      color="error"
-      timeout="-1"
-      top
+        v-model="showNTWarning"
+        color="error"
+        timeout="-1"
+        top
     >
       {{ $store.state.settings.networkSettings.runNTServer ?
         "NetworkTables server enabled! PhotonLib may not work." :
         "NetworkTables not connected! Are you on a network with a robot?" }}
       <template v-slot:action>
         <v-btn
-          text
-          @click="hideNTWarning = true"
+            text
+            @click="hideNTWarning = true"
         >
           Hide
         </v-btn>
@@ -216,12 +216,12 @@
     </v-snackbar>
 
     <v-dialog
-      v-model="dialog"
-      width="500"
+        v-model="dialog"
+        width="500"
     >
       <v-card
-        color="primary"
-        dark
+          color="primary"
+          dark
       >
         <v-card-title>
           Current resolution not calibrated
@@ -230,9 +230,9 @@
         <v-card-text>
           Because the current resolution {{ this.$store.getters.currentVideoFormat.width }} x {{ this.$store.getters.currentVideoFormat.height }} is not yet calibrated, 3D mode cannot be enabled. Please
           <a
-            href="/#/cameras"
-            class="white--text"
-            @click="$emit('switch-to-cameras')"
+              href="/#/cameras"
+              class="white--text"
+              @click="$emit('switch-to-cameras')"
           > visit the Cameras tab</a> to calibrate this resolution. For now, SolvePNP will do nothing.
         </v-card-text>
 
@@ -241,9 +241,9 @@
         <v-card-actions>
           <v-spacer />
           <v-btn
-            color="white"
-            text
-            @click="closeUncalibratedDialog"
+              color="white"
+              text
+              @click="closeUncalibratedDialog"
           >
             OK
           </v-btn>
@@ -261,223 +261,267 @@ import ThresholdTab from './PipelineViews/ThresholdTab';
 import ContoursTab from './PipelineViews/ContoursTab';
 import OutputTab from './PipelineViews/OutputTab';
 import TargetsTab from "./PipelineViews/TargetsTab";
+import Map3DTab from './PipelineViews/Map3DTab';
 import PnPTab from './PipelineViews/PnPTab';
+import AprilTagTab from './PipelineViews/AprilTagTab';
+import ArucoTab from './PipelineViews/ArucoTab';
 
 export default {
-    name: 'Pipeline',
-    components: {
-        CameraAndPipelineSelect,
-        cvImage,
-        InputTab,
-        ThresholdTab,
-        ContoursTab,
-        OutputTab,
-        TargetsTab,
-        PnPTab,
-    },
-    data() {
-        return {
-            selectedTabsData: [0, 0, 0, 0],
-            counterData: 0,
-            dialog: false,
-            processingModeOverride: false,
-            hideNTWarning: false,
-        }
-    },
-    computed: {
-        selectedTabs: {
-            get() {
-                return this.$store.getters.isDriverMode ? [0] : this.selectedTabsData;
-            },
-            set(value) {
-                this.selectedTabsData = value;
-            }
-        },
-        tabGroups: {
-            get() {
-                let tabs = {
-                    input: {
-                        name: "Input",
-                        component: "InputTab",
-                    },
-                    threshold: {
-                        name: "Threshold",
-                        component: "ThresholdTab",
-                    },
-                    contours: {
-                        name: "Contours",
-                        component: "ContoursTab",
-                    },
-                    output: {
-                        name: "Output",
-                        component: "OutputTab",
-                    },
-                    targets: {
-                        name: "Target Info",
-                        component: "TargetsTab",
-                    },
-                    pnp: {
-                        name: "3D",
-                        component: "PnPTab",
-                    }
-                };
-
-                // 2D array of tab names and component names; each sub-array is a separate tab group
-                let ret = [];
-                if (this.$vuetify.breakpoint.smAndDown || this.$store.getters.isDriverMode || (this.$vuetify.breakpoint.mdAndDown && !this.$store.state.compactMode)) {
-                    // One big tab group with all the tabs
-                    ret[0] = Object.values(tabs);
-                } else if (this.$vuetify.breakpoint.mdAndDown || !this.$store.state.compactMode) {
-                    // Two tab groups, one with "input, threshold, contours, output" and the other with "target info, 3D"
-                    ret[0] = [tabs.input, tabs.threshold, tabs.contours, tabs.output];
-                    ret[1] = [tabs.targets, tabs.pnp];
-                } else if (this.$vuetify.breakpoint.lgAndDown) {
-                    // Three tab groups, one with "input", one with "threshold, contours, output", and the other with "target info, 3D"
-                    ret[0] = [tabs.input];
-                    ret[1] = [tabs.threshold, tabs.contours, tabs.output];
-                    ret[2] = [tabs.targets, tabs.pnp];
-                } else if (this.$vuetify.breakpoint.xl) {
-                    // Three tab groups, one with "input", one with "threshold, contours", and the other with "output, target info, 3D"
-                    ret[0] = [tabs.input];
-                    ret[1] = [tabs.threshold];
-                    ret[2] = [tabs.contours, tabs.output];
-                    ret[3] = [tabs.targets, tabs.pnp];
-                }
-
-                return ret;
-            }
-        },
-        processingMode: {
-            get() {
-                return (this.$store.getters.currentPipelineSettings.solvePNPEnabled || this.processingModeOverride) ? 1 : 0;
-            },
-            set(value) {
-                if (this.$store.getters.isCalibrated) {
-                    this.$store.getters.currentPipelineSettings.solvePNPEnabled = value === 1;
-                    this.handlePipelineUpdate("solvePNPEnabled", value === 1);
-                }
-            }
-        },
-        driverMode: {
-            get() {
-                return this.$store.getters.isDriverMode;
-            },
-            set(value) {
-                this.$store.getters.currentCameraSettings.currentPipelineIndex = value ? -1 : 0;
-                this.handleInputWithIndex('currentPipeline', value ? -1 : 0);
-            }
-        },
-        selectedOutputs: {
-            // All this logic exists to deal with the reality that the output select buttons sometimes need an array and sometimes need a number (depending on whether or not they're exclusive)
-            get() {
-                // We switch the selector to single-select only on sm-and-down size devices, so we have to return a Number instead of an Array in that state
-                let ret = [];
-                if (this.$store.state.colorPicking) {
-                    ret = [0]; // We want the input stream only while color picking
-                } else if (this.$store.getters.isDriverMode) {
-                    ret = [1]; // We want only the output stream in driver mode
-                } else {
-                  if (this.$store.getters.currentPipelineSettings.inputShouldShow) ret = ret.concat([0]);
-                  if (this.$store.getters.currentPipelineSettings.outputShouldShow) ret = ret.concat([1]);
-                  if (!ret.length) ret = [0];
-                }
-
-                if (this.$vuetify.breakpoint.mdAndUp) {
-                    return ret;
-                } else {
-                    return ret[0] || 0;
-                }
-            },
-            set(value) {
-                let valToCommit = [0];
-                if (value instanceof Array) {
-                  // Value is already an array, we don't need to do anything
-                  valToCommit = value;
-                } else if (value) {
-                  // Value is assumed to be a number, so we wrap it into an array
-                  valToCommit = [value];
-                }
-
-                this.$store.commit("mutatePipeline", {"inputShouldShow": valToCommit.includes(0)});
-                this.$store.commit("mutatePipeline", {"outputShouldShow": valToCommit.includes(1)});
-                this.handlePipelineUpdate("inputShouldShow", valToCommit.includes(0));
-            }
-        },
-        fpsTooLow: {
-            get() {
-                // For now we only show the FPS is too low warning when GPU acceleration is enabled, because we don't really trust the presented video modes otherwise
-                return this.$store.state.pipelineResults.fps - this.$store.getters.currentVideoFormat.fps < -5 && this.$store.state.pipelineResults.fps !== 0 && !this.$store.getters.isDriverMode && this.$store.state.settings.general.gpuAcceleration;
-            }
-        },
-        latency: {
-            get() {
-                return this.$store.getters.currentPipelineResults.latency;
-            }
-        },
-        isCalibrated: {
-          get() {
-            const resolution = this.$store.getters.videoFormatList[this.$store.getters.currentPipelineSettings.cameraVideoModeIndex];
-            return this.$store.getters.currentCameraSettings.calibrations
-                    .some(e => e.width === resolution.width && e.height === resolution.height)
-          }
-        },
-        isRobotConnected: {
-          get() {
-            // return this.$store.state.ntConnectionInfo.connected && this.$store.state.backendConnected;
-            return true;
-          }
-        },
-        showNTWarning: {
-          get() {
-            return (!this.$store.state.ntConnectionInfo.connected || this.$store.state.settings.networkSettings.runNTServer) && this.$store.state.settings.networkSettings.teamNumber > 0 && this.$store.state.backendConnected && !this.hideNTWarning;
-          }
-        },
-    },
-    created() {
-        this.$store.state.connectedCallbacks.push(this.reloadStreams)
-    },
-    methods: {
-        reloadStreams() {
-            // Reload the streams as we technically close and reopen them
-            this.$refs.streams.forEach(it => it.reload())
-        },
-        onImageClick(event) {
-            // Only run on the input stream
-            if (event.target.alt !== "Stream0") return;
-            // Get a reference to the threshold tab (if it is shown) and call its "onClick" method
-            let ref = this.$refs["Threshold"];
-            if (ref && ref[0])
-                ref[0].onClick(event)
-        },
-        on3DClick() {
-            if (!this.$store.getters.isCalibrated) {
-                this.dialog = true;
-                this.processingModeOverride = true;
-            }
-        },
-        closeUncalibratedDialog() {
-            this.dialog = false;
-            this.processingModeOverride = false;
-            // this.$store.getters.currentPipelineSettings.solvePNPEnabled = false;
-            this.handlePipelineUpdate("solvePNPEnabled", false);
-        }
+  name: 'Pipeline',
+  components: {
+    CameraAndPipelineSelect,
+    cvImage,
+    InputTab,
+    ThresholdTab,
+    ContoursTab,
+    OutputTab,
+    TargetsTab,
+    Map3DTab,
+    PnPTab,
+    AprilTagTab,
+    ArucoTab,
+  },
+  data() {
+    return {
+      selectedTabsData: [0, 0, 0, 0],
+      counterData: 0,
+      dialog: false,
+      processingModeOverride: false,
+      hideNTWarning: false,
     }
+  },
+  computed: {
+    selectedTabs: {
+      get() {
+        return this.$store.getters.isDriverMode ? [0] : this.selectedTabsData;
+      },
+      set(value) {
+        this.selectedTabsData = value;
+      }
+    },
+    tabGroups: {
+      get() {
+        let tabs = {
+          input: {
+            name: "Input",
+            component: "InputTab",
+          },
+          threshold: {
+            name: "Threshold",
+            component: "ThresholdTab",
+          },
+          contours: {
+            name: "Contours",
+            component: "ContoursTab",
+          },
+          apriltag: {
+            name: "AprilTag",
+            component: "AprilTagTab",
+          },
+          aruco: {
+            name: "Aruco",
+            component: "ArucoTab",
+          },
+          output: {
+            name: "Output",
+            component: "OutputTab",
+          },
+          targets: {
+            name: "Targets",
+            component: "TargetsTab",
+          },
+          pnp: {
+            name: "PnP",
+            component: "PnPTab",
+          },
+          map3d: {
+            name: "3D",
+            component: "Map3DTab",
+          }
+        };
+
+        // If not in 3d, name "3D" is illegal
+        const allow3d = this.$store.getters.currentPipelineSettings.solvePNPEnabled;
+        // If in apriltag, "Threshold" and "Contours" are illegal -- otherwise "AprilTag" is
+        const isAprilTag = (this.$store.getters.currentPipelineSettings.pipelineType - 2) === 2;
+        const isAruco = (this.$store.getters.currentPipelineSettings.pipelineType - 2) === 3;
+
+        // 2D array of tab names and component names; each sub-array is a separate tab group
+        let ret = [];
+        if (this.$vuetify.breakpoint.smAndDown || this.$store.getters.isDriverMode || (this.$vuetify.breakpoint.mdAndDown && !this.$store.state.compactMode)) {
+          // One big tab group with all the tabs
+          ret[0] = Object.values(tabs);
+        } else if (this.$vuetify.breakpoint.mdAndDown || !this.$store.state.compactMode) {
+          // Two tab groups, one with "input, threshold, contours, output" and the other with "target info, 3D"
+          ret[0] = [tabs.input, tabs.threshold, tabs.contours, tabs.apriltag, tabs.aruco, tabs.output];
+          ret[1] = [tabs.targets, tabs.pnp, tabs.map3d];
+        } else if (this.$vuetify.breakpoint.lgAndDown) {
+          // Three tab groups, one with "input", one with "threshold, contours, output", and the other with "target info, 3D"
+          ret[0] = [tabs.input];
+          ret[1] = [tabs.threshold, tabs.contours, tabs.apriltag,tabs.aruco, tabs.output];
+          ret[2] = [tabs.targets, tabs.pnp, tabs.map3d];
+        } else if (this.$vuetify.breakpoint.xl) {
+          // Three tab groups, one with "input", one with "threshold, contours", and the other with "output, target info, 3D"
+          ret[0] = [tabs.input];
+          ret[1] = [tabs.threshold];
+          ret[2] = [tabs.contours, tabs.apriltag, tabs.aruco,tabs.output];
+          ret[3] = [tabs.targets, tabs.pnp, tabs.map3d];
+        }
+
+        for(let i = 0; i < ret.length; i++) {
+          const group = ret[i];
+
+          // All the tabs we allow
+          const filteredGroup = group.filter(it =>
+              !(!allow3d && it.name === "3D") //Filter out 3D tab any time 3D isn't calibrated
+              && !((!allow3d || isAprilTag || isAruco) && it.name === "PnP") //Filter out the PnP config tab if 3D isn't available, or we're doing Apriltags
+              && !((isAprilTag || isAruco) && (it.name === "Threshold")) //Filter out threshold tab if we're doing apriltags
+              && !((isAprilTag || isAruco)&& (it.name === "Contours")) //Filter out contours if we're doing Apriltag
+              && !(!isAprilTag && it.name === "AprilTag") //Filter out apriltag unless we actually are doing Apriltags
+              && !(!isAruco && it.name === "Aruco")
+          );
+          ret[i] = filteredGroup;
+        }
+
+        // One last filter to remove empty lists
+        return ret.filter(it => it !== undefined && it.length > 0);
+      }
+    },
+    processingMode: {
+      get() {
+        return (this.$store.getters.currentPipelineSettings.solvePNPEnabled || this.processingModeOverride) ? 1 : 0;
+      },
+      set(value) {
+        if (this.$store.getters.isCalibrated) {
+          this.$store.getters.currentPipelineSettings.solvePNPEnabled = value === 1;
+          this.handlePipelineUpdate("solvePNPEnabled", value === 1);
+        }
+      }
+    },
+    driverMode: {
+      get() {
+        return this.$store.getters.isDriverMode;
+      },
+      set(value) {
+        this.$store.getters.currentCameraSettings.currentPipelineIndex = value ? -1 : 0;
+        this.handleInputWithIndex('currentPipeline', value ? -1 : 0);
+      }
+    },
+    selectedOutputs: {
+      // All this logic exists to deal with the reality that the output select buttons sometimes need an array and sometimes need a number (depending on whether or not they're exclusive)
+      get() {
+        // We switch the selector to single-select only on sm-and-down size devices, so we have to return a Number instead of an Array in that state
+        let ret = [];
+        if (this.$store.state.colorPicking) {
+          ret = [0]; // We want the input stream only while color picking
+        } else if (this.$store.getters.isDriverMode) {
+          ret = [1]; // We want only the output stream in driver mode
+        } else {
+          if (this.$store.getters.currentPipelineSettings.inputShouldShow) ret = ret.concat([0]);
+          if (this.$store.getters.currentPipelineSettings.outputShouldShow) ret = ret.concat([1]);
+          if (!ret.length) ret = [0];
+        }
+
+        if (this.$vuetify.breakpoint.mdAndUp) {
+          return ret;
+        } else {
+          return ret[0] || 0;
+        }
+      },
+      set(value) {
+        let valToCommit = [0];
+        if (value instanceof Array) {
+          // Value is already an array, we don't need to do anything
+          valToCommit = value;
+        } else if (value) {
+          // Value is assumed to be a number, so we wrap it into an array
+          valToCommit = [value];
+        }
+
+        this.$store.commit("mutatePipeline", {"inputShouldShow": valToCommit.includes(0)});
+        this.$store.commit("mutatePipeline", {"outputShouldShow": valToCommit.includes(1)});
+        this.handlePipelineUpdate("inputShouldShow", valToCommit.includes(0));
+      }
+    },
+    fpsTooLow: {
+      get() {
+        // For now we only show the FPS is too low warning when GPU acceleration is enabled, because we don't really trust the presented video modes otherwise
+        const currFPS = this.$store.state.pipelineResults.fps;
+        const targetFPS = this.$store.getters.currentVideoFormat.fps;
+        const driverMode = this.$store.getters.isDriverMode;
+        const gpuAccel = this.$store.state.settings.general.gpuAcceleration === true;
+        const isReflective = this.$store.getters.pipelineType === 2;
+
+        return (currFPS - targetFPS) < -5 && this.$store.state.pipelineResults.fps !== 0 && !driverMode && gpuAccel && isReflective;
+      }
+    },
+    latency: {
+      get() {
+        return this.$store.getters.currentPipelineResults.latency;
+      }
+    },
+    isCalibrated: {
+      get() {
+        const resolution = this.$store.getters.videoFormatList[this.$store.getters.currentPipelineSettings.cameraVideoModeIndex];
+        return this.$store.getters.currentCameraSettings.calibrations
+            .some(e => e.width === resolution.width && e.height === resolution.height)
+      }
+    },
+    isRobotConnected: {
+      get() {
+        // return this.$store.state.ntConnectionInfo.connected && this.$store.state.backendConnected;
+        return true;
+      }
+    },
+    showNTWarning: {
+      get() {
+        return (!this.$store.state.ntConnectionInfo.connected || this.$store.state.settings.networkSettings.runNTServer) && this.$store.state.settings.networkSettings.teamNumber > 0 && this.$store.state.backendConnected && !this.hideNTWarning;
+      }
+    },
+  },
+  created() {
+    this.$store.state.connectedCallbacks.push(this.reloadStreams)
+  },
+  methods: {
+    reloadStreams() {
+      // Reload the streams as we technically close and reopen them
+      this.$refs.streams.forEach(it => it.reload())
+    },
+    onImageClick(event) {
+      // Get a reference to the threshold tab (if it is shown) and call its "onClick" method
+      let ref = this.$refs["Threshold"];
+      if (ref && ref[0])
+        ref[0].onClick(event)
+    },
+    on3DClick() {
+      if (!this.$store.getters.isCalibrated) {
+        this.dialog = true;
+        this.processingModeOverride = true;
+      }
+    },
+    closeUncalibratedDialog() {
+      this.dialog = false;
+      this.processingModeOverride = false;
+      // this.$store.getters.currentPipelineSettings.solvePNPEnabled = false;
+      this.handlePipelineUpdate("solvePNPEnabled", false);
+    }
+  }
 }
 </script>
 
 <style scoped>
 .v-btn-toggle.fill {
-    width: 100%;
-    height: 100%;
+  width: 100%;
+  height: 100%;
 }
 
 .v-btn-toggle.fill > .v-btn {
-    width: 50%;
-    height: 100%;
+  width: 50%;
+  height: 100%;
 }
 
 th {
-    width: 80px;
-    text-align: center;
+  width: 80px;
+  text-align: center;
 }
 </style>

photon-client/src/views/PipelineViews/AprilTagTab.vue

@@ -0,0 +1,154 @@
+<template>
+  <div>
+    <CVselect
+        v-model="selectedFamily"
+        name="Target family"
+        :list="['AprilTag family 36h11', 'AprilTag family 25h9', 'AprilTag family 16h5']"
+        select-cols="8"
+        @input="handlePipelineUpdate('tagFamily', selectedFamily)"
+    />
+    <CVslider
+        v-model="decimate"
+        class="pt-2"
+        slider-cols="8"
+        name="Decimate"
+        min="1"
+        max="8"
+        step="1.0"
+        tooltip="Increases FPS at the expense of range by reducing image resolution initially"
+        @input="handlePipelineData('decimate')"
+    />
+    <CVslider
+        v-model="blur"
+        class="pt-2"
+        slider-cols="8"
+        name="Blur"
+        min="0"
+        max="5"
+        step=".01"
+        tooltip="Gaussian blur added to the image, high FPS cost for slightly decreased noise"
+        @input="handlePipelineData('blur')"
+    />
+    <CVslider
+        v-model="threads"
+        class="pt-2"
+        slider-cols="8"
+        name="Threads"
+        min="1"
+        max="8"
+        step="1"
+        tooltip="Number of threads spawned by the AprilTag detector"
+        @input="handlePipelineData('threads')"
+    />
+    <CVswitch
+        v-model="refineEdges"
+        class="pt-2"
+        slider-cols="8"
+        name="Refine Edges"
+        tooltip="Further refines the apriltag corner position initial estimate, suggested left on"
+        @input="handlePipelineData('refineEdges')"
+    />
+    <CVslider
+        v-model="decisionMargin"
+        class="pt-2 pb-4"
+        slider-cols="8"
+        name="Decision Margin Cutoff"
+        min="0"
+        max="250"
+        step="1"
+        tooltip="Tags with a 'margin' (decoding quality score) less than this wil be rejected. Increase this to reduce the number of false positive detections"
+        @input="handlePipelineData('decisionMargin')"
+    />
+    <CVslider
+        v-model="numIterations"
+        class="pt-2 pb-4"
+        slider-cols="8"
+        name="Pose Estimation Iterations"
+        min="0"
+        max="500"
+        step="1"
+        tooltip="Number of iterations the pose estimation algorithm will run, 50-100 is a good starting point"
+        @input="handlePipelineData('numIterations')"
+    />
+  </div>
+</template>
+
+<script>
+import CVslider from '../../components/common/cv-slider'
+import CVswitch from '../../components/common/cv-switch'
+import CVselect from '../../components/common/cv-select'
+
+export default {
+  name: "AprilTag",
+  components: {
+    CVslider,
+    CVswitch,
+    CVselect,
+  },
+  data() {
+    return {
+      familyList: ["AprilTag family 36h11", "AprilTag family 25h9", "AprilTag family 16h5"],
+    }
+  },
+  computed: {
+    selectedFamily: {
+      get() {
+        return this.$store.getters.currentPipelineSettings.tagFamily
+      },
+      set(val) {
+        this.$store.commit("mutatePipeline", {"tagFamily": val})
+      }
+    },
+    decimate: {
+      get() {
+        return this.$store.getters.currentPipelineSettings.decimate
+      },
+      set(val) {
+        this.$store.commit("mutatePipeline", {"decimate": val});
+      }
+    },
+    decisionMargin: {
+      get() {
+        return this.$store.getters.currentPipelineSettings.decisionMargin
+      },
+      set(val) {
+        this.$store.commit("mutatePipeline", {"decisionMargin": val});
+      }
+    },
+    numIterations: {
+      get() {
+        return this.$store.getters.currentPipelineSettings.numIterations
+      },
+      set(val) {
+        this.$store.commit("mutatePipeline", {"numIterations": val});
+      }
+    },
+    blur: {
+      get() {
+        return this.$store.getters.currentPipelineSettings.blur
+      },
+      set(val) {
+        this.$store.commit("mutatePipeline", {"blur": val});
+      }
+    },
+    threads: {
+      get() {
+        return this.$store.getters.currentPipelineSettings.threads
+      },
+      set(val) {
+        this.$store.commit("mutatePipeline", {"threads": val});
+      }
+    },
+    refineEdges: {
+      get() {
+        return this.$store.getters.currentPipelineSettings.refineEdges
+      },
+      set(val) {
+        this.$store.commit("mutatePipeline", {"refineEdges": val});
+      }
+    },
+  },
+  methods: {
+  }
+}
+</script>

photon-client/src/views/PipelineViews/ArucoTab.vue

@@ -0,0 +1,76 @@
+<template>
+  <div>
+    <CVslider
+        v-model="decimate"
+        class="pt-2"
+        slider-cols="8"
+        name="Decimate"
+        min="1"
+        max="8"
+        step=".5"
+        tooltip="Increases FPS at the expense of range by reducing image resolution initially"
+        @input="handlePipelineData('decimate')"
+    />
+    <CVslider
+        v-model="numIterations"
+        class="pt-2"
+        slider-cols="8"
+        name="Corner Iterations"
+        min="30"
+        max="1000"
+        step="5"
+        tooltip="How many iterations are going to be used in order to refine corners. Higher values are lead to more accuracy at the cost of performance"
+        @input="handlePipelineData('numIterations')"
+    />
+    <CVslider
+        v-model="cornerAccuracy"
+        class="pt-2"
+        slider-cols="8"
+        name="Corner Accuracy"
+        min=".01"
+        max="100"
+        step=".01"
+        tooltip="Minimum accuracy for the corners, lower is better but more performance intensive "
+        @input="handlePipelineData('cornerAccuracy')"
+    />
+  </div>
+</template>
+
+<script>
+import CVslider from '../../components/common/cv-slider'
+
+export default {
+  name: "Aruco",
+  components: {
+    CVslider
+  },
+  computed: {
+    decimate: {
+      get() {
+        return this.$store.getters.currentPipelineSettings.decimate
+      },
+      set(val) {
+        this.$store.commit("mutatePipeline", {"decimate": val});
+      },
+    },
+      numIterations: {
+        get() {
+          return this.$store.getters.currentPipelineSettings.numIterations
+        },
+        set(val) {
+          this.$store.commit("mutatePipeline", {"numIterations": val});
+        },
+      },
+        cornerAccuracy: {
+          get() {
+            return this.$store.getters.currentPipelineSettings.cornerAccuracy
+          },
+          set(val) {
+            this.$store.commit("mutatePipeline", {"cornerAccuracy": val});
+          },
+        },
+    },
+    methods: {
+    }
+  }
+</script>

photon-client/src/views/PipelineViews/ContoursTab.vue

@@ -18,6 +18,14 @@
       step="0.1"
       @input="handlePipelineData('contourRatio')"
     />
+    <CVselect
+      v-model="contourTargetOrientation"
+      name="Target Orientation"
+      tooltip="Used to determine how to calculate target landmarks, as well as aspect ratio"
+      :list="['Portrait', 'Landscape']"
+      @input="handlePipelineData('contourTargetOrientation')"
+      @rollback="e=> rollback('contourTargetOrientation', e)"
+    />
     <CVrangeSlider
       v-if="currentPipelineType() !== 3"
       v-model="contourFullness"
@@ -203,6 +211,14 @@ export default {
         this.$store.commit("mutatePipeline", {"contourRatio": val});
       }
     },
+    contourTargetOrientation: {
+      get() {
+        return this.$store.getters.currentPipelineSettings.contourTargetOrientation
+      },
+      set(val) {
+        this.$store.commit("mutatePipeline", {"contourTargetOrientation": val});
+      }
+    },
     contourFullness: {
       get() {
         return this.$store.getters.currentPipelineSettings.contourFullness

photon-client/src/views/PipelineViews/InputTab.vue

@@ -2,11 +2,12 @@
   <div>
     <CVslider
       v-model="cameraExposure"
+      :disabled="cameraAutoExposure"
       name="Exposure"
       min="0"
       max="100"
       step="0.1"
-      tooltip="Directly controls how much light is allowed to fall onto the sensor, which affects brightness"
+      tooltip="Directly controls how much light is allowed to fall onto the sensor, which affects apparent brightness"
       :slider-cols="largeBox"
       @input="handlePipelineData('cameraExposure')"
       @rollback="e => rollback('cameraExposure', e)"
@@ -21,10 +22,28 @@
       @input="handlePipelineData('cameraBrightness')"
       @rollback="e => rollback('cameraBrightness', e)"
     />
+    <CVswitch
+      v-model="cameraAutoExposure"
+      class="pt-2"
+      name="Auto Exposure"
+      tooltip="Enables or Disables camera automatic adjustment for current lighting conditions"
+      @input="handlePipelineData('cameraAutoExposure')"
+    />
+    <CVslider
+      v-if="cameraGain >= 0"
+      v-model="cameraGain"
+      name="Camera Gain"
+      min="0"
+      max="100"
+      tooltip="Controls camera gain, similar to brightness"
+      :slider-cols="largeBox"
+      @input="handlePipelineData('cameraGain')"
+      @rollback="e => rollback('cameraGain', e)"
+    />
     <CVslider
       v-if="cameraRedGain !== -1"
       v-model="cameraRedGain"
-      name="Red AWB Gain"
+      name="Red Balance"
       min="0"
       max="100"
       tooltip="Controls red automatic white balance gain, which affects how the camera captures colors in different conditions"
@@ -35,7 +54,7 @@
     <CVslider
       v-if="cameraBlueGain !== -1"
       v-model="cameraBlueGain"
-      name="Blue AWB Gain"
+      name="Blue Balance"
       min="0"
       max="100"
       tooltip="Controls blue automatic white balance gain, which affects how the camera captures colors in different conditions"
@@ -43,11 +62,13 @@
       @input="handlePipelineData('cameraBlueGain')"
       @rollback="e => rollback('cameraBlueGain', e)"
     />
+    <!-- TODO: stop filtering out the 90 degree rotation modes when we fix those in libcamera -->
     <CVselect
       v-model="inputImageRotationMode"
       name="Orientation"
       tooltip="Rotates the camera stream"
       :list="['Normal','90° CW','180°','90° CCW']"
+      :filtered-indices="this.$store.state.settings.general.gpuAcceleration ? new Set([1, 3]) : undefined"
       :select-cols="largeBox"
       @input="handlePipelineData('inputImageRotationMode')"
       @rollback="e => rollback('inputImageRotationMode',e)"
@@ -75,6 +96,7 @@
 <script>
     import CVslider from '../../components/common/cv-slider'
     import CVselect from '../../components/common/cv-select'
+    import CVswitch from '../../components/common/cv-switch'
 
     const unfilteredStreamDivisors = [1, 2, 4, 6];
 
@@ -83,14 +105,10 @@
         components: {
             CVslider,
             CVselect,
+            CVswitch,
         },
         // eslint-disable-next-line vue/require-prop-types
         props: ['value'],
-        data() {
-            return {
-              rawStreamDivisorIndex: 0,
-            }
-        },
         computed: {
             largeBox: {
               get() {
@@ -108,6 +126,14 @@
                     this.$store.commit("mutatePipeline", {"cameraExposure": parseFloat(val)});
                 }
             },
+            cameraAutoExposure: {
+                get() {
+                    return this.$store.getters.currentPipelineSettings.cameraAutoExposure;
+                },
+                set(val) {
+                    this.$store.commit("mutatePipeline", {"cameraAutoExposure": val});
+                }
+            },
             cameraBrightness: {
                 get() {
                     return parseInt(this.$store.getters.currentPipelineSettings.cameraBrightness)
@@ -116,6 +142,14 @@
                     this.$store.commit("mutatePipeline", {"cameraBrightness": parseInt(val)});
                 }
             },
+            cameraGain: {
+                get() {
+                    return parseInt(this.$store.getters.currentPipelineSettings.cameraGain)
+                },
+                set(val) {
+                    this.$store.commit("mutatePipeline", {"cameraGain": parseInt(val)});
+                }
+            },
             cameraRedGain: {
                 get() {
                     return parseInt(this.$store.getters.currentPipelineSettings.cameraRedGain)
@@ -148,15 +182,22 @@
                     this.$store.commit("mutatePipeline", {"cameraVideoModeIndex": val});
 
                     this.handlePipelineUpdate("streamingFrameDivisor", this.getNumSkippedStreamDivisors());
-                    this.rawStreamDivisorIndex = 0;
+                    this.$store.commit("mutatePipeline", {"streamingFrameDivisor": 0});
+
+                    // If we don't have 3d mode calibrated at the new resolution either, we should disable it here
+                    // (TODO) This probably belongs in the backend (Matt)
+                    if (!this.$store.getters.isCalibrated) {
+                        this.handlePipelineUpdate("solvePNPEnabled", false);
+                        this.$store.commit("mutatePipeline", {"solvePNPEnabled": false});
+                    }
                 }
             },
             streamingFrameDivisor: {
                 get() {
-                    return this.rawStreamDivisorIndex;
+                    return this.$store.getters.currentPipelineSettings.streamingFrameDivisor;
                 },
                 set(val) {
-                    this.rawStreamDivisorIndex = val;
+                    this.$store.commit("mutatePipeline", {"streamingFrameDivisor": val});
                     this.handlePipelineUpdate("streamingFrameDivisor", this.getNumSkippedStreamDivisors() + val);
                 }
             },

photon-client/src/views/PipelineViews/Map3DTab.vue

@@ -0,0 +1,53 @@
+<template>
+  <div>
+    <mini-map
+      class="miniMapClass"
+      :targets="targets"
+      :horizontal-f-o-v="horizontalFOV"
+    />
+  </div>
+</template>
+
+<script>
+    import miniMap from '../../components/pipeline/3D/MiniMap';
+
+    export default {
+        name: "Map3D",
+        components: {
+            miniMap
+        },
+        data() {
+            return {
+            }
+        },
+        computed: {
+            targets: {
+                get() {
+                    return this.$store.getters.currentPipelineResults.targets;
+                }
+            },
+            horizontalFOV: {
+                get() {
+                    let index = this.$store.getters.currentPipelineSettings.cameraVideoModeIndex;
+                    let FOV = this.$store.getters.currentCameraSettings.fov;
+                    let resolution = this.$store.getters.videoFormatList[index];
+                    let diagonalView = FOV * (Math.PI / 180);
+                    let diagonalAspect = Math.hypot(resolution.width, resolution.height);
+                    return Math.atan(Math.tan(diagonalView / 2) * (resolution.width / diagonalAspect)) * 2 * (180 / Math.PI)
+                }
+            },
+        },
+        methods: {
+        }
+    }
+</script>
+
+<style scoped>
+    .miniMapClass {
+        width: 400px !important;
+        height: 100% !important;
+
+        margin-left: auto;
+        margin-right: auto;
+    }
+</style>

photon-client/src/views/PipelineViews/OutputTab.vue

@@ -10,6 +10,7 @@
     />
 
     <CVselect
+      v-if="!isTagPipeline"
       v-model="contourTargetOrientation"
       name="Target Orientation"
       tooltip="Used to determine how to calculate target landmarks (e.g. the top, left, or bottom of the target)"
@@ -21,7 +22,8 @@
     <CVswitch
       v-model="outputShowMultipleTargets"
       name="Show Multiple Targets"
-      tooltip="If enabled, up to five targets will be displayed and sent to user code"
+      tooltip="If enabled, up to five targets will be displayed and sent to user code, instead of just one"
+      :disabled="isTagPipeline"
       class="mb-4"
       text-cols="3"
       @input="handlePipelineData('outputShowMultipleTargets')"
@@ -137,6 +139,11 @@
                 get() {
                     return undefined; // TODO fix
                 }
+            },
+            isTagPipeline: {
+                get() {
+                     return this.$store.getters.currentPipelineSettings.pipelineType > 3;
+                }
             }
         },
         methods: {

photon-client/src/views/PipelineViews/PnPTab.vue

@@ -6,7 +6,6 @@
       type="file"
       accept=".csv"
       style="display: none;"
-
       @change="readFile"
     >
 
@@ -32,11 +31,7 @@
       @input="handlePipelineData('cornerDetectionAccuracyPercentage')"
       @rollback="e => rollback('cornerDetectionAccuracyPercentage', e)"
     />
-    <mini-map
-      class="miniMapClass"
-      :targets="targets"
-      :horizontal-f-o-v="horizontalFOV"
-    />
+
     <v-snackbar
       v-model="snack"
       top
@@ -49,18 +44,16 @@
 
 <script>
     import Papa from 'papaparse';
-    import miniMap from '../../components/pipeline/3D/MiniMap';
     import CVslider from '../../components/common/cv-slider'
 
     export default {
         name: "PnP",
         components: {
-            CVslider,
-            miniMap
+            CVslider
         },
         data() {
             return {
-                targetList: ['2020 High Goal Outer', '2020 High Goal Inner', '2019 Dual Target', '2020 Power Cell (7in)','2022 Cargo Ball (9.5in)', '2016 High Goal'], //Keep in sync with TargetModel.java
+                targetList: ['2020 High Goal Outer', '2020 High Goal Inner', '2019 Dual Target', '2020 Power Cell (7in)','2022 Cargo Ball (9.5in)', '2016 High Goal', '6.5in (36h11) AprilTag', '6in (16h5) AprilTag'], //Keep in sync with TargetModel.java
                 snackbar: {
                     color: "Success",
                     text: ""
@@ -72,7 +65,6 @@
             selectedModel: {
                 get() {
                     let ret = this.$store.getters.currentPipelineSettings.targetModel
-                    console.log(ret)
                     return this.targetList[ret];
                 },
                 set(val) {
@@ -87,21 +79,6 @@
                     this.$store.commit("mutatePipeline", {"cornerDetectionAccuracyPercentage": val});
                 }
             },
-            targets: {
-                get() {
-                    return this.$store.getters.currentPipelineResults.targets;
-                }
-            },
-            horizontalFOV: {
-                get() {
-                    let index = this.$store.getters.currentPipelineSettings.cameraVideoModeIndex;
-                    let FOV = this.$store.getters.currentCameraSettings.fov;
-                    let resolution = this.$store.getters.videoFormatList[index];
-                    let diagonalView = FOV * (Math.PI / 180);
-                    let diagonalAspect = Math.hypot(resolution.width, resolution.height);
-                    return Math.atan(Math.tan(diagonalView / 2) * (resolution.width / diagonalAspect)) * 2 * (180 / Math.PI)
-                }
-            },
         },
         methods: {
             readFile(event) {

photon-client/src/views/PipelineViews/TargetsTab.vue

@@ -18,29 +18,40 @@
               <th class="text-center">
                 Target
               </th>
+              <th
+                  v-if="$store.getters.pipelineType === 4 || (($store.getters.pipelineType - 2) === 3)"
+                class="text-center"
+              >
+                Fiducial ID
+              </th>
               <template v-if="!$store.getters.currentPipelineSettings.solvePNPEnabled">
                 <th class="text-center">
-                  Pitch
+                  Pitch,&nbsp;&deg;
                 </th>
                 <th class="text-center">
-                  Yaw
+                  Yaw,&nbsp;&deg;
                 </th>
                 <th class="text-center">
-                  Skew
+                  Skew,&nbsp;&deg;
+                </th>
+                <th class="text-center">
+                  Area, %
                 </th>
               </template>
-              <th class="text-center">
-                Area
-              </th>
-              <template v-if="$store.getters.currentPipelineSettings.solvePNPEnabled">
+              <template v-else>
                 <th class="text-center">
-                  X
+                  X,&nbsp;m
                 </th>
                 <th class="text-center">
-                  Y
+                  Y,&nbsp;m
                 </th>
                 <th class="text-center">
-                  Angle
+                  Z Angle,&nbsp;&deg;
+                </th>
+              </template>
+              <template v-if="$store.getters.pipelineType === 4 && $store.getters.currentPipelineSettings.solvePNPEnabled">
+                <th class="text-center">
+                  Ambiguity
                 </th>
               </template>
             </tr>
@@ -51,17 +62,29 @@
               :key="index"
             >
               <td>{{ index }}</td>
+              <td v-if="$store.getters.pipelineType === 4 || (($store.getters.pipelineType - 2) === 3)">
+                {{ parseInt(value.fiducialId) }}
+              </td>
               <template v-if="!$store.getters.currentPipelineSettings.solvePNPEnabled">
                 <td>{{ parseFloat(value.pitch).toFixed(2) }}</td>
                 <td>{{ parseFloat(value.yaw).toFixed(2) }}</td>
                 <td>{{ parseFloat(value.skew).toFixed(2) }}</td>
+                <td>{{ parseFloat(value.area).toFixed(2) }}</td>
               </template>
-              <td>{{ parseFloat(value.area).toFixed(2) }}</td>
-              <template v-if="$store.getters.currentPipelineSettings.solvePNPEnabled">
-                <!-- TODO: Make sure that units are correct -->
+              <template v-else-if="$store.getters.currentPipelineSettings.solvePNPEnabled && $store.getters.pipelineType === 4">
                 <td>{{ parseFloat(value.pose.x).toFixed(2) }}&nbsp;m</td>
                 <td>{{ parseFloat(value.pose.y).toFixed(2) }}&nbsp;m</td>
-                <td>{{ parseFloat(value.pose.rot).toFixed(2) }}&deg;</td>
+                <td>{{ (parseFloat(value.pose.angle_z) * 180 / Math.PI).toFixed(2) }}&deg;</td>
+              </template>
+              <template v-else-if="$store.getters.currentPipelineSettings.solvePNPEnabled">
+                <td>{{ parseFloat(value.pose.x).toFixed(2) }}&nbsp;m</td>
+                <td>{{ parseFloat(value.pose.y).toFixed(2) }}&nbsp;m</td>
+                <td>{{ (parseFloat(value.pose.angle_z) * 180 / Math.PI).toFixed(2) }}&deg;</td>
+              </template>
+              <template v-if="$store.getters.pipelineType === 4 && $store.getters.currentPipelineSettings.solvePNPEnabled">
+                <td>
+                  {{ parseFloat(value.ambiguity).toFixed(2) }}
+                </td>
               </template>
             </tr>
           </tbody>

photon-client/src/views/PipelineViews/ThresholdTab.vue

@@ -247,7 +247,7 @@ export default {
             'cameraIndex': this.$store.state.currentCameraIndex
           }
         });
-        this.$socket.send(msg);
+        this.$store.state.websocket.ws.send(msg);
         this.$emit('update');
       }
     },

photon-client/src/views/SettingsView.vue

@@ -37,7 +37,8 @@
     import Networking from './SettingsViews/Networking'
     import Lighting from "./SettingsViews/Lighting";
     import cvImage from '../components/common/cv-image'
-    import General from "./SettingsViews/General";
+    import Stats from "./SettingsViews/Stats";
+    import DeviceControl from "./SettingsViews/DeviceControl";
 
     export default {
         name: 'SettingsTab',
@@ -69,7 +70,7 @@
             },
             tabList: {
                 get() {
-                    return [General, Networking].concat(this.$store.state.settings.lighting.supported ? Lighting : []);
+                    return [Stats, DeviceControl, Networking].concat(this.$store.state.settings.lighting.supported ? Lighting : []);
                 }
             }
         },

photon-client/src/views/SettingsViews/DeviceControl.vue

@@ -0,0 +1,289 @@
+<template>
+  <div>
+    <v-row>
+      <v-col cols="12" lg="4" md="6">
+        <v-btn color="red" @click="restartProgram()">
+          <v-icon left>
+            mdi-restart
+          </v-icon>
+          Restart PhotonVision
+        </v-btn>
+      </v-col>
+      <v-col cols="12" lg="4" md="6">
+        <v-btn color="red" @click="restartDevice()">
+          <v-icon left>
+            mdi-restart-alert
+          </v-icon>
+          Restart Device
+        </v-btn>
+      </v-col>
+      <v-col cols="12" lg="4">
+        <v-btn color="secondary" @click="$refs.offlineUpdate.click()">
+          <v-icon left>
+            mdi-update
+          </v-icon>
+          Offline Update
+        </v-btn>
+      </v-col>
+    </v-row>
+    <v-divider />
+    <v-row>
+      <v-col cols="12" sm="6">
+        <v-btn color="secondary" @click="$refs.exportSettings.click()">
+          <v-icon left>
+            mdi-download
+          </v-icon>
+          Export Settings
+        </v-btn>
+      </v-col>
+      <v-col cols="12" sm="6">
+        <v-btn color="secondary" @click="$refs.importSettings.click()">
+          <v-icon left>
+            mdi-upload
+          </v-icon>
+          Import Settings
+        </v-btn>
+      </v-col>
+
+      <v-col cols="12" sm="6">
+        <v-btn color="secondary" @click="$refs.exportLogFile.click()">
+          <v-icon left>
+            mdi-file
+          </v-icon>
+          Export current log
+
+          <!-- Special hidden link that gets 'clicked' when the user exports journalctl logs -->
+          <a
+            ref="exportLogFile"
+            style="color: black; text-decoration: none; display: none"
+            :href="
+              'http://' +
+                this.$address +
+                '/api/settings/photonvision-journalctl.txt'
+            "
+            download="photonvision-journalctl.txt"
+          />
+        </v-btn>
+      </v-col>
+
+      <v-col cols="12" sm="6">
+        <v-btn color="secondary" @click="showLogs()">
+          <v-icon left>
+            mdi-bug
+          </v-icon>
+          Show log viewer
+        </v-btn>
+      </v-col>
+    </v-row>
+    <v-snackbar v-model="snack" top :color="snackbar.color" timeout="-1">
+      <span>{{ snackbar.text }}</span>
+    </v-snackbar>
+
+    <!-- Special hidden upload input that gets 'clicked' when the user imports settings -->
+    <input
+      ref="importSettings"
+      type="file"
+      accept=".zip, .json"
+      style="display: none;"
+      @change="readImportedSettings"
+    />
+    <!-- Special hidden link that gets 'clicked' when the user exports settings -->
+    <a
+      ref="exportSettings"
+      style="color: black; text-decoration: none; display: none"
+      :href="
+        'http://' + this.$address + '/api/settings/photonvision_config.zip'
+      "
+      download="photonvision-settings.zip"
+    />
+
+    <!-- Special hidden new jar upload input that gets 'clicked' when the user posts a new .jar -->
+    <input
+      ref="offlineUpdate"
+      type="file"
+      accept=".jar"
+      style="display: none;"
+      @change="doOfflineUpdate"
+    />
+  </div>
+</template>
+
+<script>
+export default {
+  name: "Device Control",
+  data() {
+    return {
+      snack: false,
+      uploadPercentage: 0.0,
+      snackbar: {
+        color: "success",
+        text: "",
+      },
+    };
+  },
+  computed: {
+    settings() {
+      return this.$store.state.settings.general;
+    },
+    version() {
+      return `${this.settings.version}`;
+    },
+    hwModel() {
+      if (this.settings.hardwareModel !== "") {
+        return `${this.settings.hardwareModel}`;
+      } else {
+        return `Unknown`;
+      }
+    },
+    platform() {
+      return `${this.settings.hardwarePlatform}`;
+    },
+    gpuAccel() {
+      return `${this.settings.gpuAcceleration ? "Enabled" : "Unsupported"} ${
+        this.settings.gpuAcceleration
+          ? "(" + this.settings.gpuAcceleration + ")"
+          : ""
+      }`;
+    },
+    metrics() {
+      // console.log(this.$store.state.metrics);
+      return this.$store.state.metrics;
+    },
+  },
+  methods: {
+    restartProgram() {
+      this.axios.post("http://" + this.$address + "/api/restartProgram", {});
+    },
+    restartDevice() {
+      this.axios.post("http://" + this.$address + "/api/restartDevice", {});
+    },
+    readImportedSettings(event) {
+      let formData = new FormData();
+      formData.append("zipData", event.target.files[0]);
+      this.axios
+        .post("http://" + this.$address + "/api/settings/import", formData, {
+          headers: { "Content-Type": "multipart/form-data" },
+        })
+        .then(() => {
+          this.snackbar = {
+            color: "success",
+            text:
+              "Settings imported successfully! PhotonVision will restart in the background...",
+          };
+          this.snack = true;
+        })
+        .catch((err) => {
+          if (err.response) {
+            this.snackbar = {
+              color: "error",
+              text:
+                "Error while uploading settings file! Could not process provided file.",
+            };
+          } else if (err.request) {
+            this.snackbar = {
+              color: "error",
+              text:
+                "Error while uploading settings file! No respond to upload attempt.",
+            };
+          } else {
+            this.snackbar = {
+              color: "error",
+              text: "Error while uploading settings file!",
+            };
+          }
+          this.snack = true;
+        });
+    },
+    doOfflineUpdate(event) {
+      this.snackbar = {
+        color: "secondary",
+        text: "New Software Upload in Process...",
+      };
+      this.snack = true;
+
+      let formData = new FormData();
+      formData.append("jarData", event.target.files[0]);
+      this.axios
+        .post(
+          "http://" + this.$address + "/api/settings/offlineUpdate",
+          formData,
+          {
+            headers: { "Content-Type": "multipart/form-data" },
+            onUploadProgress: function(progressEvent) {
+              this.uploadPercentage = parseInt(
+                Math.round((progressEvent.loaded / progressEvent.total) * 100)
+              );
+              if (this.uploadPercentage < 99.5) {
+                this.snackbar.text =
+                  "New Software Upload in Process, " +
+                  this.uploadPercentage +
+                  "% complete";
+              } else {
+                this.snackbar.text = "Installing uploaded software...";
+              }
+            }.bind(this),
+          }
+        )
+        .then(() => {
+          this.snackbar = {
+            color: "success",
+            text:
+              "New .jar copied successfully! PhotonVision will restart in the background...",
+          };
+          this.snack = true;
+        })
+        .catch((err) => {
+          if (err.response) {
+            this.snackbar = {
+              color: "error",
+              text:
+                "Error while uploading new .jar file! Could not process provided file.",
+            };
+          } else if (err.request) {
+            this.snackbar = {
+              color: "error",
+              text:
+                "Error while uploading new .jar file! No respond to upload attempt.",
+            };
+          } else {
+            this.snackbar = {
+              color: "error",
+              text: "Error while uploading new .jar file!",
+            };
+          }
+          this.snack = true;
+        });
+    },
+    showLogs(event) {
+      event;
+      this.$store.state.logsOverlay = true;
+    },
+  },
+};
+</script>
+
+<style lang="css" scoped>
+.v-btn {
+  width: 100%;
+}
+
+.infoTable {
+  border: 1px solid;
+  border-collapse: separate;
+  border-spacing: 0px;
+  border-radius: 5px;
+  text-align: left;
+  margin-bottom: 10px;
+  width: 100%;
+  display: block;
+  overflow-x: auto;
+}
+
+.infoElem {
+  padding-right: 15px;
+  padding-bottom: 1px;
+  padding-top: 1px;
+  padding-left: 10px;
+  border-right: 1px solid;
+}
+</style>

photon-client/src/views/SettingsViews/Networking.vue

@@ -21,7 +21,7 @@
         Team number is unset or invalid. NetworkTables will not be able to connect.
       </v-banner>
       <CVradio
-        v-show="$store.state.settings.networkSettings.supported"
+        v-show="$store.state.settings.networkSettings.shouldManage"
         v-model="connectionType"
         :input-cols="inputCols"
         name="IP Assignment Mode"
@@ -66,7 +66,42 @@
     >
       Save
     </v-btn>
-    <v-divider class="mt-4 mb-4" />
+    <v-snackbar
+      v-model="snack"
+      top
+      :color="snackbar.color"
+      timeout="5000"
+    >
+      <span>{{ snackbar.text }}</span>
+    </v-snackbar>
+
+    <template v-if="$store.state.settings.networkSettings.shouldManage && false">
+
+      <!-- Advanced controls for changing DHCP settings and stuff -->
+      <v-divider class="mt-4 mb-4" />
+
+      <v-title> Advanced </v-title>
+
+      <CVinput
+        :input-cols="inputCols"
+        name="Set DHCP command"
+      />
+      <CVinput
+        :input-cols="inputCols"
+        name="Set static command"
+      />
+      <CVinput
+        :input-cols="inputCols"
+        name="NetworkManager interface"
+      />
+      <CVinput
+        :input-cols="inputCols"
+        name="Physical interface"
+      />
+
+    </template>
+
+    <!-- TEMP - RIO finder is not currently enabled
     <v-row>
       <v-col
         cols="12"
@@ -125,6 +160,7 @@
         </v-simple-table>
       </v-col>
     </v-row>
+    -->
   </div>
 </template>
 
@@ -236,8 +272,16 @@ export default {
             return true;
         },
         sendGeneralSettings() {
+            const changingStaticIp = !this.isDHCP;
+
+            this.snackbar = {
+                color: "secondary",
+                text: "Updating settings..."
+            };
+            this.snack = true;
+
             this.axios.post("http://" + this.$address + "/api/settings/general", this.settings).then(
-                function (response) {
+                response => {
                     if (response.status === 200) {
                         this.snackbar = {
                             color: "success",
@@ -246,11 +290,18 @@ export default {
                         this.snack = true;
                     }
                 },
-                function (error) {
+                error => {
+                  if (error.status === 504 || changingStaticIp) {
+                    this.snackbar = {
+                        color: "error",
+                        text: (error.response || {data: `Connection lost! Try the new static IP at ${this.staticIp}:5800 or ${this.hostname}:5800 ?`}).data
+                    };
+                  } else {
                     this.snackbar = {
                         color: "error",
                         text: (error.response || {data: "Couldn't save settings"}).data
                     };
+                  }
                     this.snack = true;
                 }
             )

photon-client/src/views/SettingsViews/Stats.vue

@@ -49,22 +49,46 @@
           <th class="infoElem">
             Disk Usage
           </th>
+          <th class="infoElem">
+            <v-tooltip top>
+              <template v-slot:activator="{ on, attrs }">
+                <span
+                  v-bind="attrs"
+                  v-on="on"
+                >
+                ⓘ CPU Throttling
+                </span>
+              </template>
+              <span>
+                    Current or Previous Reason for the cpu being held back from maximum performance.
+              </span>
+            </v-tooltip>
+          </th>
+          <th class="infoElem">
+            CPU Uptime
+          </th>
         </tr>
         <tr v-if="metrics.cpuUtil !== 'N/A'">
           <td class="infoElem">
-            {{ metrics.cpuUtil.replace(" ", "") }}%
+            {{ metrics.cpuUtil }}%
           </td>
           <td class="infoElem">
             {{ parseInt(metrics.cpuTemp) }}&deg;&nbsp;C
           </td>
           <td class="infoElem">
-            {{ metrics.ramUtil.replace(" ", "") }}MB of {{ metrics.cpuMem }}MB
+            {{ metrics.ramUtil }}MB of {{ metrics.cpuMem }}MB
           </td>
           <td class="infoElem">
-            {{ metrics.gpuMemUtil.replace(" ", "") }}MB of {{ metrics.gpuMem }}MB
+            {{ metrics.gpuMemUtil }}MB of {{ metrics.gpuMem }}MB
           </td>
           <td class="infoElem">
-            {{ metrics.diskUtilPct.replace(" ", "") }}
+            {{ metrics.diskUtilPct }}
+          </td>
+          <td class="infoElem">
+            {{ metrics.cpuThr }}
+          </td>
+          <td class="infoElem">
+            {{ metrics.cpuUptime }}
           </td>
         </tr>
         <tr v-if="metrics.cpuUtil === 'N/A'">
@@ -83,89 +107,21 @@
           <td class="infoElem">
             ---
           </td>
+          <td class="infoElem">
+            ---
+          </td>
+          <td class="infoElem">
+            ---
+          </td>
         </tr>
       </table>
     </v-row>
 
-    <v-row>
-      <v-col
-        cols="12"
-        sm="6"
-        md="4"
-      >
-        <v-btn
-          color="secondary"
-          @click="$refs.exportSettings.click()"
-        >
-          <v-icon left>
-            mdi-download
-          </v-icon>
-          Export Settings
-        </v-btn>
-      </v-col>
-      <v-col
-        cols="12"
-        sm="6"
-        md="4"
-      >
-        <v-btn
-          color="secondary"
-          @click="$refs.importSettings.click()"
-        >
-          <v-icon left>
-            mdi-upload
-          </v-icon>
-          Import Settings
-        </v-btn>
-      </v-col>
-      <v-col
-        cols="12"
-        md="4"
-      >
-        <v-btn
-          color="secondary"
-          @click="$refs.offlineUpdate.click()"
-        >
-          <v-icon left>
-            mdi-update
-          </v-icon>
-          Offline Update
-        </v-btn>
-      </v-col>
-      <v-col
-        cols="12"
-        lg="6"
-      >
-        <v-btn
-          color="red"
-          @click="restartProgram()"
-        >
-          <v-icon left>
-            mdi-restart
-          </v-icon>
-          Restart PhotonVision
-        </v-btn>
-      </v-col>
-      <v-col
-        cols="12"
-        lg="6"
-      >
-        <v-btn
-          color="red"
-          @click="restartDevice()"
-        >
-          <v-icon left>
-            mdi-restart-alert
-          </v-icon>
-          Restart Device
-        </v-btn>
-      </v-col>
-    </v-row>
     <v-snackbar
       v-model="snack"
       top
       :color="snackbar.color"
-      timeout="0"
+      timeout="-1"
     >
       <span>{{ snackbar.text }}</span>
     </v-snackbar>
@@ -200,7 +156,7 @@
 
 <script>
 export default {
-    name: 'General',
+    name: 'Stats',
     data() {
         return {
             snack: false,
@@ -232,8 +188,8 @@ export default {
           return  `${this.settings.gpuAcceleration ? "Enabled" : "Unsupported"} ${this.settings.gpuAcceleration ? "(" + this.settings.gpuAcceleration + ")" : ""}`
         },
         metrics() {
-          console.log(this.$store.state.metrics);
-            return this.$store.state.metrics;
+          // console.log(this.$store.state.metrics);
+          return this.$store.state.metrics;
         }
     },
     methods: {
@@ -319,6 +275,10 @@ export default {
                 this.snack = true;
             });
         },
+        showLogs(event) {
+          event;
+          this.$store.state.logsOverlay = true;
+        }
     }
 }
 </script>

photon-core/.gitignore

@@ -9,5 +9,7 @@ build
 build/*
 photonvision/*
 photonvision_config/*
+photon-server/lib/*
+photon-server/package-lock.json
 
 src/main/java/org/photonvision/PhotonVersion.java

photon-core/build.gradle

@@ -1,3 +1,7 @@
+plugins {
+    id 'edu.wpi.first.WpilibTools' version '1.0.0'
+}
+
 import java.nio.file.Path
 
 apply from: "${rootDir}/shared/common.gradle"
@@ -10,9 +14,6 @@ dependencies {
     implementation 'org.msgpack:msgpack-core:0.9.0'
     implementation 'org.msgpack:jackson-dataformat-msgpack:0.9.0'
 
-    // wpiutil
-    jniPlatforms.each { implementation "edu.wpi.first.wpiutil:wpiutil-jni:$wpilibVersion:$it" }
-
     // JOGL stuff (currently we only distribute for aarch64, which is Pi 4)
     implementation "org.jogamp.gluegen:gluegen-rt:$joglVersion"
     implementation "org.jogamp.jogl:jogl-all:$joglVersion"
@@ -22,6 +23,8 @@ dependencies {
 
     // Zip
     implementation 'org.zeroturnaround:zt-zip:1.14'
+
+    implementation wpilibTools.deps.wpilibJava("apriltag")
 }
 
 task writeCurrentVersionJava {
@@ -31,3 +34,24 @@ task writeCurrentVersionJava {
 }
 
 build.dependsOn writeCurrentVersionJava
+
+def testNativeConfigName = 'wpilibTestNative'
+def testNativeConfig = configurations.create(testNativeConfigName)
+
+def folder = project.layout.buildDirectory.dir('NativeTest')
+
+def testNativeTasks = wpilibTools.createExtractionTasks {
+    taskPostfix = "Test"
+    configurationName = testNativeConfigName
+    rootTaskFolder.set(folder)
+}
+
+testNativeTasks.addToSourceSetResources(sourceSets.test)
+
+testNativeConfig.dependencies.add wpilibTools.deps.cscore()
+testNativeConfig.dependencies.add wpilibTools.deps.wpilib("ntcore")
+testNativeConfig.dependencies.add wpilibTools.deps.wpilib("wpinet")
+testNativeConfig.dependencies.add wpilibTools.deps.wpilib("hal")
+testNativeConfig.dependencies.add wpilibTools.deps.wpilib("wpiutil")
+testNativeConfig.dependencies.add wpilibTools.deps.wpilib("apriltag")
+testNativeConfig.dependencies.add wpilibTools.deps.wpilib("wpimath")

photon-core/src/main/java/org/photonvision/common/configuration/CameraConfiguration.java

@@ -20,8 +20,8 @@ package org.photonvision.common.configuration;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import edu.wpi.first.math.geometry.Rotation2d;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import org.photonvision.common.logging.LogGroup;
 import org.photonvision.common.logging.Logger;
@@ -46,11 +46,12 @@ public class CameraConfiguration {
     /** Can be either path (ex /dev/videoX) or index (ex 1). */
     public String path = "";
 
+    @JsonIgnore public String[] otherPaths = {};
+
     public CameraType cameraType = CameraType.UsbCamera;
     public double FOV = 70;
     public final List<CameraCalibrationCoefficients> calibrations;
     public int currentPipelineIndex = 0;
-    public Rotation2d camPitch = new Rotation2d();
 
     public int streamIndex = 0; // 0 index means ports [1181, 1182], 1 means [1183, 1184], etc...
 
@@ -61,19 +62,22 @@ public class CameraConfiguration {
     public DriverModePipelineSettings driveModeSettings = new DriverModePipelineSettings();
 
     public CameraConfiguration(String baseName, String path) {
-        this(baseName, baseName, baseName, path);
+        this(baseName, baseName, baseName, path, new String[0]);
     }
 
-    public CameraConfiguration(String baseName, String uniqueName, String nickname, String path) {
+    public CameraConfiguration(
+            String baseName, String uniqueName, String nickname, String path, String[] alternates) {
         this.baseName = baseName;
         this.uniqueName = uniqueName;
         this.nickname = nickname;
         this.path = path;
         this.calibrations = new ArrayList<>();
+        this.otherPaths = alternates;
 
         logger.debug(
                 "Creating USB camera configuration for "
                         + cameraType
+                        + " "
                         + baseName
                         + " (AKA "
                         + nickname
@@ -90,8 +94,7 @@ public class CameraConfiguration {
             @JsonProperty("path") String path,
             @JsonProperty("cameraType") CameraType cameraType,
             @JsonProperty("calibration") List<CameraCalibrationCoefficients> calibrations,
-            @JsonProperty("currentPipelineIndex") int currentPipelineIndex,
-            @JsonProperty("camPitch") Rotation2d camPitch) {
+            @JsonProperty("currentPipelineIndex") int currentPipelineIndex) {
         this.baseName = baseName;
         this.uniqueName = uniqueName;
         this.nickname = nickname;
@@ -100,11 +103,11 @@ public class CameraConfiguration {
         this.cameraType = cameraType;
         this.calibrations = calibrations != null ? calibrations : new ArrayList<>();
         this.currentPipelineIndex = currentPipelineIndex;
-        this.camPitch = camPitch;
 
         logger.debug(
                 "Creating camera configuration for "
                         + cameraType
+                        + " "
                         + baseName
                         + " (AKA "
                         + nickname
@@ -147,4 +150,33 @@ public class CameraConfiguration {
                 .ifPresent(calibrations::remove);
         calibrations.add(calibration);
     }
+
+    @Override
+    public String toString() {
+        return "CameraConfiguration [baseName="
+                + baseName
+                + ", uniqueName="
+                + uniqueName
+                + ", nickname="
+                + nickname
+                + ", path="
+                + path
+                + ", otherPaths="
+                + Arrays.toString(otherPaths)
+                + ", cameraType="
+                + cameraType
+                + ", FOV="
+                + FOV
+                + ", calibrations="
+                + calibrations
+                + ", currentPipelineIndex="
+                + currentPipelineIndex
+                + ", streamIndex="
+                + streamIndex
+                + ", pipelineSettings="
+                + pipelineSettings
+                + ", driveModeSettings="
+                + driveModeSettings
+                + "]";
+    }
 }

photon-core/src/main/java/org/photonvision/common/configuration/ConfigManager.java

@@ -438,7 +438,7 @@ public class ConfigManager {
             try {
                 Thread.sleep(1000);
             } catch (InterruptedException e) {
-                logger.error("Exception waiting for settings semaphor", e);
+                logger.error("Exception waiting for settings semaphore", e);
             }
         }
     }

photon-core/src/main/java/org/photonvision/common/configuration/HardwareConfig.java

@@ -41,6 +41,8 @@ public class HardwareConfig {
     public final String cpuTempCommand;
     public final String cpuMemoryCommand;
     public final String cpuUtilCommand;
+    public final String cpuThrottleReasonCmd;
+    public final String cpuUptimeCommand;
     public final String gpuMemoryCommand;
     public final String ramUtilCommand;
     public final String gpuMemUsageCommand;
@@ -65,6 +67,8 @@ public class HardwareConfig {
         cpuTempCommand = "";
         cpuMemoryCommand = "";
         cpuUtilCommand = "";
+        cpuThrottleReasonCmd = "";
+        cpuUptimeCommand = "";
         gpuMemoryCommand = "";
         ramUtilCommand = "";
         ledBlinkCommand = "";
@@ -91,6 +95,8 @@ public class HardwareConfig {
             String cpuTempCommand,
             String cpuMemoryCommand,
             String cpuUtilCommand,
+            String cpuThrottleReasonCmd,
+            String cpuUptimeCommand,
             String gpuMemoryCommand,
             String ramUtilCommand,
             String gpuMemUsageCommand,
@@ -111,6 +117,8 @@ public class HardwareConfig {
         this.cpuTempCommand = cpuTempCommand;
         this.cpuMemoryCommand = cpuMemoryCommand;
         this.cpuUtilCommand = cpuUtilCommand;
+        this.cpuThrottleReasonCmd = cpuThrottleReasonCmd;
+        this.cpuUptimeCommand = cpuUptimeCommand;
         this.gpuMemoryCommand = gpuMemoryCommand;
         this.ramUtilCommand = ramUtilCommand;
         this.gpuMemUsageCommand = gpuMemUsageCommand;
@@ -120,7 +128,22 @@ public class HardwareConfig {
         this.blacklistedResIndices = blacklistedResIndices;
     }
 
+    /** @return True if the FOV has been preset to a sane value, false otherwise */
     public final boolean hasPresetFOV() {
         return vendorFOV > 0;
     }
+
+    /** @return True if any command has been configured to a non-default empty, false otherwise */
+    public final boolean hasCommandsConfigured() {
+        return cpuTempCommand != ""
+                || cpuMemoryCommand != ""
+                || cpuUtilCommand != ""
+                || cpuThrottleReasonCmd != ""
+                || cpuUptimeCommand != ""
+                || gpuMemoryCommand != ""
+                || ramUtilCommand != ""
+                || ledBlinkCommand != ""
+                || gpuMemUsageCommand != ""
+                || diskUsageCommand != "";
+    }
 }

photon-core/src/main/java/org/photonvision/common/configuration/NetworkConfig.java

@@ -19,12 +19,15 @@ package org.photonvision.common.configuration;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonGetter;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonSetter;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import java.util.HashMap;
 import java.util.Map;
 import org.photonvision.common.hardware.Platform;
 import org.photonvision.common.networking.NetworkMode;
+import org.photonvision.common.util.file.JacksonUtils;
 
 public class NetworkConfig {
     public int teamNumber = 0;
@@ -33,6 +36,16 @@ public class NetworkConfig {
     public String hostname = "photonvision";
     public boolean runNTServer = false;
 
+    @JsonIgnore public static final String NM_IFACE_STRING = "${interface}";
+    @JsonIgnore public static final String NM_IP_STRING = "${ipaddr}";
+
+    public String networkManagerIface = "Wired\\ connection\\ 1";
+    public String physicalInterface = "eth0";
+    public String setStaticCommand =
+            "nmcli con mod ${interface} ipv4.addresses ${ipaddr}/8 ipv4.method \"manual\" ipv6.method \"disabled\"";
+    public String setDHCPcommand =
+            "nmcli con mod ${interface} ipv4.method \"auto\" ipv6.method \"disabled\"";
+
     private boolean shouldManage;
 
     public NetworkConfig() {
@@ -46,46 +59,48 @@ public class NetworkConfig {
             @JsonProperty("staticIp") String staticIp,
             @JsonProperty("hostname") String hostname,
             @JsonProperty("runNTServer") boolean runNTServer,
-            @JsonProperty("shouldManage") boolean shouldManage) {
+            @JsonProperty("shouldManage") boolean shouldManage,
+            @JsonProperty("networkManagerIface") String networkManagerIface,
+            @JsonProperty("physicalInterface") String physicalInterface,
+            @JsonProperty("setStaticCommand") String setStaticCommand,
+            @JsonProperty("setDHCPcommand") String setDHCPcommand) {
         this.teamNumber = teamNumber;
         this.connectionType = connectionType;
         this.staticIp = staticIp;
         this.hostname = hostname;
         this.runNTServer = runNTServer;
+        this.networkManagerIface = networkManagerIface;
+        this.physicalInterface = physicalInterface;
+        this.setStaticCommand = setStaticCommand;
+        this.setDHCPcommand = setDHCPcommand;
         setShouldManage(shouldManage);
     }
 
     public static NetworkConfig fromHashMap(Map<String, Object> map) {
-        // teamNumber (int), supported (bool), connectionType (int),
-        // staticIp (str), netmask (str), hostname (str)
-        var ret = new NetworkConfig();
-        ret.teamNumber = Integer.parseInt(map.get("teamNumber").toString());
-        ret.connectionType = NetworkMode.values()[(Integer) map.get("connectionType")];
-        ret.staticIp = (String) map.get("staticIp");
-        ret.hostname = (String) map.get("hostname");
-        ret.runNTServer = (Boolean) map.get("runNTServer");
-        ret.setShouldManage((Boolean) map.get("supported"));
-        return ret;
+        try {
+            return new ObjectMapper().convertValue(map, NetworkConfig.class);
+        } catch (Exception e) {
+            e.printStackTrace();
+            return new NetworkConfig();
+        }
     }
 
-    public HashMap<String, Object> toHashMap() {
-        HashMap<String, Object> tmp = new HashMap<>();
-        tmp.put("teamNumber", teamNumber);
-        tmp.put("supported", shouldManage());
-        tmp.put("connectionType", connectionType.ordinal());
-        tmp.put("staticIp", staticIp);
-        tmp.put("hostname", hostname);
-        tmp.put("runNTServer", runNTServer);
-        return tmp;
+    public Map<String, Object> toHashMap() {
+        try {
+            return new ObjectMapper().convertValue(this, JacksonUtils.UIMap.class);
+        } catch (Exception e) {
+            e.printStackTrace();
+            return new HashMap<>();
+        }
     }
 
     @JsonGetter("shouldManage")
     public boolean shouldManage() {
-        return this.shouldManage || Platform.isRaspberryPi();
+        return this.shouldManage || Platform.isLinux();
     }
 
     @JsonSetter("shouldManage")
     public void setShouldManage(boolean shouldManage) {
-        this.shouldManage = shouldManage || Platform.isRaspberryPi();
+        this.shouldManage = shouldManage || Platform.isLinux();
     }
 }

photon-core/src/main/java/org/photonvision/common/configuration/PhotonConfiguration.java

@@ -25,7 +25,7 @@ import java.util.stream.Collectors;
 import org.photonvision.PhotonVersion;
 import org.photonvision.common.hardware.Platform;
 import org.photonvision.common.util.SerializationUtils;
-import org.photonvision.raspi.PicamJNI;
+import org.photonvision.raspi.LibCameraJNI;
 import org.photonvision.vision.processes.VisionModule;
 import org.photonvision.vision.processes.VisionModuleManager;
 import org.photonvision.vision.processes.VisionSource;
@@ -110,11 +110,11 @@ public class PhotonConfiguration {
         generalSubmap.put("version", PhotonVersion.versionString);
         generalSubmap.put(
                 "gpuAcceleration",
-                PicamJNI.isSupported()
-                        ? "Zerocopy MMAL on " + PicamJNI.getSensorModel().getFriendlyName()
+                LibCameraJNI.isSupported()
+                        ? "Zerocopy Libcamera on " + LibCameraJNI.getSensorModel().getFriendlyName()
                         : ""); // TODO add support for other types of GPU accel
         generalSubmap.put("hardwareModel", hardwareConfig.deviceName);
-        generalSubmap.put("hardwarePlatform", Platform.getCurrentPlatform().toString());
+        generalSubmap.put("hardwarePlatform", Platform.getPlatformName());
         settingsSubmap.put("general", generalSubmap);
 
         map.put("settings", settingsSubmap);
@@ -128,7 +128,8 @@ public class PhotonConfiguration {
 
     public static class UICameraConfiguration {
         @SuppressWarnings("unused")
-        public double fov, tiltDegrees;
+        public double fov;
+
         public String nickname;
         public HashMap<String, Object> currentPipelineSettings;
         public int currentPipelineIndex;

photon-core/src/main/java/org/photonvision/common/dataflow/networktables/NTDataChangeListener.java

@@ -17,22 +17,29 @@
 
 package org.photonvision.common.dataflow.networktables;
 
-import edu.wpi.first.networktables.EntryListenerFlags;
-import edu.wpi.first.networktables.EntryNotification;
-import edu.wpi.first.networktables.NetworkTableEntry;
+import edu.wpi.first.networktables.NetworkTableEvent;
+import edu.wpi.first.networktables.NetworkTableInstance;
+import edu.wpi.first.networktables.Subscriber;
+import java.util.EnumSet;
 import java.util.function.Consumer;
 
 public class NTDataChangeListener {
-    private final NetworkTableEntry watchedEntry;
+    private final NetworkTableInstance instance;
+    private final Subscriber watchedEntry;
     private final int listenerID;
 
     public NTDataChangeListener(
-            NetworkTableEntry watchedEntry, Consumer<EntryNotification> dataChangeConsumer) {
-        this.watchedEntry = watchedEntry;
-        listenerID = watchedEntry.addListener(dataChangeConsumer, EntryListenerFlags.kUpdate);
+            NetworkTableInstance instance,
+            Subscriber watchedSubscriber,
+            Consumer<NetworkTableEvent> dataChangeConsumer) {
+        this.watchedEntry = watchedSubscriber;
+        this.instance = instance;
+        listenerID =
+                this.instance.addListener(
+                        watchedEntry, EnumSet.of(NetworkTableEvent.Kind.kValueAll), dataChangeConsumer);
     }
 
     public void remove() {
-        watchedEntry.removeListener(listenerID);
+        this.instance.removeListener(listenerID);
     }
 }

photon-core/src/main/java/org/photonvision/common/dataflow/networktables/NTDataPublisher.java

@@ -17,9 +17,8 @@
 
 package org.photonvision.common.dataflow.networktables;
 
-import edu.wpi.first.networktables.EntryNotification;
 import edu.wpi.first.networktables.NetworkTable;
-import edu.wpi.first.networktables.NetworkTableEntry;
+import edu.wpi.first.networktables.NetworkTableEvent;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.function.BooleanSupplier;
@@ -28,6 +27,9 @@ import java.util.function.Supplier;
 import org.opencv.core.Point;
 import org.photonvision.common.dataflow.CVPipelineResultConsumer;
 import org.photonvision.common.dataflow.structures.Packet;
+import org.photonvision.common.logging.LogGroup;
+import org.photonvision.common.logging.Logger;
+import org.photonvision.common.networktables.NTTopicSet;
 import org.photonvision.targeting.PhotonPipelineResult;
 import org.photonvision.targeting.PhotonTrackedTarget;
 import org.photonvision.targeting.TargetCorner;
@@ -35,31 +37,21 @@ import org.photonvision.vision.pipeline.result.CVPipelineResult;
 import org.photonvision.vision.target.TrackedTarget;
 
 public class NTDataPublisher implements CVPipelineResultConsumer {
+    private final Logger logger = new Logger(NTDataPublisher.class, LogGroup.General);
+
     private final NetworkTable rootTable = NetworkTablesManager.getInstance().kRootTable;
-    private NetworkTable subTable;
-    private NetworkTableEntry rawBytesEntry;
 
-    private NetworkTableEntry pipelineIndexEntry;
-    private final Consumer<Integer> pipelineIndexConsumer;
-    private NTDataChangeListener pipelineIndexListener;
-    private NetworkTableEntry driverModeEntry;
-    private final Consumer<Boolean> driverModeConsumer;
-    private NTDataChangeListener driverModeListener;
-
-    private NetworkTableEntry latencyMillisEntry;
-    private NetworkTableEntry hasTargetEntry;
-    private NetworkTableEntry targetPitchEntry;
-    private NetworkTableEntry targetYawEntry;
-    private NetworkTableEntry targetAreaEntry;
-    private NetworkTableEntry targetPoseEntry;
-    private NetworkTableEntry targetSkewEntry;
-
-    // The raw position of the best target, in pixels.
-    private NetworkTableEntry bestTargetPosX;
-    private NetworkTableEntry bestTargetPosY;
+    private NTTopicSet ts = new NTTopicSet();
 
+    NTDataChangeListener pipelineIndexListener;
     private final Supplier<Integer> pipelineIndexSupplier;
+    private final Consumer<Integer> pipelineIndexConsumer;
+
+    NTDataChangeListener driverModeListener;
     private final BooleanSupplier driverModeSupplier;
+    private final Consumer<Boolean> driverModeConsumer;
+
+    private long heartbeatCounter = 0;
 
     public NTDataPublisher(
             String cameraNickname,
@@ -76,93 +68,67 @@ public class NTDataPublisher implements CVPipelineResultConsumer {
         updateEntries();
     }
 
-    private void onPipelineIndexChange(EntryNotification entryNotification) {
-        var newIndex = (int) entryNotification.value.getDouble();
+    private void onPipelineIndexChange(NetworkTableEvent entryNotification) {
+        var newIndex = (int) entryNotification.valueData.value.getInteger();
         var originalIndex = pipelineIndexSupplier.get();
 
         // ignore indexes below 0
         if (newIndex < 0) {
-            pipelineIndexEntry.forceSetNumber(originalIndex);
+            ts.pipelineIndexPublisher.set(originalIndex);
             return;
         }
 
         if (newIndex == originalIndex) {
-            // TODO: Log
+            logger.debug("Pipeline index is already " + newIndex);
             return;
         }
 
         pipelineIndexConsumer.accept(newIndex);
         var setIndex = pipelineIndexSupplier.get();
         if (newIndex != setIndex) { // set failed
-            pipelineIndexEntry.forceSetNumber(setIndex);
+            ts.pipelineIndexPublisher.set(setIndex);
             // TODO: Log
         }
-        // TODO: Log
+        logger.debug("Successfully set pipeline index to " + newIndex);
     }
 
-    private void onDriverModeChange(EntryNotification entryNotification) {
-        var newDriverMode = entryNotification.value.getBoolean();
+    private void onDriverModeChange(NetworkTableEvent entryNotification) {
+        var newDriverMode = entryNotification.valueData.value.getBoolean();
         var originalDriverMode = driverModeSupplier.getAsBoolean();
 
         if (newDriverMode == originalDriverMode) {
-            // TODO: Log
+            logger.debug("Driver mode is already " + newDriverMode);
             return;
         }
 
         driverModeConsumer.accept(newDriverMode);
-        // TODO: Log
+        logger.debug("Successfully set driver mode to " + newDriverMode);
     }
 
-    @SuppressWarnings("DuplicatedCode")
     private void removeEntries() {
-        if (rawBytesEntry != null) rawBytesEntry.delete();
         if (pipelineIndexListener != null) pipelineIndexListener.remove();
-        if (pipelineIndexEntry != null) pipelineIndexEntry.delete();
         if (driverModeListener != null) driverModeListener.remove();
-        if (driverModeEntry != null) driverModeEntry.delete();
-        if (latencyMillisEntry != null) latencyMillisEntry.delete();
-        if (hasTargetEntry != null) hasTargetEntry.delete();
-        if (targetPitchEntry != null) targetPitchEntry.delete();
-        if (targetAreaEntry != null) targetAreaEntry.delete();
-        if (targetYawEntry != null) targetYawEntry.delete();
-        if (targetPoseEntry != null) targetPoseEntry.delete();
-        if (targetSkewEntry != null) targetSkewEntry.delete();
-        if (bestTargetPosX != null) bestTargetPosX.delete();
-        if (bestTargetPosY != null) bestTargetPosY.delete();
+        ts.removeEntries();
     }
 
     private void updateEntries() {
-        rawBytesEntry = subTable.getEntry("rawBytes");
+        if (pipelineIndexListener != null) pipelineIndexListener.remove();
+        if (driverModeListener != null) driverModeListener.remove();
+
+        ts.updateEntries();
 
-        if (pipelineIndexListener != null) {
-            pipelineIndexListener.remove();
-        }
-        pipelineIndexEntry = subTable.getEntry("pipelineIndex");
         pipelineIndexListener =
-                new NTDataChangeListener(pipelineIndexEntry, this::onPipelineIndexChange);
+                new NTDataChangeListener(
+                        ts.subTable.getInstance(), ts.pipelineIndexSubscriber, this::onPipelineIndexChange);
 
-        if (driverModeListener != null) {
-            driverModeListener.remove();
-        }
-        driverModeEntry = subTable.getEntry("driverMode");
-        driverModeListener = new NTDataChangeListener(driverModeEntry, this::onDriverModeChange);
-
-        latencyMillisEntry = subTable.getEntry("latencyMillis");
-        hasTargetEntry = subTable.getEntry("hasTarget");
-
-        targetPitchEntry = subTable.getEntry("targetPitch");
-        targetAreaEntry = subTable.getEntry("targetArea");
-        targetYawEntry = subTable.getEntry("targetYaw");
-        targetPoseEntry = subTable.getEntry("targetPose");
-        targetSkewEntry = subTable.getEntry("targetSkew");
-
-        bestTargetPosX = subTable.getEntry("targetPixelsX");
-        bestTargetPosY = subTable.getEntry("targetPixelsY");
+        driverModeListener =
+                new NTDataChangeListener(
+                        ts.subTable.getInstance(), ts.driverModeSubscriber, this::onDriverModeChange);
     }
 
     public void updateCameraNickname(String newCameraNickname) {
         removeEntries();
-        subTable = rootTable.getSubTable(newCameraNickname);
+        ts.subTable = rootTable.getSubTable(newCameraNickname);
         updateEntries();
     }
 
@@ -174,49 +140,70 @@ public class NTDataPublisher implements CVPipelineResultConsumer {
         Packet packet = new Packet(simplified.getPacketSize());
         simplified.populatePacket(packet);
 
-        rawBytesEntry.forceSetRaw(packet.getData());
+        ts.rawBytesEntry.set(packet.getData());
 
-        pipelineIndexEntry.forceSetNumber(pipelineIndexSupplier.get());
-        driverModeEntry.forceSetBoolean(driverModeSupplier.getAsBoolean());
-        latencyMillisEntry.forceSetDouble(result.getLatencyMillis());
-        hasTargetEntry.forceSetBoolean(result.hasTargets());
+        ts.pipelineIndexPublisher.set(pipelineIndexSupplier.get());
+        ts.driverModePublisher.set(driverModeSupplier.getAsBoolean());
+        ts.latencyMillisEntry.set(result.getLatencyMillis());
+        ts.hasTargetEntry.set(result.hasTargets());
 
         if (result.hasTargets()) {
             var bestTarget = result.targets.get(0);
 
-            targetPitchEntry.forceSetDouble(bestTarget.getPitch());
-            targetYawEntry.forceSetDouble(bestTarget.getYaw());
-            targetAreaEntry.forceSetDouble(bestTarget.getArea());
-            targetSkewEntry.forceSetDouble(bestTarget.getSkew());
+            ts.targetPitchEntry.set(bestTarget.getPitch());
+            ts.targetYawEntry.set(bestTarget.getYaw());
+            ts.targetAreaEntry.set(bestTarget.getArea());
+            ts.targetSkewEntry.set(bestTarget.getSkew());
 
-            var poseX = bestTarget.getCameraToTarget().getTranslation().getX();
-            var poseY = bestTarget.getCameraToTarget().getTranslation().getY();
-            var poseRot = bestTarget.getCameraToTarget().getRotation().getDegrees();
-            targetPoseEntry.forceSetDoubleArray(new double[] {poseX, poseY, poseRot});
+            var pose = bestTarget.getBestCameraToTarget3d();
+            ts.targetPoseEntry.set(
+                    new double[] {
+                        pose.getTranslation().getX(),
+                        pose.getTranslation().getY(),
+                        pose.getTranslation().getZ(),
+                        pose.getRotation().getQuaternion().getW(),
+                        pose.getRotation().getQuaternion().getX(),
+                        pose.getRotation().getQuaternion().getY(),
+                        pose.getRotation().getQuaternion().getZ()
+                    });
 
             var targetOffsetPoint = bestTarget.getTargetOffsetPoint();
-            bestTargetPosX.forceSetDouble(targetOffsetPoint.x);
-            bestTargetPosY.forceSetDouble(targetOffsetPoint.y);
+            ts.bestTargetPosX.set(targetOffsetPoint.x);
+            ts.bestTargetPosY.set(targetOffsetPoint.y);
         } else {
-            targetPitchEntry.forceSetDouble(0);
-            targetYawEntry.forceSetDouble(0);
-            targetAreaEntry.forceSetDouble(0);
-            targetSkewEntry.forceSetDouble(0);
-            targetPoseEntry.forceSetDoubleArray(new double[] {0, 0, 0});
-            bestTargetPosX.forceSetDouble(0);
-            bestTargetPosY.forceSetDouble(0);
+            ts.targetPitchEntry.set(0);
+            ts.targetYawEntry.set(0);
+            ts.targetAreaEntry.set(0);
+            ts.targetSkewEntry.set(0);
+            ts.targetPoseEntry.set(new double[] {0, 0, 0});
+            ts.bestTargetPosX.set(0);
+            ts.bestTargetPosY.set(0);
         }
+
+        ts.heartbeatPublisher.set(heartbeatCounter++);
+
+        // TODO...nt4... is this needed?
         rootTable.getInstance().flush();
     }
 
     public static List<PhotonTrackedTarget> simpleFromTrackedTargets(List<TrackedTarget> targets) {
         var ret = new ArrayList<PhotonTrackedTarget>();
         for (var t : targets) {
-            var points = new Point[4];
-            t.getMinAreaRect().points(points);
-            var cornerList = new ArrayList<TargetCorner>();
-
-            for (int i = 0; i < 4; i++) cornerList.add(new TargetCorner(points[i].x, points[i].y));
+            var minAreaRectCorners = new ArrayList<TargetCorner>();
+            var detectedCorners = new ArrayList<TargetCorner>();
+            {
+                var points = new Point[4];
+                t.getMinAreaRect().points(points);
+                for (int i = 0; i < 4; i++) {
+                    minAreaRectCorners.add(new TargetCorner(points[i].x, points[i].y));
+                }
+            }
+            {
+                var points = t.getTargetCorners();
+                for (int i = 0; i < points.size(); i++) {
+                    detectedCorners.add(new TargetCorner(points.get(i).x, points.get(i).y));
+                }
+            }
 
             ret.add(
                     new PhotonTrackedTarget(
@@ -224,8 +211,12 @@ public class NTDataPublisher implements CVPipelineResultConsumer {
                             t.getPitch(),
                             t.getArea(),
                             t.getSkew(),
-                            t.getCameraToTarget(),
-                            cornerList));
+                            t.getFiducialId(),
+                            t.getBestCameraToTarget3d(),
+                            t.getAltCameraToTarget3d(),
+                            t.getPoseAmbiguity(),
+                            minAreaRectCorners,
+                            detectedCorners));
         }
         return ret;
     }

photon-core/src/main/java/org/photonvision/common/dataflow/networktables/NetworkTablesManager.java

@@ -17,12 +17,13 @@
 
 package org.photonvision.common.dataflow.networktables;
 
-import edu.wpi.first.networktables.LogMessage;
 import edu.wpi.first.networktables.NetworkTable;
+import edu.wpi.first.networktables.NetworkTableEvent;
 import edu.wpi.first.networktables.NetworkTableInstance;
 import java.util.HashMap;
 import java.util.function.Consumer;
 import org.photonvision.PhotonVersion;
+import org.photonvision.common.configuration.ConfigManager;
 import org.photonvision.common.configuration.NetworkConfig;
 import org.photonvision.common.dataflow.DataChangeService;
 import org.photonvision.common.dataflow.events.OutgoingUIEvent;
@@ -37,8 +38,11 @@ public class NetworkTablesManager {
     private final String kRootTableName = "/photonvision";
     public final NetworkTable kRootTable = ntInstance.getTable(kRootTableName);
 
+    private boolean isRetryingConnection = false;
+
     private NetworkTablesManager() {
-        ntInstance.addLogger(new NTLogger(), 0, 255); // to hide error messages
+        ntInstance.addLogger(0, 255, new NTLogger()); // to hide error messages
+        TimedTaskManager.getInstance().addTask("NTManager", this::ntTick, 5000);
     }
 
     private static NetworkTablesManager INSTANCE;
@@ -50,17 +54,17 @@ public class NetworkTablesManager {
 
     private static final Logger logger = new Logger(NetworkTablesManager.class, LogGroup.General);
 
-    private static class NTLogger implements Consumer<LogMessage> {
+    private static class NTLogger implements Consumer<NetworkTableEvent> {
         private boolean hasReportedConnectionFailure = false;
         private long lastConnectMessageMillis = 0;
 
         @Override
-        public void accept(LogMessage logMessage) {
-            if (!hasReportedConnectionFailure && logMessage.message.contains("timed out")) {
+        public void accept(NetworkTableEvent event) {
+            if (!hasReportedConnectionFailure && event.logMessage.message.contains("timed out")) {
                 logger.error("NT Connection has failed! Will retry in background.");
                 hasReportedConnectionFailure = true;
                 getInstance().broadcastConnectedStatus();
-            } else if (logMessage.message.contains("connected")
+            } else if (event.logMessage.message.contains("connected")
                     && System.currentTimeMillis() - lastConnectMessageMillis > 125) {
                 logger.info("NT Connected!");
                 hasReportedConnectionFailure = false;
@@ -109,17 +113,11 @@ public class NetworkTablesManager {
     }
 
     private void setClientMode(int teamNumber) {
-        logger.info("Starting NT Client");
+        if (!isRetryingConnection) logger.info("Starting NT Client");
         ntInstance.stopServer();
-
-        ntInstance.startClientTeam(teamNumber);
+        ntInstance.startClient4("photonvision");
+        ntInstance.setServerTeam(teamNumber);
         ntInstance.startDSClient();
-        if (ntInstance.isConnected()) {
-            logger.info("[NetworkTablesManager] Connected to the robot!");
-        } else {
-            logger.error(
-                    "[NetworkTablesManager] Could not connect to the robot! Will retry in the background...");
-        }
         broadcastVersion();
     }
 
@@ -129,4 +127,22 @@ public class NetworkTablesManager {
         ntInstance.startServer();
         broadcastVersion();
     }
+
+    // So it seems like if Photon starts before the robot NT server does, and both aren't static IP,
+    // it'll never connect. This hack works around it by restarting the client/server while the nt
+    // instance
+    // isn't connected, same as clicking the save button in the settings menu (or restarting the
+    // service)
+    private void ntTick() {
+        if (!ntInstance.isConnected()
+                && !ConfigManager.getInstance().getConfig().getNetworkConfig().runNTServer) {
+            setConfig(ConfigManager.getInstance().getConfig().getNetworkConfig());
+        }
+
+        if (!ntInstance.isConnected() && !isRetryingConnection) {
+            isRetryingConnection = true;
+            logger.error(
+                    "[NetworkTablesManager] Could not connect to the robot! Will retry in the background...");
+        }
+    }
 }

photon-core/src/main/java/org/photonvision/common/hardware/HardwareManager.java

@@ -17,7 +17,7 @@
 
 package org.photonvision.common.hardware;
 
-import edu.wpi.first.networktables.NetworkTableEntry;
+import edu.wpi.first.networktables.IntegerEntry;
 import java.io.IOException;
 import org.photonvision.common.ProgramStatus;
 import org.photonvision.common.configuration.ConfigManager;
@@ -27,7 +27,7 @@ import org.photonvision.common.dataflow.networktables.NTDataChangeListener;
 import org.photonvision.common.dataflow.networktables.NetworkTablesManager;
 import org.photonvision.common.hardware.GPIO.CustomGPIO;
 import org.photonvision.common.hardware.GPIO.pi.PigpioSocket;
-import org.photonvision.common.hardware.metrics.MetricsBase;
+import org.photonvision.common.hardware.metrics.MetricsManager;
 import org.photonvision.common.logging.LogGroup;
 import org.photonvision.common.logging.Logger;
 import org.photonvision.common.util.ShellExec;
@@ -41,11 +41,13 @@ public class HardwareManager {
     private final HardwareConfig hardwareConfig;
     private final HardwareSettings hardwareSettings;
 
+    private final MetricsManager metricsManager;
+
     @SuppressWarnings({"FieldCanBeLocal", "unused"})
     private final StatusLED statusLED;
 
     @SuppressWarnings("FieldCanBeLocal")
-    private final NetworkTableEntry ledModeEntry;
+    private final IntegerEntry ledModeEntry;
 
     @SuppressWarnings({"FieldCanBeLocal", "unused"})
     private final NTDataChangeListener ledModeListener;
@@ -65,8 +67,11 @@ public class HardwareManager {
     private HardwareManager(HardwareConfig hardwareConfig, HardwareSettings hardwareSettings) {
         this.hardwareConfig = hardwareConfig;
         this.hardwareSettings = hardwareSettings;
+
+        this.metricsManager = new MetricsManager();
+        this.metricsManager.setConfig(hardwareConfig);
+
         CustomGPIO.setConfig(hardwareConfig);
-        MetricsBase.setConfig(hardwareConfig);
 
         if (Platform.isRaspberryPi()) {
             pigpioSocket = new PigpioSocket();
@@ -89,12 +94,16 @@ public class HardwareManager {
                                 hasBrightnessRange ? hardwareConfig.ledBrightnessRange.get(1) : 100,
                                 pigpioSocket);
 
-        ledModeEntry = NetworkTablesManager.getInstance().kRootTable.getEntry("ledMode");
-        ledModeEntry.setNumber(VisionLEDMode.kDefault.value);
+        ledModeEntry =
+                NetworkTablesManager.getInstance().kRootTable.getIntegerTopic("ledMode").getEntry(0);
+        ledModeEntry.set(VisionLEDMode.kDefault.value);
         ledModeListener =
                 visionLED == null
                         ? null
-                        : new NTDataChangeListener(ledModeEntry, visionLED::onLedModeChange);
+                        : new NTDataChangeListener(
+                                NetworkTablesManager.getInstance().kRootTable.getInstance(),
+                                ledModeEntry,
+                                visionLED::onLedModeChange);
 
         Runtime.getRuntime().addShutdownHook(new Thread(this::onJvmExit));
 
@@ -122,7 +131,7 @@ public class HardwareManager {
     }
 
     public boolean restartDevice() {
-        if (Platform.isRaspberryPi()) {
+        if (Platform.isLinux()) {
             try {
                 return shellExec.executeBashCommand("reboot now") == 0;
             } catch (IOException e) {
@@ -158,4 +167,8 @@ public class HardwareManager {
     public HardwareConfig getConfig() {
         return hardwareConfig;
     }
+
+    public void publishMetrics() {
+        metricsManager.publishMetrics();
+    }
 }

photon-core/src/main/java/org/photonvision/common/hardware/PiVersion.java

@@ -17,6 +17,9 @@
 
 package org.photonvision.common.hardware;
 
+import java.io.IOException;
+import org.photonvision.common.util.ShellExec;
+
 public enum PiVersion {
     PI_B("Pi Model B"),
     COMPUTE_MODULE("Compute Module Rev"),
@@ -28,17 +31,41 @@ public enum PiVersion {
     UNKNOWN("UNKNOWN");
 
     private final String identifier;
+    private static final ShellExec shell = new ShellExec(true, false);
+    private static final PiVersion currentPiVersion = calcPiVersion();
 
-    PiVersion(String s) {
+    private PiVersion(String s) {
         this.identifier = s.toLowerCase();
     }
 
     public static PiVersion getPiVersion() {
+        return currentPiVersion;
+    }
+
+    private static PiVersion calcPiVersion() {
         if (!Platform.isRaspberryPi()) return PiVersion.UNKNOWN;
-        String piString = Platform.currentPiVersionStr;
+        String piString = getPiVersionString();
         for (PiVersion p : PiVersion.values()) {
             if (piString.toLowerCase().contains(p.identifier)) return p;
         }
         return UNKNOWN;
     }
+
+    // Query /proc/device-tree/model. This should return the model of the pi
+    // Versions here:
+    // https://github.com/raspberrypi/linux/blob/rpi-5.10.y/arch/arm/boot/dts/bcm2710-rpi-cm3.dts
+    private static String getPiVersionString() {
+        if (!Platform.isRaspberryPi()) return "";
+        try {
+            shell.executeBashCommand("cat /proc/device-tree/model");
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        if (shell.getExitCode() == 0) {
+            // We expect it to be in the format "raspberry pi X model X"
+            return shell.getOutput();
+        }
+
+        return "";
+    }
 }

photon-core/src/main/java/org/photonvision/common/hardware/Platform.java

@@ -17,58 +17,91 @@
 
 package org.photonvision.common.hardware;
 
+import com.jogamp.common.os.Platform.OSType;
 import edu.wpi.first.util.RuntimeDetector;
+import java.io.BufferedReader;
 import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import org.photonvision.common.util.ShellExec;
 
 @SuppressWarnings("unused")
 public enum Platform {
     // WPILib Supported (JNI)
-    WINDOWS_32("Windows x32"),
-    WINDOWS_64("Windows x64"),
-    LINUX_64("Linux x64"),
-    LINUX_RASPBIAN("Linux Raspbian"), // Raspberry Pi 3/4
-    LINUX_AARCH64BIONIC("Linux AARCH64 Bionic"), // Jetson Nano, Jetson TX2
-    // PhotonVision Supported (Manual install)
-    LINUX_ARM32("Linux ARM32"), // ODROID XU4, C1+
-    LINUX_ARM64("Linux ARM64"), // ODROID C2, N2
+    WINDOWS_64("Windows x64", false, OSType.WINDOWS, true),
+    LINUX_32("Linux x86", false, OSType.LINUX, true),
+    LINUX_64("Linux x64", false, OSType.LINUX, true),
+    LINUX_RASPBIAN32(
+            "Linux Raspbian 32-bit", true, OSType.LINUX, true), // Raspberry Pi 3/4 with a 32-bit image
+    LINUX_RASPBIAN64(
+            "Linux Raspbian 64-bit", true, OSType.LINUX, true), // Raspberry Pi 3/4 with a 64-bit image
+    LINUX_AARCH64("Linux AARCH64", false, OSType.LINUX, true), // Jetson Nano, Jetson TX2
+
+    // PhotonVision Supported (Manual build/install)
+    LINUX_ARM32("Linux ARM32", false, OSType.LINUX, true), // ODROID XU4, C1+
+    LINUX_ARM64("Linux ARM64", false, OSType.LINUX, true), // ODROID C2, N2
 
     // Completely unsupported
-    UNSUPPORTED("Unsupported Platform");
+    WINDOWS_32("Windows x86", false, OSType.WINDOWS, false),
+    MACOS("Mac OS", false, OSType.MACOS, false),
+    UNKNOWN("Unsupported Platform", false, OSType.UNKNOWN, false);
+
+    private enum OSType {
+        WINDOWS,
+        LINUX,
+        MACOS,
+        UNKNOWN
+    }
 
     private static final ShellExec shell = new ShellExec(true, false);
-    public final String value;
-    public static final boolean isRoot = checkForRoot();
+    public final String description;
+    public final boolean isPi;
+    public final OSType osType;
+    public final boolean isSupported;
 
-    Platform(String value) {
-        this.value = value;
+    // Set once at init, shouldn't be needed after.
+    private static final Platform currentPlatform = getCurrentPlatform();
+    private static final boolean isRoot = checkForRoot();
+
+    Platform(String description, boolean isPi, OSType osType, boolean isSupported) {
+        this.description = description;
+        this.isPi = isPi;
+        this.osType = osType;
+        this.isSupported = isSupported;
     }
 
-    private static final String OS_NAME = System.getProperty("os.name");
-    private static final String OS_ARCH = System.getProperty("os.arch");
-
-    // These are queried on init and should never change after
-    public static final Platform currentPlatform = getCurrentPlatform();
-    protected static final String currentPiVersionStr = getPiVersionString();
-    public static final PiVersion currentPiVersion = PiVersion.getPiVersion();
-
-    private static String UnknownPlatformString =
-            String.format("Unknown Platform. OS: %s, Architecture: %s", OS_NAME, OS_ARCH);
-
-    public boolean isWindows() {
-        return this == WINDOWS_64 || this == WINDOWS_32;
-    }
+    //////////////////////////////////////////////////////
+    // Public API
 
+    // Checks specifically if unix shell and API are supported
     public static boolean isLinux() {
-        return getCurrentPlatform() == LINUX_64
-                || getCurrentPlatform() == LINUX_RASPBIAN
-                || getCurrentPlatform() == LINUX_ARM64;
+        return currentPlatform.osType == OSType.LINUX;
     }
 
     public static boolean isRaspberryPi() {
-        return currentPlatform.equals(LINUX_RASPBIAN);
+        return currentPlatform.isPi;
     }
 
+    public static String getPlatformName() {
+        if (currentPlatform.equals(UNKNOWN)) {
+            return UnknownPlatformString;
+        } else {
+            return currentPlatform.description;
+        }
+    }
+
+    public static boolean isRoot() {
+        return isRoot;
+    }
+
+    //////////////////////////////////////////////////////
+
+    // Debug info related to unknown platforms for debug help
+    private static final String OS_NAME = System.getProperty("os.name");
+    private static final String OS_ARCH = System.getProperty("os.arch");
+    private static final String UnknownPlatformString =
+            String.format("Unknown Platform. OS: %s, Architecture: %s", OS_NAME, OS_ARCH);
+
     @SuppressWarnings("StatementWithEmptyBody")
     private static boolean checkForRoot() {
         if (isLinux()) {
@@ -92,49 +125,92 @@ public enum Platform {
         return false;
     }
 
-    public static Platform getCurrentPlatform() {
+    private static Platform getCurrentPlatform() {
         if (RuntimeDetector.isWindows()) {
-            if (RuntimeDetector.is32BitIntel()) return WINDOWS_32;
-            if (RuntimeDetector.is64BitIntel()) return WINDOWS_64;
+            if (RuntimeDetector.is32BitIntel()) {
+                return WINDOWS_32;
+            } else if (RuntimeDetector.is64BitIntel()) {
+                return WINDOWS_64;
+            } else {
+                // please don't try this
+                return UNKNOWN;
+            }
         }
 
         if (RuntimeDetector.isMac()) {
-            if (RuntimeDetector.is32BitIntel()) return UNSUPPORTED;
+            // TODO - once we have real support, this might have to be more granular
+            return MACOS;
         }
 
         if (RuntimeDetector.isLinux()) {
-            if (RuntimeDetector.is32BitIntel()) return UNSUPPORTED;
-            if (RuntimeDetector.is64BitIntel()) return LINUX_64;
-            if (RuntimeDetector.isRaspbian()) return LINUX_RASPBIAN;
+            if (isPiSBC()) {
+                if (RuntimeDetector.isArm32()) {
+                    return LINUX_RASPBIAN32;
+                } else if (RuntimeDetector.isArm64()) {
+                    return LINUX_RASPBIAN64;
+                } else {
+                    // Unknown/exotic installation
+                    return UNKNOWN;
+                }
+            } else if (isJetsonSBC()) {
+                if (RuntimeDetector.isArm64()) {
+                    // TODO - do we need to check OS version?
+                    return LINUX_AARCH64;
+                } else {
+                    // Unknown/exotic installation
+                    return UNKNOWN;
+                }
+            } else if (RuntimeDetector.is64BitIntel()) {
+                return LINUX_64;
+            } else if (RuntimeDetector.is32BitIntel()) {
+                return LINUX_32;
+            } else if (RuntimeDetector.isArm64()) {
+                // TODO - os detection needed?
+                return LINUX_AARCH64;
+            } else {
+                // Unknown or otherwise unsupported platform
+                return Platform.UNKNOWN;
+            }
         }
 
-        System.out.println(UnknownPlatformString);
-        return Platform.UNSUPPORTED;
+        // If we fall through all the way to here,
+        return Platform.UNKNOWN;
     }
 
-    public String toString() {
-        if (this.equals(UNSUPPORTED)) {
-            return UnknownPlatformString;
-        } else {
-            return this.value;
-        }
+    // Check for various known SBC types
+    private static boolean isPiSBC() {
+        return fileHasText("/proc/cpuinfo", "Raspberry Pi");
     }
 
-    // Querry /proc/device-tree/model. This should return the model of the pi
-    // Versions here:
-    // https://github.com/raspberrypi/linux/blob/rpi-5.10.y/arch/arm/boot/dts/bcm2710-rpi-cm3.dts
-    private static String getPiVersionString() {
-        if (!isRaspberryPi()) return "";
-        try {
-            shell.executeBashCommand("cat /proc/device-tree/model");
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-        if (shell.getExitCode() == 0) {
-            // We expect it to be in the format "raspberry pi X model X"
-            return shell.getOutput();
-        }
+    private static boolean isJetsonSBC() {
+        // https://forums.developer.nvidia.com/t/how-to-recognize-jetson-nano-device/146624
+        return fileHasText("/proc/device-tree/model", "NVIDIA Jetson");
+    }
 
-        return "";
+    // Checks for various names of linux OS
+    private static boolean isStretch() {
+        // TODO - this is a total guess
+        return fileHasText("/etc/os-release", "Stretch");
+    }
+
+    private static boolean isBuster() {
+        // TODO - this is a total guess
+        return fileHasText("/etc/os-release", "Buster");
+    }
+
+    private static boolean fileHasText(String filename, String text) {
+        try (BufferedReader reader = Files.newBufferedReader(Paths.get(filename))) {
+            while (true) {
+                String value = reader.readLine();
+                if (value == null) {
+                    return false;
+
+                } else if (value.contains(text)) {
+                    return true;
+                } // else, next line
+            }
+        } catch (IOException ex) {
+            return false;
+        }
     }
 }

photon-core/src/main/java/org/photonvision/common/hardware/VisionLED.java

@@ -17,7 +17,7 @@
 
 package org.photonvision.common.hardware;
 
-import edu.wpi.first.networktables.EntryNotification;
+import edu.wpi.first.networktables.NetworkTableEvent;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.function.BooleanSupplier;
@@ -85,6 +85,8 @@ public class VisionLED {
                 pigpioSocket.generateAndSendWaveform(pulseLengthMillis, blinkCount, ledPins);
             } catch (PigpioException e) {
                 logger.error("Failed to blink!", e);
+            } catch (NullPointerException e) {
+                logger.error("Failed to blink, pigpio internal issue!", e);
             }
         } else {
             for (GPIOBase led : visionLEDs) {
@@ -100,13 +102,19 @@ public class VisionLED {
                 pigpioSocket.waveTxStop();
             } catch (PigpioException e) {
                 logger.error("Failed to stop blink!", e);
+            } catch (NullPointerException e) {
+                logger.error("Failed to blink, pigpio internal issue!", e);
             }
         }
-        // if the user has set an LED brightness other than 100%, use that instead
-        if (mappedBrightnessPercentage == 100 || !state) {
-            visionLEDs.forEach((led) -> led.setState(state));
-        } else {
-            visionLEDs.forEach((led) -> led.setBrightness(mappedBrightnessPercentage));
+        try {
+            // if the user has set an LED brightness other than 100%, use that instead
+            if (mappedBrightnessPercentage == 100 || !state) {
+                visionLEDs.forEach((led) -> led.setState(state));
+            } else {
+                visionLEDs.forEach((led) -> led.setBrightness(mappedBrightnessPercentage));
+            }
+        } catch (NullPointerException e) {
+            logger.error("Failed to blink, pigpio internal issue!", e);
         }
     }
 
@@ -114,8 +122,8 @@ public class VisionLED {
         setInternal(on ? VisionLEDMode.kOn : VisionLEDMode.kOff, false);
     }
 
-    void onLedModeChange(EntryNotification entryNotification) {
-        var newLedModeRaw = (int) entryNotification.value.getDouble();
+    void onLedModeChange(NetworkTableEvent entryNotification) {
+        var newLedModeRaw = (int) entryNotification.valueData.value.getDouble();
         if (newLedModeRaw != currentLedMode.value) {
             VisionLEDMode newLedMode;
             switch (newLedModeRaw) {
@@ -177,6 +185,9 @@ public class VisionLED {
                     case kOn:
                         setStateImpl(true);
                         break;
+                    case kBlink:
+                        blinkImpl(85, -1);
+                        break;
                 }
             }
             logger.info("Changing LED internal state to " + newLedMode.toString());

photon-core/src/main/java/org/photonvision/common/hardware/metrics/MetricsBase.java

@@ -1,92 +0,0 @@
-/*
- * Copyright (C) Photon Vision.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package org.photonvision.common.hardware.metrics;
-
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import org.photonvision.common.configuration.HardwareConfig;
-import org.photonvision.common.hardware.Platform;
-import org.photonvision.common.logging.LogGroup;
-import org.photonvision.common.logging.Logger;
-import org.photonvision.common.util.ShellExec;
-
-public abstract class MetricsBase {
-    private static final Logger logger = new Logger(MetricsBase.class, LogGroup.General);
-    // CPU
-    public static String cpuMemoryCommand = "vcgencmd get_mem arm | grep -Eo '[0-9]+'";
-    public static String cpuTemperatureCommand =
-            "sed 's/.\\{3\\}$/.&/' <<< cat /sys/class/thermal/thermal_zone0/temp";
-    public static String cpuUtilizationCommand =
-            "top -bn1 | grep \"Cpu(s)\" | sed \"s/.*, *\\([0-9.]*\\)%* id.*/\\1/\" | awk '{print 100 - $1}'";
-
-    // GPU
-    public static String gpuMemoryCommand = "vcgencmd get_mem gpu | grep -Eo '[0-9]+'";
-    public static String gpuMemUsageCommand = "vcgencmd get_mem malloc | grep -Eo '[0-9]+'";
-
-    // RAM
-    public static String ramUsageCommand = "free --mega | awk -v i=2 -v j=3 'FNR == i {print $j}'";
-
-    // Disk
-    public static String diskUsageCommand = "df ./ --output=pcent | tail -n +2";
-
-    private static ShellExec runCommand = new ShellExec(true, true);
-
-    public static void setConfig(HardwareConfig config) {
-        if (Platform.isRaspberryPi()) return;
-        cpuMemoryCommand = config.cpuMemoryCommand;
-        cpuTemperatureCommand = config.cpuTempCommand;
-        cpuUtilizationCommand = config.cpuUtilCommand;
-
-        gpuMemoryCommand = config.gpuMemoryCommand;
-        gpuMemUsageCommand = config.gpuMemUsageCommand;
-
-        diskUsageCommand = config.diskUsageCommand;
-
-        ramUsageCommand = config.ramUtilCommand;
-    }
-
-    public static synchronized String execute(String command) {
-        try {
-            runCommand.executeBashCommand(command);
-            return runCommand.getOutput();
-        } catch (Exception e) {
-            StringWriter sw = new StringWriter();
-            PrintWriter pw = new PrintWriter(sw);
-            e.printStackTrace(pw);
-
-            logger.error(
-                    "Command: \""
-                            + command
-                            + "\" returned an error!"
-                            + "\nOutput Received: "
-                            + runCommand.getOutput()
-                            + "\nStandard Error: "
-                            + runCommand.getError()
-                            + "\nCommand completed: "
-                            + runCommand.isOutputCompleted()
-                            + "\nError completed: "
-                            + runCommand.isErrorCompleted()
-                            + "\nExit code: "
-                            + runCommand.getExitCode()
-                            + "\n Exception: "
-                            + e.toString()
-                            + sw.toString());
-            return "";
-        }
-    }
-}

photon-core/src/main/java/org/photonvision/common/hardware/metrics/MetricsManager.java

@@ -0,0 +1,161 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.common.hardware.metrics;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.HashMap;
+import org.photonvision.common.configuration.HardwareConfig;
+import org.photonvision.common.dataflow.DataChangeService;
+import org.photonvision.common.dataflow.events.OutgoingUIEvent;
+import org.photonvision.common.hardware.Platform;
+import org.photonvision.common.hardware.metrics.cmds.CmdBase;
+import org.photonvision.common.hardware.metrics.cmds.FileCmds;
+import org.photonvision.common.hardware.metrics.cmds.LinuxCmds;
+import org.photonvision.common.hardware.metrics.cmds.PiCmds;
+import org.photonvision.common.logging.LogGroup;
+import org.photonvision.common.logging.Logger;
+import org.photonvision.common.util.ShellExec;
+
+public class MetricsManager {
+    final Logger logger = new Logger(MetricsManager.class, LogGroup.General);
+
+    CmdBase cmds;
+
+    private ShellExec runCommand = new ShellExec(true, true);
+
+    public void setConfig(HardwareConfig config) {
+        if (config.hasCommandsConfigured()) {
+            cmds = new FileCmds();
+        } else if (Platform.isRaspberryPi()) {
+            cmds = new PiCmds(); // Pi's can use a hardcoded command set
+        } else if (Platform.isLinux()) {
+            cmds = new LinuxCmds(); // Linux/Unix platforms assume a nominal command set
+        } else {
+            cmds = new CmdBase(); // default - base has no commands
+        }
+
+        cmds.initCmds(config);
+    }
+
+    public String safeExecute(String str) {
+        if (str.isEmpty()) return "";
+        try {
+            return execute(str);
+        } catch (Exception e) {
+            return "****";
+        }
+    }
+
+    private String cpuMemSave = null;
+
+    public String getMemory() {
+        if (cmds.cpuMemoryCommand.isEmpty()) return "";
+        if (cpuMemSave == null) {
+            // save the value and only run it once
+            cpuMemSave = execute(cmds.cpuMemoryCommand);
+        }
+        return cpuMemSave;
+    }
+
+    public String getTemp() {
+        return safeExecute(cmds.cpuTemperatureCommand);
+    }
+
+    public String getUtilization() {
+        return safeExecute(cmds.cpuUtilizationCommand);
+    }
+
+    public String getUptime() {
+        return safeExecute(cmds.cpuUptimeCommand);
+    }
+
+    public String getThrottleReason() {
+        return safeExecute(cmds.cpuThrottleReasonCmd);
+    }
+
+    private String gpuMemSave = null;
+
+    public String getGPUMemorySplit() {
+        if (gpuMemSave == null) {
+            // only needs to run once
+            gpuMemSave = safeExecute(cmds.gpuMemoryCommand);
+        }
+        return gpuMemSave;
+    }
+
+    public String getMallocedMemory() {
+        return safeExecute(cmds.gpuMemUsageCommand);
+    }
+
+    public String getUsedDiskPct() {
+        return safeExecute(cmds.diskUsageCommand);
+    }
+
+    // TODO: Output in MBs for consistency
+    public String getUsedRam() {
+        return safeExecute(cmds.ramUsageCommand);
+    }
+
+    public void publishMetrics() {
+        logger.debug("Publishing Metrics...");
+        final var metrics = new HashMap<String, String>();
+
+        metrics.put("cpuTemp", this.getTemp());
+        metrics.put("cpuUtil", this.getUtilization());
+        metrics.put("cpuMem", this.getMemory());
+        metrics.put("cpuThr", this.getThrottleReason());
+        metrics.put("cpuUptime", this.getUptime());
+        metrics.put("gpuMem", this.getGPUMemorySplit());
+        metrics.put("ramUtil", this.getUsedRam());
+        metrics.put("gpuMemUtil", this.getMallocedMemory());
+        metrics.put("diskUtilPct", this.getUsedDiskPct());
+
+        DataChangeService.getInstance().publishEvent(OutgoingUIEvent.wrappedOf("metrics", metrics));
+    }
+
+    public synchronized String execute(String command) {
+        try {
+            runCommand.executeBashCommand(command);
+            return runCommand.getOutput();
+        } catch (Exception e) {
+            StringWriter sw = new StringWriter();
+            PrintWriter pw = new PrintWriter(sw);
+            e.printStackTrace(pw);
+
+            logger.error(
+                    "Command: \""
+                            + command
+                            + "\" returned an error!"
+                            + "\nOutput Received: "
+                            + runCommand.getOutput()
+                            + "\nStandard Error: "
+                            + runCommand.getError()
+                            + "\nCommand completed: "
+                            + runCommand.isOutputCompleted()
+                            + "\nError completed: "
+                            + runCommand.isErrorCompleted()
+                            + "\nExit code: "
+                            + runCommand.getExitCode()
+                            + "\n Exception: "
+                            + e.toString()
+                            + sw.toString());
+            return "";
+        }
+    }
+}

photon-core/src/main/java/org/photonvision/common/hardware/metrics/MetricsPublisher.java

@@ -1,74 +0,0 @@
-/*
- * Copyright (C) Photon Vision.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package org.photonvision.common.hardware.metrics;
-
-import java.util.HashMap;
-import org.photonvision.common.dataflow.DataChangeService;
-import org.photonvision.common.dataflow.events.OutgoingUIEvent;
-import org.photonvision.common.hardware.Platform;
-import org.photonvision.common.logging.LogGroup;
-import org.photonvision.common.logging.Logger;
-import org.photonvision.common.util.TimedTaskManager;
-
-public class MetricsPublisher {
-    private static final Logger logger = new Logger(MetricsPublisher.class, LogGroup.General);
-    private static CPUMetrics cpuMetrics;
-    private static GPUMetrics gpuMetrics;
-    private static RAMMetrics ramMetrics;
-    private static DiskMetrics diskMetrics;
-
-    public static MetricsPublisher getInstance() {
-        return Singleton.INSTANCE;
-    }
-
-    private MetricsPublisher() {
-        cpuMetrics = new CPUMetrics();
-        gpuMetrics = new GPUMetrics();
-        ramMetrics = new RAMMetrics();
-        diskMetrics = new DiskMetrics();
-    }
-
-    public void stopTask() {
-        TimedTaskManager.getInstance().cancelTask("Metrics");
-        logger.info("This device does not support running bash commands. Stopped metrics thread.");
-    }
-
-    public void publish() {
-        if (!Platform.isRaspberryPi()) {
-            logger.debug("Ignoring metrics on non-Pi devices");
-            return;
-        }
-
-        logger.debug("Publishing Metrics...");
-        final var metrics = new HashMap<String, String>();
-
-        metrics.put("cpuTemp", cpuMetrics.getTemp());
-        metrics.put("cpuUtil", cpuMetrics.getUtilization());
-        metrics.put("cpuMem", cpuMetrics.getMemory());
-        metrics.put("gpuMem", gpuMetrics.getGPUMemorySplit());
-        metrics.put("ramUtil", ramMetrics.getUsedRam());
-        metrics.put("gpuMemUtil", gpuMetrics.getMallocedMemory());
-        metrics.put("diskUtilPct", diskMetrics.getUsedDiskPct());
-
-        DataChangeService.getInstance().publishEvent(OutgoingUIEvent.wrappedOf("metrics", metrics));
-    }
-
-    private static class Singleton {
-        public static final MetricsPublisher INSTANCE = new MetricsPublisher();
-    }
-}

photon-core/src/main/java/org/photonvision/common/hardware/metrics/cmds/CmdBase.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.common.hardware.metrics.cmds;
+
+import org.photonvision.common.configuration.HardwareConfig;
+
+public class CmdBase {
+    // CPU
+    public String cpuMemoryCommand = "";
+    public String cpuTemperatureCommand = "";
+    public String cpuUtilizationCommand = "";
+    public String cpuThrottleReasonCmd = "";
+    public String cpuUptimeCommand = "";
+    // GPU
+    public String gpuMemoryCommand = "";
+    public String gpuMemUsageCommand = "";
+    // RAM
+    public String ramUsageCommand = "";
+    // Disk
+    public String diskUsageCommand = "";
+
+    public void initCmds(HardwareConfig config) {
+        return; // default - do nothing
+    }
+}

photon-core/src/main/java/org/photonvision/common/hardware/metrics/cmds/FileCmds.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.common.hardware.metrics.cmds;
+
+import org.photonvision.common.configuration.HardwareConfig;
+
+public class FileCmds extends CmdBase {
+    @Override
+    public void initCmds(HardwareConfig config) {
+        cpuMemoryCommand = config.cpuMemoryCommand;
+        cpuTemperatureCommand = config.cpuTempCommand;
+        cpuUtilizationCommand = config.cpuUtilCommand;
+        cpuThrottleReasonCmd = config.cpuThrottleReasonCmd;
+        cpuUptimeCommand = config.cpuUptimeCommand;
+
+        gpuMemoryCommand = config.gpuMemoryCommand;
+        gpuMemUsageCommand = config.gpuMemUsageCommand;
+
+        diskUsageCommand = config.diskUsageCommand;
+
+        ramUsageCommand = config.ramUtilCommand;
+    }
+}

photon-core/src/main/java/org/photonvision/common/hardware/metrics/cmds/LinuxCmds.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.common.hardware.metrics.cmds;
+
+import org.photonvision.common.configuration.HardwareConfig;
+
+public class LinuxCmds extends CmdBase {
+    public void initCmds(HardwareConfig config) {
+        // CPU
+        cpuMemoryCommand = "awk '/MemTotal:/ {print int($2 / 1000);}' /proc/meminfo";
+
+        // TODO: boards have lots of thermal devices. Hard to pick the CPU
+
+        cpuUtilizationCommand =
+                "top -bn1 | grep \"Cpu(s)\" | sed \"s/.*, *\\([0-9.]*\\)%* id.*/\\1/\" | awk '{print 100 - $1}'";
+
+        cpuUptimeCommand = "uptime -p | cut -c 4-";
+
+        // RAM
+        ramUsageCommand = "awk '/MemFree:/ {print int($2 / 1000);}' /proc/meminfo";
+
+        // Disk
+        diskUsageCommand = "df ./ --output=pcent | tail -n +2";
+    }
+}

photon-core/src/main/java/org/photonvision/common/hardware/metrics/cmds/PiCmds.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.common.hardware.metrics.cmds;
+
+import org.photonvision.common.configuration.HardwareConfig;
+
+public class PiCmds extends LinuxCmds {
+    /** Applies pi-specific commands, ignoring any input configuration */
+    public void initCmds(HardwareConfig config) {
+        super.initCmds(config);
+
+        // CPU
+        cpuMemoryCommand = "vcgencmd get_mem arm | grep -Eo '[0-9]+'";
+        cpuTemperatureCommand = "sed 's/.\\{3\\}$/.&/' /sys/class/thermal/thermal_zone0/temp";
+        cpuThrottleReasonCmd =
+                "if   ((  $(( $(vcgencmd get_throttled | grep -Eo 0x[0-9a-fA-F]*) & 0x01 )) != 0x00 )); then echo \"LOW VOLTAGE\"; "
+                        + " elif ((  $(( $(vcgencmd get_throttled | grep -Eo 0x[0-9a-fA-F]*) & 0x08 )) != 0x00 )); then echo \"HIGH TEMP\"; "
+                        + " elif ((  $(( $(vcgencmd get_throttled | grep -Eo 0x[0-9a-fA-F]*) & 0x10000 )) != 0x00 )); then echo \"Prev. Low Voltage\"; "
+                        + " elif ((  $(( $(vcgencmd get_throttled | grep -Eo 0x[0-9a-fA-F]*) & 0x80000 )) != 0x00 )); then echo \"Prev. High Temp\"; "
+                        + " else echo \"None\"; fi";
+
+        // GPU
+        gpuMemoryCommand = "vcgencmd get_mem gpu | grep -Eo '[0-9]+'";
+        gpuMemUsageCommand = "vcgencmd get_mem malloc | grep -Eo '[0-9]+'";
+    }
+}

photon-core/src/main/java/org/photonvision/common/networking/NetworkManager.java

@@ -18,6 +18,7 @@
 package org.photonvision.common.networking;
 
 import org.photonvision.common.configuration.ConfigManager;
+import org.photonvision.common.configuration.NetworkConfig;
 import org.photonvision.common.hardware.Platform;
 import org.photonvision.common.logging.LogGroup;
 import org.photonvision.common.logging.Logger;
@@ -47,9 +48,8 @@ public class NetworkManager {
         var config = ConfigManager.getInstance().getConfig().getNetworkConfig();
         logger.info("Setting " + config.connectionType + " with team team " + config.teamNumber);
         if (Platform.isLinux()) {
-            if (!Platform.isRoot) {
-                logger.error("Cannot manage network without root!");
-                return;
+            if (!Platform.isRoot()) {
+                logger.error("Cannot manage hostname without root!");
             }
 
             // always set hostname
@@ -96,10 +96,11 @@ public class NetworkManager {
             if (config.connectionType == NetworkMode.DHCP) {
                 var shell = new ShellExec();
                 try {
-                    if (!config.staticIp.equals("")) {
-                        shell.executeBashCommand("ip addr del " + config.staticIp + "/8 dev eth0");
-                    }
-                    shell.executeBashCommand("dhclient eth0", false);
+                    // set nmcli back to DHCP, and re-run dhclient -- this ought to grab a new IP address
+                    shell.executeBashCommand(
+                            config.setDHCPcommand.replace(
+                                    NetworkConfig.NM_IFACE_STRING, config.networkManagerIface));
+                    shell.executeBashCommand("dhclient " + config.physicalInterface, false);
                 } catch (Exception e) {
                     logger.error("Exception while setting DHCP!");
                 }
@@ -107,7 +108,30 @@ public class NetworkManager {
                 var shell = new ShellExec();
                 if (config.staticIp.length() > 0) {
                     try {
-                        shell.executeBashCommand("ip addr add " + config.staticIp + "/8" + " dev eth0");
+                        shell.executeBashCommand(
+                                config
+                                        .setStaticCommand
+                                        .replace(NetworkConfig.NM_IFACE_STRING, config.networkManagerIface)
+                                        .replace(NetworkConfig.NM_IP_STRING, config.staticIp));
+
+                        if (Platform.isRaspberryPi()) {
+                            // Pi's need to manually have their interface adjusted?? and the 5 second sleep is
+                            // integral in my testing (Matt)
+                            shell.executeBashCommand(
+                                    "sh -c 'nmcli con down "
+                                            + config.networkManagerIface
+                                            + "; nmcli con up "
+                                            + config.networkManagerIface
+                                            + "'");
+                        } else {
+                            // for now just bring down /up -- more testing needed on beelink et al
+                            shell.executeBashCommand(
+                                    "sh -c 'nmcli con down "
+                                            + config.networkManagerIface
+                                            + "; nmcli con up "
+                                            + config.networkManagerIface
+                                            + "'");
+                        }
                     } catch (Exception e) {
                         logger.error("Error while setting static IP!", e);
                     }

photon-core/src/main/java/org/photonvision/common/networking/NetworkMode.java

@@ -17,7 +17,14 @@
 
 package org.photonvision.common.networking;
 
+import com.fasterxml.jackson.annotation.JsonValue;
+
 public enum NetworkMode {
     DHCP,
-    STATIC
+    STATIC;
+
+    @JsonValue
+    public int toValue() {
+        return ordinal();
+    }
 }

photon-core/src/main/java/org/photonvision/common/scripting/ScriptManager.java

@@ -128,7 +128,7 @@ public class ScriptManager {
     }
 
     public static void queueEvent(ScriptEventType eventType) {
-        if (!Platform.currentPlatform.isWindows()) {
+        if (Platform.isLinux()) {
             try {
                 queuedEvents.putLast(eventType);
                 logger.info("Queued event: " + eventType.name());

photon-core/src/main/java/org/photonvision/common/util/NativeLibHelper.java

@@ -15,19 +15,26 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package org.photonvision.common.hardware.metrics;
+package org.photonvision.common.util;
 
-public class GPUMetrics extends MetricsBase {
-    private String gpuMemSplit = null;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 
-    public String getGPUMemorySplit() {
-        if (gpuMemSplit == null) {
-            gpuMemSplit = execute(gpuMemoryCommand);
+public class NativeLibHelper {
+    private static NativeLibHelper INSTANCE;
+
+    public static NativeLibHelper getInstance() {
+        if (INSTANCE == null) {
+            INSTANCE = new NativeLibHelper();
         }
-        return gpuMemSplit;
+
+        return INSTANCE;
     }
 
-    public String getMallocedMemory() {
-        return execute(gpuMemUsageCommand);
+    public final Path NativeLibPath;
+
+    private NativeLibHelper() {
+        String home = System.getProperty("user.home");
+        NativeLibPath = Paths.get(home, ".pvlibs", "nativecache");
     }
 }

photon-core/src/main/java/org/photonvision/common/util/TestUtils.java

@@ -18,17 +18,58 @@
 package org.photonvision.common.util;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import edu.wpi.first.apriltag.jni.AprilTagJNI;
 import edu.wpi.first.cscore.CameraServerCvJNI;
+import edu.wpi.first.cscore.CameraServerJNI;
+import edu.wpi.first.hal.JNIWrapper;
+import edu.wpi.first.math.geometry.Translation2d;
 import edu.wpi.first.math.util.Units;
+import edu.wpi.first.net.WPINetJNI;
+import edu.wpi.first.networktables.NetworkTablesJNI;
+import edu.wpi.first.util.CombinedRuntimeLoader;
+import edu.wpi.first.util.RuntimeLoader;
+import edu.wpi.first.util.WPIUtilJNI;
 import java.awt.*;
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Path;
+import org.opencv.core.Core;
 import org.opencv.core.Mat;
 import org.opencv.highgui.HighGui;
 import org.photonvision.vision.calibration.CameraCalibrationCoefficients;
 
 public class TestUtils {
+    public static boolean loadLibraries() {
+        JNIWrapper.Helper.setExtractOnStaticLoad(false);
+        WPIUtilJNI.Helper.setExtractOnStaticLoad(false);
+        NetworkTablesJNI.Helper.setExtractOnStaticLoad(false);
+        WPINetJNI.Helper.setExtractOnStaticLoad(false);
+        CameraServerJNI.Helper.setExtractOnStaticLoad(false);
+        CameraServerCvJNI.Helper.setExtractOnStaticLoad(false);
+        AprilTagJNI.Helper.setExtractOnStaticLoad(false);
+
+        try {
+            var loader =
+                    new RuntimeLoader<>(
+                            Core.NATIVE_LIBRARY_NAME, RuntimeLoader.getDefaultExtractionRoot(), Core.class);
+            loader.loadLibrary();
+
+            CombinedRuntimeLoader.loadLibraries(
+                    TestUtils.class,
+                    "wpiutiljni",
+                    "ntcorejni",
+                    "wpinetjni",
+                    "wpiHaljni",
+                    "cscorejni",
+                    "cscorejnicvstatic",
+                    "apriltagjni");
+            return true;
+        } catch (IOException e) {
+            e.printStackTrace();
+            return false;
+        }
+    }
+
     @SuppressWarnings("unused")
     public enum WPI2019Image {
         kCargoAngledDark48in(1.2192),
@@ -99,6 +140,34 @@ public class TestUtils {
         }
     }
 
+    public enum WPI2023Apriltags {
+        k162_36_Angle,
+        k162_36_Straight,
+        k383_60_Angle2;
+
+        public static double FOV = 68.5;
+
+        public final Translation2d approxPose;
+        public final Path path;
+
+        Path getPath() {
+            var filename = this.toString().substring(1);
+            return Path.of("2023", "AprilTags", filename + ".png");
+        }
+
+        Translation2d getPose() {
+            var names = this.toString().substring(1).split("_");
+            var x = Units.inchesToMeters(Integer.parseInt(names[0]));
+            var y = Units.inchesToMeters(Integer.parseInt(names[1]));
+            return new Translation2d(x, y);
+        }
+
+        WPI2023Apriltags() {
+            this.approxPose = getPose();
+            this.path = getPath();
+        }
+    }
+
     public enum WPI2022Image {
         kTerminal12ft6in(Units.feetToMeters(12.5)),
         kTerminal22ft6in(Units.feetToMeters(22.5));
@@ -154,9 +223,43 @@ public class TestUtils {
         }
     }
 
-    private static Path getResourcesFolderPath(boolean testMode) {
+    public enum ApriltagTestImages {
+        kRobots,
+        kTag1_640_480,
+        kTag1_16h5_1280,
+        kTag_corner_1280;
+
+        public final Path path;
+
+        Path getPath() {
+            // Strip leading k
+            var filename = this.toString().substring(1).toLowerCase();
+            var extension = ".jpg";
+            if (filename.equals("tag1_16h5_1280")) extension = ".png";
+            return Path.of("apriltag", filename + extension);
+        }
+
+        ApriltagTestImages() {
+            this.path = getPath();
+        }
+    }
+
+    public static Path getResourcesFolderPath(boolean testMode) {
         System.out.println("CWD: " + Path.of("").toAbsolutePath().toString());
-        return Path.of("test-resources").toAbsolutePath();
+
+        // VSCode likes to make this path relative to the wrong root directory, so a fun hack to tell
+        // if it's wrong
+        Path ret = Path.of("test-resources").toAbsolutePath();
+        if (Path.of("test-resources")
+                .toAbsolutePath()
+                .toString()
+                .replace("/", "")
+                .replace("\\", "")
+                .toLowerCase()
+                .matches(".*photon-[a-z]*test-resources")) {
+            ret = Path.of("../test-resources").toAbsolutePath();
+        }
+        return ret;
     }
 
     public static Path getTestMode2019ImagePath() {
@@ -168,7 +271,7 @@ public class TestUtils {
     public static Path getTestMode2020ImagePath() {
         return getResourcesFolderPath(true)
                 .resolve("testimages")
-                .resolve(WPI2020Image.kBlueGoal_108in_Center.path);
+                .resolve(WPI2020Image.kBlueGoal_156in_Left.path);
     }
 
     public static Path getTestMode2022ImagePath() {
@@ -177,6 +280,12 @@ public class TestUtils {
                 .resolve(WPI2022Image.kTerminal22ft6in.path);
     }
 
+    public static Path getTestModeApriltagPath() {
+        return getResourcesFolderPath(true)
+                .resolve("testimages")
+                .resolve(ApriltagTestImages.kRobots.path);
+    }
+
     public static Path getTestImagesPath(boolean testMode) {
         return getResourcesFolderPath(testMode).resolve("testimages");
     }
@@ -201,6 +310,10 @@ public class TestUtils {
         return getTestImagesPath(testMode).resolve(image.path);
     }
 
+    public static Path getApriltagImagePath(ApriltagTestImages image, boolean testMode) {
+        return getTestImagesPath(testMode).resolve(image.path);
+    }
+
     public static Path getPowercellImagePath(PowercellTestImages image, boolean testMode) {
         return getPowercellPath(testMode).resolve(image.path);
     }
@@ -222,6 +335,8 @@ public class TestUtils {
 
     private static final String LIFECAM_240P_CAL_FILE = "lifecam240p.json";
     private static final String LIFECAM_480P_CAL_FILE = "lifecam480p.json";
+    public static final String LIFECAM_1280P_CAL_FILE = "lifecam_1280.json";
+    public static final String LIMELIGHT_480P_CAL_FILE = "limelight_1280_720.json";
 
     public static CameraCalibrationCoefficients getCoeffs(String filename, boolean testMode) {
         try {
@@ -243,17 +358,14 @@ public class TestUtils {
         return getCoeffs(LIFECAM_480P_CAL_FILE, testMode);
     }
 
-    public static void loadLibraries() {
-        try {
-            CameraServerCvJNI.forceLoad();
-        } catch (IOException e) {
-            // ignored
-        }
+    public static CameraCalibrationCoefficients getLaptop() {
+        return getCoeffs("laptop.json", true);
     }
 
     private static int DefaultTimeoutMillis = 5000;
 
     public static void showImage(Mat frame, String title, int timeoutMs) {
+        if (frame.empty()) return;
         try {
             HighGui.imshow(title, frame);
             HighGui.waitKey(timeoutMs);
@@ -273,4 +385,14 @@ public class TestUtils {
     public static void showImage(Mat frame) {
         showImage(frame, DefaultTimeoutMillis);
     }
+
+    public static Path getTestMode2023ImagePath() {
+        return getResourcesFolderPath(true)
+                .resolve("testimages")
+                .resolve(WPI2022Image.kTerminal22ft6in.path);
+    }
+
+    public static CameraCalibrationCoefficients get2023LifeCamCoeffs(boolean testMode) {
+        return getCoeffs(LIFECAM_1280P_CAL_FILE, testMode);
+    }
 }

photon-core/src/main/java/org/photonvision/common/util/file/FileUtils.java

@@ -78,7 +78,7 @@ public class FileUtils {
     }
 
     public static void setFilePerms(Path path) throws IOException {
-        if (!Platform.currentPlatform.isWindows()) {
+        if (Platform.isLinux()) {
             File thisFile = path.toFile();
             Set<PosixFilePermission> perms =
                     Files.readAttributes(path, PosixFileAttributes.class).permissions();
@@ -96,7 +96,7 @@ public class FileUtils {
     }
 
     public static void setAllPerms(Path path) {
-        if (!Platform.currentPlatform.isWindows()) {
+        if (Platform.isLinux()) {
             String command = String.format("chmod 777 -R %s", path.toString());
             try {
                 Process p = Runtime.getRuntime().exec(command);

photon-core/src/main/java/org/photonvision/common/util/file/JacksonUtils.java

@@ -31,10 +31,13 @@ import java.io.FileDescriptor;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.HashMap;
 
 public class JacksonUtils {
+    public static class UIMap extends HashMap<String, Object> {}
+
     public static <T> void serialize(Path path, T object) throws IOException {
-        serialize(path, object, false);
+        serialize(path, object, true);
     }
 
     public static <T> void serialize(Path path, T object, boolean forceSync) throws IOException {
@@ -80,7 +83,7 @@ public class JacksonUtils {
 
     public static <T> void serialize(Path path, T object, Class<T> ref, StdSerializer<T> serializer)
             throws IOException {
-        serialize(path, object, ref, serializer, false);
+        serialize(path, object, ref, serializer, true);
     }
 
     public static <T> void serialize(

photon-core/src/main/java/org/photonvision/common/util/math/MathUtils.java

@@ -17,9 +17,24 @@
 
 package org.photonvision.common.util.math;
 
+import edu.wpi.first.math.MatBuilder;
+import edu.wpi.first.math.Matrix;
+import edu.wpi.first.math.Nat;
+import edu.wpi.first.math.VecBuilder;
+import edu.wpi.first.math.geometry.CoordinateSystem;
+import edu.wpi.first.math.geometry.Pose3d;
+import edu.wpi.first.math.geometry.Quaternion;
+import edu.wpi.first.math.geometry.Rotation3d;
+import edu.wpi.first.math.geometry.Transform3d;
+import edu.wpi.first.math.numbers.N3;
+import edu.wpi.first.math.util.Units;
 import edu.wpi.first.util.WPIUtilJNI;
 import java.util.Arrays;
 import java.util.List;
+import org.ejml.data.DMatrixRMaj;
+import org.ejml.dense.row.factory.DecompositionFactory_DDRM;
+import org.ejml.simple.SimpleMatrix;
+import org.opencv.core.Mat;
 
 public class MathUtils {
     MathUtils() {}
@@ -89,7 +104,7 @@ public class MathUtils {
             return list.get(0); // always return single value for n = 1
         }
 
-        // Sort array.  We avoid a third copy here by just creating the
+        // Sort array. We avoid a third copy here by just creating the
         // list directly.
         double[] sorted = new double[list.size()];
         for (int i = 0; i < list.size(); i++) {
@@ -130,4 +145,125 @@ public class MathUtils {
     public static double lerp(double startValue, double endValue, double t) {
         return startValue + (endValue - startValue) * t;
     }
+
+    public static Pose3d EDNtoNWU(final Pose3d pose) {
+        // Change of basis matrix from EDN to NWU. Each column vector is one of the
+        // old basis vectors mapped to its representation in the new basis.
+        //
+        // E (+X) -> N (-Y), D (+Y) -> W (-Z), N (+Z) -> U (+X)
+        var R = new MatBuilder<>(Nat.N3(), Nat.N3()).fill(0, 0, 1, -1, 0, 0, 0, -1, 0);
+
+        // https://www.euclideanspace.com/maths/geometry/rotations/conversions/matrixToQuaternion/
+        double w = Math.sqrt(1.0 + R.get(0, 0) + R.get(1, 1) + R.get(2, 2)) / 2.0;
+        double x = (R.get(2, 1) - R.get(1, 2)) / (4.0 * w);
+        double y = (R.get(0, 2) - R.get(2, 0)) / (4.0 * w);
+        double z = (R.get(1, 0) - R.get(0, 1)) / (4.0 * w);
+        var rotationQuat = new Rotation3d(new Quaternion(w, x, y, z));
+
+        return new Pose3d(
+                pose.getTranslation().rotateBy(rotationQuat), pose.getRotation().rotateBy(rotationQuat));
+    }
+
+    /**
+     * All our solvepnp code returns a tag with X left, Y up, and Z out of the tag To better match
+     * wpilib, we want to apply another rotation so that we get Z up, X out of the tag, and Y to the
+     * right. We apply the following change of basis: X -> Y Y -> Z Z -> X
+     */
+    private static final Rotation3d WPILIB_BASE_ROTATION =
+            new Rotation3d(new MatBuilder<>(Nat.N3(), Nat.N3()).fill(0, 1, 0, 0, 0, 1, 1, 0, 0));
+
+    public static Transform3d convertOpenCVtoPhotonTransform(Transform3d cameraToTarget3d) {
+        // TODO: Refactor into new pipe?
+        // CameraToTarget _should_ be in opencv-land EDN
+        var nwu =
+                CoordinateSystem.convert(
+                        new Pose3d().transformBy(cameraToTarget3d),
+                        CoordinateSystem.EDN(),
+                        CoordinateSystem.NWU());
+        return new Transform3d(nwu.getTranslation(), WPILIB_BASE_ROTATION.rotateBy(nwu.getRotation()));
+    }
+
+    public static Pose3d convertOpenCVtoPhotonPose(Transform3d cameraToTarget3d) {
+        // TODO: Refactor into new pipe?
+        // CameraToTarget _should_ be in opencv-land EDN
+        var nwu =
+                CoordinateSystem.convert(
+                        new Pose3d().transformBy(cameraToTarget3d),
+                        CoordinateSystem.EDN(),
+                        CoordinateSystem.NWU());
+        return new Pose3d(nwu.getTranslation(), WPILIB_BASE_ROTATION.rotateBy(nwu.getRotation()));
+    }
+
+    /*
+     * The AprilTag pose rotation outputs are X left, Y down, Z away from the tag
+     * with the tag facing
+     * the camera upright and the camera facing the target parallel to the floor.
+     * But our OpenCV
+     * solvePNP code would have X left, Y up, Z towards the camera with the target
+     * facing the camera
+     * and both parallel to the floor. So we apply a base rotation to the rotation
+     * component of the
+     * apriltag pose to make it consistent with the EDN system that OpenCV uses,
+     * internally a 180
+     * rotation about the X axis
+     */
+    private static final Rotation3d APRILTAG_BASE_ROTATION =
+            new Rotation3d(VecBuilder.fill(1, 0, 0), Units.degreesToRadians(180));
+
+    /**
+     * Apply a 180 degree rotation about X to the rotation component of a given Apriltag pose. This
+     * aligns it with the OpenCV poses we use in other places.
+     */
+    public static Transform3d convertApriltagtoOpenCV(Transform3d pose) {
+        var ocvRotation = APRILTAG_BASE_ROTATION.rotateBy(pose.getRotation());
+        return new Transform3d(pose.getTranslation(), ocvRotation);
+    }
+
+    public static Pose3d convertArucotoOpenCV(Transform3d pose) {
+        var ocvRotation =
+                APRILTAG_BASE_ROTATION.rotateBy(
+                        new Rotation3d(VecBuilder.fill(0, 0, 1), Units.degreesToRadians(180))
+                                .rotateBy(pose.getRotation()));
+        return new Pose3d(pose.getTranslation(), ocvRotation);
+    }
+
+    public static void rotationToOpencvRvec(Rotation3d rotation, Mat rvecOutput) {
+        var angle = rotation.getAngle();
+        var axis = rotation.getAxis().times(angle);
+        rvecOutput.put(0, 0, axis.getData());
+    }
+
+    /**
+     * Orthogonalize an input matrix using a QR decomposition. QR decompositions decompose a
+     * rectangular matrix 'A' such that 'A=QR', where Q is the closest orthogonal matrix to the input,
+     * and R is an upper triangular matrix.
+     *
+     * <p>The following function is released under the BSD license avaliable in
+     * LICENSE_MathUtils_orthogonalizeRotationMatrix.txt.
+     */
+    public static Matrix<N3, N3> orthogonalizeRotationMatrix(Matrix<N3, N3> input) {
+        var a = DecompositionFactory_DDRM.qr(3, 3);
+        if (!a.decompose(input.getStorage().getDDRM())) {
+            // best we can do is return the input
+            return input;
+        }
+
+        // Grab results (thanks for this _great_ api, EJML)
+        var Q = new DMatrixRMaj(3, 3);
+        var R = new DMatrixRMaj(3, 3);
+        a.getQ(Q, false);
+        a.getR(R, false);
+
+        // Fix signs in R if they're < 0 so it's close to an identity matrix
+        // (our QR decomposition implementation sometimes flips the signs of columns)
+        for (int colR = 0; colR < 3; ++colR) {
+            if (R.get(colR, colR) < 0) {
+                for (int rowQ = 0; rowQ < 3; ++rowQ) {
+                    Q.set(rowQ, colR, -Q.get(rowQ, colR));
+                }
+            }
+        }
+
+        return new Matrix<>(new SimpleMatrix(Q));
+    }
 }

photon-core/src/main/java/org/photonvision/raspi/LibCameraJNI.java

@@ -0,0 +1,191 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.raspi;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.photonvision.common.logging.LogGroup;
+import org.photonvision.common.logging.Logger;
+
+public class LibCameraJNI {
+    private static boolean libraryLoaded = false;
+    private static Logger logger = new Logger(LibCameraJNI.class, LogGroup.Camera);
+
+    public static final Object CAMERA_LOCK = new Object();
+
+    public static synchronized void forceLoad() throws IOException {
+        if (libraryLoaded) return;
+
+        try {
+            File libDirectory = Path.of("lib/").toFile();
+            if (!libDirectory.exists()) {
+                Files.createDirectory(libDirectory.toPath()).toFile();
+            }
+
+            // We always extract the shared object (we could hash each so, but that's a lot of work)
+            URL resourceURL = LibCameraJNI.class.getResource("/nativelibraries/libphotonlibcamera.so");
+            File libFile = Path.of("lib/libphotonlibcamera.so").toFile();
+            try (InputStream in = resourceURL.openStream()) {
+                if (libFile.exists()) Files.delete(libFile.toPath());
+                Files.copy(in, libFile.toPath());
+            } catch (Exception e) {
+                logger.error("Could not extract the native library!");
+            }
+            System.load(libFile.getAbsolutePath());
+
+            libraryLoaded = true;
+            logger.info("Successfully loaded libpicam shared object");
+        } catch (UnsatisfiedLinkError e) {
+            logger.error("Couldn't load libpicam shared object");
+            e.printStackTrace();
+        }
+    }
+
+    public enum SensorModel {
+        Disconnected,
+        OV5647, // Picam v1
+        IMX219, // Picam v2
+        IMX477, // Picam HQ
+        OV9281,
+        OV7251,
+        Unknown;
+
+        public String getFriendlyName() {
+            switch (this) {
+                case Disconnected:
+                    return "Disconnected Camera";
+                case OV5647:
+                    return "Camera Module v1";
+                case IMX219:
+                    return "Camera Module v2";
+                case IMX477:
+                    return "HQ Camera";
+                case OV9281:
+                    return "OV9281";
+                case OV7251:
+                    return "OV7251";
+                case Unknown:
+                default:
+                    return "Unknown Camera";
+            }
+        }
+    }
+
+    public static SensorModel getSensorModel() {
+        int model = getSensorModelRaw();
+        return SensorModel.values()[model];
+    }
+
+    public static boolean isSupported() {
+        return libraryLoaded
+                // && getSensorModel() != PicamJNI.SensorModel.Disconnected
+                // && Platform.isRaspberryPi()
+                && isLibraryWorking();
+    }
+
+    private static native boolean isLibraryWorking();
+
+    public static native int getSensorModelRaw();
+
+    // ======================================================== //
+
+    /**
+     * Creates a new runner with a given width/height/fps
+     *
+     * @param width Camera video mode width in pixels
+     * @param height Camera video mode height in pixels
+     * @param fps Camera video mode FPS
+     * @return success of creating a camera object
+     */
+    public static native boolean createCamera(int width, int height, int rotation);
+
+    /**
+     * Starts the camera thresholder and display threads running. Make sure that this function is
+     * called syncronously with stopCamera and returnFrame!
+     */
+    public static native boolean startCamera();
+
+    /** Stops the camera runner. Make sure to call prior to destroying the camera! */
+    public static native boolean stopCamera();
+
+    // Destroy all native resources associated with a camera. Ensure stop is called prior!
+    public static native boolean destroyCamera();
+
+    // ======================================================== //
+
+    // Set thresholds on [0..1]
+    public static native boolean setThresholds(
+            double hl, double sl, double vl, double hu, double su, double vu, boolean hueInverted);
+
+    public static native boolean setAutoExposure(boolean doAutoExposure);
+
+    // Exposure time, in microseconds
+    public static native boolean setExposure(int exposureUs);
+
+    // Set brighness on [-1, 1]
+    public static native boolean setBrightness(double brightness);
+
+    // Unknown ranges for red and blue AWB gain
+    public static native boolean setAwbGain(double red, double blue);
+
+    /**
+     * Get the time when the first pixel exposure was started, in the same timebase as libcamera gives
+     * the frame capture time. Units are nanoseconds.
+     */
+    public static native long getFrameCaptureTime();
+
+    /**
+     * Get the current time, in the same timebase as libcamera gives the frame capture time. Units are
+     * nanoseconds.
+     */
+    public static native long getLibcameraTimestamp();
+
+    public static native long setFramesToCopy(boolean copyIn, boolean copyOut);
+
+    // Analog gain multiplier to apply to all color channels, on [1, Big Number]
+    public static native boolean setAnalogGain(double analog);
+
+    /** Block until a new frame is avaliable from native code. */
+    public static native boolean awaitNewFrame();
+
+    /**
+     * Get a pointer to the most recent color mat generated. Call this immediatly after awaitNewFrame,
+     * and call onlly once per new frame!
+     */
+    public static native long takeColorFrame();
+
+    /**
+     * Get a pointer to the most recent processed mat generated. Call this immediatly after
+     * awaitNewFrame, and call onlly once per new frame!
+     */
+    public static native long takeProcessedFrame();
+
+    /**
+     * Set the GPU processing type we should do. Enum of [none, HSV, greyscale, adaptive threshold].
+     */
+    public static native boolean setGpuProcessType(int type);
+
+    public static native int getGpuProcessType();
+
+    // /** Release a frame pointer back to the libcamera driver code to be filled again */
+    // public static native long returnFrame(long frame);
+}

photon-core/src/main/java/org/photonvision/raspi/PicamJNI.java

@@ -1,156 +0,0 @@
-/*
- * Copyright (C) Photon Vision.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package org.photonvision.raspi;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import org.photonvision.common.hardware.PiVersion;
-import org.photonvision.common.hardware.Platform;
-import org.photonvision.common.logging.LogGroup;
-import org.photonvision.common.logging.Logger;
-
-public class PicamJNI {
-    private static boolean libraryLoaded = false;
-    private static Logger logger = new Logger(PicamJNI.class, LogGroup.Camera);
-
-    public enum SensorModel {
-        Disconnected,
-        OV5647, // Picam v1
-        IMX219, // Picam v2
-        IMX477, // Picam HQ
-        Unknown;
-
-        public String getFriendlyName() {
-            switch (this) {
-                case Disconnected:
-                    return "Disconnected Camera";
-                case OV5647:
-                    return "Camera Module v1";
-                case IMX219:
-                    return "Camera Module v2";
-                case IMX477:
-                    return "HQ Camera";
-                case Unknown:
-                default:
-                    return "Unknown Camera";
-            }
-        }
-    }
-
-    public static synchronized void forceLoad() throws IOException {
-        if (libraryLoaded || !Platform.isRaspberryPi()) return;
-
-        try {
-            File libDirectory = Path.of("lib/").toFile();
-            if (!libDirectory.exists()) {
-                Files.createDirectory(libDirectory.toPath()).toFile();
-            }
-
-            // We always extract the shared object (we could hash each so, but that's a lot of work)
-            URL resourceURL = PicamJNI.class.getResource("/nativelibraries/libpicam.so");
-            File libFile = Path.of("lib/libpicam.so").toFile();
-            try (InputStream in = resourceURL.openStream()) {
-                if (libFile.exists()) Files.delete(libFile.toPath());
-                Files.copy(in, libFile.toPath());
-            } catch (Exception e) {
-                logger.error("Could not extract the native library!");
-            }
-            System.load(libFile.getAbsolutePath());
-
-            libraryLoaded = true;
-            logger.info("Successfully loaded libpicam shared object");
-        } catch (UnsatisfiedLinkError e) {
-            logger.error("Couldn't load libpicam shared object");
-            e.printStackTrace();
-        }
-    }
-
-    public static boolean isSupported() {
-        return libraryLoaded
-                && isVCSMSupported()
-                && getSensorModel() != SensorModel.Disconnected
-                && Platform.isRaspberryPi()
-                && (Platform.currentPiVersion == PiVersion.PI_3
-                        || Platform.currentPiVersion == PiVersion.COMPUTE_MODULE_3
-                        || Platform.currentPiVersion == PiVersion.ZERO_2_W);
-    }
-
-    public static SensorModel getSensorModel() {
-        switch (getSensorModelRaw().toLowerCase()) {
-            case "":
-                return SensorModel.Disconnected;
-            case "ov5647":
-                return SensorModel.OV5647;
-            case "imx219":
-                return SensorModel.IMX219;
-            case "imx477":
-                return SensorModel.IMX477;
-            default:
-                return SensorModel.Unknown;
-        }
-    }
-
-    private static native String getSensorModelRaw();
-
-    // This is the main thing we need that isn't supported on Pi 4s, which makes it a good check
-    private static native boolean isVCSMSupported();
-
-    // Everything here is static because multiple picams are unsupported at the hardware level
-
-    /**
-     * Called once for each video mode change. Starts a native thread running MMAL that stays alive
-     * until destroyCamera is called.
-     *
-     * @return true on error.
-     */
-    public static native boolean createCamera(int width, int height, int fps);
-
-    /**
-     * Destroys MMAL and EGL contexts. Called once for each video mode change *before* createCamera.
-     *
-     * @return true on error.
-     */
-    public static native boolean destroyCamera();
-
-    public static native void setThresholds(
-            double hL, double sL, double vL, double hU, double sU, double vU);
-
-    public static native void setInvertHue(boolean shouldInvert);
-
-    public static native boolean setExposure(int exposure);
-
-    public static native boolean setBrightness(int brightness);
-
-    // This adjusts the analog gain (normalized to 0-100); ignores the digital gain
-    public static native boolean setGain(int gain);
-
-    // Adjusts the auto white balance gains, which are normalized 0-100 in the native code
-    public static native boolean setAwbGain(int red, int blue);
-
-    public static native boolean setRotation(int rotation);
-
-    public static native void setShouldCopyColor(boolean shouldCopyColor);
-
-    public static native long getFrameLatency();
-
-    public static native long grabFrame(boolean shouldReturnColor);
-}

photon-core/src/main/java/org/photonvision/vision/apriltag/AprilTagFamily.java

@@ -15,12 +15,20 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package org.photonvision.common.hardware.metrics;
+package org.photonvision.vision.apriltag;
 
-public class RAMMetrics extends MetricsBase {
-    // TODO: Output in MBs for consistency
-    public String getUsedRam() {
-        if (ramUsageCommand.isEmpty()) return "";
-        return execute(ramUsageCommand);
+public enum AprilTagFamily {
+    kTag36h11,
+    kTag25h9,
+    kTag16h5,
+    kTagCircle21h7,
+    kTagCircle49h12,
+    kTagStandard41h12,
+    kTagStandard52h13,
+    kTagCustom48h11;
+
+    public String getNativeName() {
+        // We wanna strip the leading kT and replace with "t"
+        return this.name().replaceFirst("kT", "t");
     }
 }

photon-core/src/main/java/org/photonvision/vision/aruco/ArucoDetectionResult.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.vision.aruco;
+
+import java.util.Arrays;
+import org.photonvision.common.logging.LogGroup;
+import org.photonvision.common.logging.Logger;
+
+public class ArucoDetectionResult {
+    private static final Logger logger =
+            new Logger(ArucoDetectionResult.class, LogGroup.VisionModule);
+    double[] xCorners;
+    double[] yCorners;
+
+    int id;
+
+    double[] tvec, rvec;
+
+    public ArucoDetectionResult(
+            double[] xCorners, double[] yCorners, int id, double[] tvec, double[] rvec) {
+        this.xCorners = xCorners;
+        this.yCorners = yCorners;
+        this.id = id;
+        this.tvec = tvec;
+        this.rvec = rvec;
+        // logger.debug("Creating a new detection result: " + this.toString());
+    }
+
+    public double[] getTvec() {
+        return tvec;
+    }
+
+    public double[] getRvec() {
+        return rvec;
+    }
+
+    public double[] getxCorners() {
+        return xCorners;
+    }
+
+    public double[] getyCorners() {
+        return yCorners;
+    }
+
+    public int getId() {
+        return id;
+    }
+
+    public double getCenterX() {
+        return (xCorners[0] + xCorners[1] + xCorners[2] + xCorners[3]) * .25;
+    }
+
+    public double getCenterY() {
+        return (yCorners[0] + yCorners[1] + yCorners[2] + yCorners[3]) * .25;
+    }
+
+    @Override
+    public String toString() {
+        return "ArucoDetectionResult{"
+                + "xCorners="
+                + Arrays.toString(xCorners)
+                + ", yCorners="
+                + Arrays.toString(yCorners)
+                + ", id="
+                + id
+                + '}';
+    }
+}

photon-core/src/main/java/org/photonvision/vision/aruco/ArucoDetectorParams.java

@@ -0,0 +1,76 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.vision.aruco;
+
+import org.opencv.aruco.Aruco;
+import org.opencv.aruco.ArucoDetector;
+import org.opencv.aruco.DetectorParameters;
+import org.opencv.aruco.Dictionary;
+import org.photonvision.common.logging.LogGroup;
+import org.photonvision.common.logging.Logger;
+
+public class ArucoDetectorParams {
+    private static final Logger logger = new Logger(PhotonArucoDetector.class, LogGroup.VisionModule);
+
+    private float m_decimate = -1;
+    private int m_iterations = -1;
+    private double m_accuracy = -1;
+
+    DetectorParameters parameters = DetectorParameters.create();
+    ArucoDetector detector;
+
+    public ArucoDetectorParams() {
+        setDecimation(1);
+        setCornerAccuracy(25);
+        setCornerRefinementMaxIterations(100);
+
+        detector = new ArucoDetector(Dictionary.get(Aruco.DICT_APRILTAG_16h5), parameters);
+    }
+
+    public void setDecimation(float decimate) {
+        if (decimate == m_decimate) return;
+
+        logger.info("Setting decimation from " + m_decimate + " to " + decimate);
+
+        // We only need to mutate the parameters -- the detector keeps a poitner to the parameters
+        // object internally, so it should automatically update
+        parameters.set_aprilTagQuadDecimate((float) decimate);
+        m_decimate = decimate;
+    }
+
+    public void setCornerRefinementMaxIterations(int iters) {
+        if (iters == m_iterations || iters <= 0) return;
+
+        parameters.set_cornerRefinementMethod(Aruco.CORNER_REFINE_SUBPIX);
+        parameters.set_cornerRefinementMaxIterations(iters); // 200
+
+        m_iterations = iters;
+    }
+
+    public void setCornerAccuracy(double accuracy) {
+        if (accuracy == m_accuracy || accuracy <= 0) return;
+
+        parameters.set_cornerRefinementMinAccuracy(
+                accuracy / 1000.0); // divides by 1000 because the UI multiplies it by 1000
+        m_accuracy = accuracy;
+    }
+
+    public ArucoDetector getDetector() {
+        return detector;
+    }
+}

photon-core/src/main/java/org/photonvision/vision/aruco/PhotonArucoDetector.java

@@ -0,0 +1,132 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.vision.aruco;
+
+import edu.wpi.first.math.VecBuilder;
+import edu.wpi.first.math.geometry.Pose3d;
+import edu.wpi.first.math.geometry.Rotation3d;
+import edu.wpi.first.math.geometry.Translation3d;
+import edu.wpi.first.math.util.Units;
+import java.util.ArrayList;
+import org.opencv.aruco.Aruco;
+import org.opencv.aruco.ArucoDetector;
+import org.opencv.core.Mat;
+import org.photonvision.common.logging.LogGroup;
+import org.photonvision.common.logging.Logger;
+import org.photonvision.vision.calibration.CameraCalibrationCoefficients;
+
+public class PhotonArucoDetector {
+    private static final Logger logger = new Logger(PhotonArucoDetector.class, LogGroup.VisionModule);
+
+    private static final Rotation3d ARUCO_BASE_ROTATION =
+            new Rotation3d(VecBuilder.fill(0, 0, 1), Units.degreesToRadians(180));
+
+    Mat ids;
+
+    Mat tvecs;
+    Mat rvecs;
+    ArrayList<Mat> corners;
+
+    Mat cornerMat;
+    Translation3d translation;
+    Rotation3d rotation;
+    double timeStartDetect;
+    double timeEndDetect;
+    Pose3d tagPose;
+    double timeStartProcess;
+    double timeEndProcess;
+    double[] xCorners = new double[4];
+    double[] yCorners = new double[4];
+
+    public PhotonArucoDetector() {
+        logger.debug("New Aruco Detector");
+        ids = new Mat();
+        tvecs = new Mat();
+        rvecs = new Mat();
+        corners = new ArrayList<Mat>();
+        tagPose = new Pose3d();
+        translation = new Translation3d();
+        rotation = new Rotation3d();
+    }
+
+    public ArucoDetectionResult[] detect(
+            Mat grayscaleImg,
+            float tagSize,
+            CameraCalibrationCoefficients coeffs,
+            ArucoDetector detector) {
+        detector.detectMarkers(grayscaleImg, corners, ids);
+        if (coeffs != null) {
+            Aruco.estimatePoseSingleMarkers(
+                    corners,
+                    tagSize,
+                    coeffs.getCameraIntrinsicsMat(),
+                    coeffs.getDistCoeffsMat(),
+                    rvecs,
+                    tvecs);
+        }
+
+        ArucoDetectionResult[] toReturn = new ArucoDetectionResult[corners.size()];
+        timeStartProcess = System.currentTimeMillis();
+        for (int i = 0; i < corners.size(); i++) {
+            cornerMat = corners.get(i);
+            // logger.debug(cornerMat.dump());
+            xCorners =
+                    new double[] {
+                        cornerMat.get(0, 0)[0],
+                        cornerMat.get(0, 1)[0],
+                        cornerMat.get(0, 2)[0],
+                        cornerMat.get(0, 3)[0]
+                    };
+            yCorners =
+                    new double[] {
+                        cornerMat.get(0, 0)[1],
+                        cornerMat.get(0, 1)[1],
+                        cornerMat.get(0, 2)[1],
+                        cornerMat.get(0, 3)[1]
+                    };
+            cornerMat.release();
+
+            double[] tvec;
+            double[] rvec;
+            if (coeffs != null) {
+                // Need to apply a 180 rotation about Z
+                var origRvec = rvecs.get(i, 0);
+                var axisangle = VecBuilder.fill(origRvec[0], origRvec[1], origRvec[2]);
+                Rotation3d rotation = new Rotation3d(axisangle, axisangle.normF());
+                var ocvRotation = ARUCO_BASE_ROTATION.rotateBy(rotation);
+
+                var angle = ocvRotation.getAngle();
+                var finalAxisAngle = ocvRotation.getAxis().times(angle);
+
+                tvec = tvecs.get(i, 0);
+                rvec = finalAxisAngle.getData();
+            } else {
+                tvec = new double[] {0, 0, 0};
+                rvec = new double[] {0, 0, 0};
+            }
+
+            toReturn[i] =
+                    new ArucoDetectionResult(xCorners, yCorners, (int) ids.get(i, 0)[0], tvec, rvec);
+        }
+        rvecs.release();
+        tvecs.release();
+        ids.release();
+
+        return toReturn;
+    }
+}

photon-core/src/main/java/org/photonvision/vision/calibration/CameraCalibrationCoefficients.java

@@ -17,9 +17,11 @@
 
 package org.photonvision.vision.calibration;
 
+import com.fasterxml.jackson.annotation.JsonAlias;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.JsonNode;
 import org.opencv.core.Mat;
 import org.opencv.core.MatOfDouble;
 import org.opencv.core.Size;
@@ -33,7 +35,8 @@ public class CameraCalibrationCoefficients implements Releasable {
     public final JsonMat cameraIntrinsics;
 
     @JsonProperty("cameraExtrinsics")
-    public final JsonMat cameraExtrinsics;
+    @JsonAlias({"cameraExtrinsics", "distCoeffs"})
+    public final JsonMat distCoeffs;
 
     @JsonProperty("perViewErrors")
     public final double[] perViewErrors;
@@ -45,12 +48,12 @@ public class CameraCalibrationCoefficients implements Releasable {
     public CameraCalibrationCoefficients(
             @JsonProperty("resolution") Size resolution,
             @JsonProperty("cameraIntrinsics") JsonMat cameraIntrinsics,
-            @JsonProperty("cameraExtrinsics") JsonMat cameraExtrinsics,
+            @JsonProperty("cameraExtrinsics") JsonMat distCoeffs,
             @JsonProperty("perViewErrors") double[] perViewErrors,
             @JsonProperty("standardDeviation") double standardDeviation) {
         this.resolution = resolution;
         this.cameraIntrinsics = cameraIntrinsics;
-        this.cameraExtrinsics = cameraExtrinsics;
+        this.distCoeffs = distCoeffs;
         this.perViewErrors = perViewErrors;
         this.standardDeviation = standardDeviation;
     }
@@ -61,8 +64,8 @@ public class CameraCalibrationCoefficients implements Releasable {
     }
 
     @JsonIgnore
-    public MatOfDouble getCameraExtrinsicsMat() {
-        return cameraExtrinsics.getAsMatOfDouble();
+    public MatOfDouble getDistCoeffsMat() {
+        return distCoeffs.getAsMatOfDouble();
     }
 
     @JsonIgnore
@@ -78,6 +81,45 @@ public class CameraCalibrationCoefficients implements Releasable {
     @Override
     public void release() {
         cameraIntrinsics.release();
-        cameraExtrinsics.release();
+        distCoeffs.release();
+    }
+
+    public static CameraCalibrationCoefficients parseFromCalibdbJson(JsonNode json) {
+        // camera_matrix is a row major, array of arrays
+        var cam_matrix = json.get("camera_matrix");
+
+        double[] cam_arr =
+                new double[] {
+                    cam_matrix.get(0).get(0).doubleValue(),
+                    cam_matrix.get(0).get(1).doubleValue(),
+                    cam_matrix.get(0).get(2).doubleValue(),
+                    cam_matrix.get(1).get(0).doubleValue(),
+                    cam_matrix.get(1).get(1).doubleValue(),
+                    cam_matrix.get(1).get(2).doubleValue(),
+                    cam_matrix.get(2).get(0).doubleValue(),
+                    cam_matrix.get(2).get(1).doubleValue(),
+                    cam_matrix.get(2).get(2).doubleValue()
+                };
+
+        var dist_coefs = json.get("distortion_coefficients");
+
+        double[] dist_array =
+                new double[] {
+                    dist_coefs.get(0).doubleValue(),
+                    dist_coefs.get(1).doubleValue(),
+                    dist_coefs.get(2).doubleValue(),
+                    dist_coefs.get(3).doubleValue(),
+                    dist_coefs.get(4).doubleValue(),
+                };
+
+        var cam_jsonmat = new JsonMat(3, 3, cam_arr);
+        var distortion_jsonmat = new JsonMat(1, 5, dist_array);
+
+        var error = json.get("avg_reprojection_error").asDouble();
+        var width = json.get("img_size").get(0).doubleValue();
+        var height = json.get("img_size").get(1).doubleValue();
+
+        return new CameraCalibrationCoefficients(
+                new Size(width, height), cam_jsonmat, distortion_jsonmat, new double[] {error}, 0);
     }
 }

photon-core/src/main/java/org/photonvision/vision/camera/CameraQuirk.java

@@ -23,5 +23,11 @@ public enum CameraQuirk {
     /** For the Raspberry Pi Camera */
     PiCam,
     /** Cap at 100FPS for high-bandwidth cameras */
-    FPSCap100
+    FPSCap100,
+    /** Separate red/blue gain controls available */
+    AWBGain,
+    /** Will not work with photonvision - Logitec C270 at least */
+    CompletelyBroken,
+    /** Has adjustable focus and autofocus switch */
+    AdjustableFocus,
 }

photon-core/src/main/java/org/photonvision/vision/camera/FileVisionSource.java

@@ -43,7 +43,6 @@ public class FileVisionSource extends VisionSource {
                         Path.of(cameraConfiguration.path),
                         cameraConfiguration.FOV,
                         FileFrameProvider.MAX_FPS,
-                        cameraConfiguration.camPitch,
                         calibration);
         settables =
                 new FileSourceSettables(cameraConfiguration, frameProvider.get().frameStaticProperties);
@@ -92,6 +91,8 @@ public class FileVisionSource extends VisionSource {
         @Override
         public void setExposure(double exposure) {}
 
+        public void setAutoExposure(boolean cameraAutoExposure) {}
+
         @Override
         public void setBrightness(int brightness) {}
 

photon-core/src/main/java/org/photonvision/vision/camera/LibcameraGpuSettables.java

@@ -0,0 +1,242 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.vision.camera;
+
+import edu.wpi.first.cscore.VideoMode;
+import edu.wpi.first.math.Pair;
+import java.util.HashMap;
+import org.photonvision.common.configuration.CameraConfiguration;
+import org.photonvision.common.util.math.MathUtils;
+import org.photonvision.raspi.LibCameraJNI;
+import org.photonvision.vision.camera.LibcameraGpuSource.FPSRatedVideoMode;
+import org.photonvision.vision.opencv.ImageRotationMode;
+import org.photonvision.vision.processes.VisionSourceSettables;
+
+public class LibcameraGpuSettables extends VisionSourceSettables {
+    private FPSRatedVideoMode currentVideoMode;
+    private double lastManualExposure = 50;
+    private int lastBrightness = 50;
+    private boolean lastAutoExposureActive;
+    private int lastGain = 50;
+    private Pair<Integer, Integer> lastAwbGains = new Pair<>(18, 18);
+    private boolean m_initialized = false;
+
+    private final LibCameraJNI.SensorModel sensorModel;
+
+    private ImageRotationMode m_rotationMode;
+
+    public void setRotation(ImageRotationMode rotationMode) {
+        if (rotationMode != m_rotationMode) {
+            m_rotationMode = rotationMode;
+
+            setVideoModeInternal(getCurrentVideoMode());
+        }
+    }
+
+    public LibcameraGpuSettables(CameraConfiguration configuration) {
+        super(configuration);
+
+        videoModes = new HashMap<>();
+
+        sensorModel = LibCameraJNI.getSensorModel();
+
+        if (sensorModel == LibCameraJNI.SensorModel.IMX219) {
+            // Settings for the IMX219 sensor, which is used on the Pi Camera Module v2
+            videoModes.put(
+                    0, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 320, 240, 120, 120, .39));
+            videoModes.put(
+                    1, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 320, 240, 30, 30, .39));
+            videoModes.put(
+                    2, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 640, 480, 65, 90, .39));
+            videoModes.put(
+                    3, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 640, 480, 30, 30, .39));
+            // TODO: fix 1280x720 in the native code and re-add it
+            videoModes.put(
+                    4, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1920, 1080, 15, 20, .53));
+            videoModes.put(
+                    5, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 3280 / 2, 2464 / 2, 15, 20, 1));
+            videoModes.put(
+                    6, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 3280 / 4, 2464 / 4, 15, 20, 1));
+        } else if (sensorModel == LibCameraJNI.SensorModel.OV9281) {
+            videoModes.put(
+                    0, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 320, 240, 30, 30, .39));
+            videoModes.put(
+                    1, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1280 / 2, 800 / 2, 60, 60, 1));
+            videoModes.put(
+                    2, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 640, 480, 65, 90, .39));
+            videoModes.put(
+                    3, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1280, 800, 60, 60, 1));
+
+        } else {
+            if (sensorModel == LibCameraJNI.SensorModel.IMX477) {
+                LibcameraGpuSource.logger.warn(
+                        "It appears you are using a Pi HQ Camera. This camera is not officially supported. You will have to set your camera FOV differently based on resolution.");
+            } else if (sensorModel == LibCameraJNI.SensorModel.Unknown) {
+                LibcameraGpuSource.logger.warn(
+                        "You have an unknown sensor connected to your Pi over CSI! This is likely a bug. If it is not, then you will have to set your camera FOV differently based on resolution.");
+            }
+
+            // Settings for the OV5647 sensor, which is used by the Pi Camera Module v1
+            videoModes.put(0, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 320, 240, 90, 90, 1));
+            videoModes.put(1, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 640, 480, 85, 90, 1));
+            videoModes.put(
+                    2, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 960, 720, 45, 49, 0.74));
+            // Half the size of the active areas on the OV5647
+            videoModes.put(
+                    3, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 2592 / 2, 1944 / 2, 20, 20, 1));
+            videoModes.put(
+                    4, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1280, 720, 30, 45, 0.91));
+            videoModes.put(
+                    5, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1920, 1080, 15, 20, 0.72));
+        }
+
+        // TODO need to add more video modes for new sensors here
+
+        currentVideoMode = (FPSRatedVideoMode) videoModes.get(0);
+    }
+
+    @Override
+    public double getFOV() {
+        return getCurrentVideoMode().fovMultiplier * getConfiguration().FOV;
+    }
+
+    @Override
+    public void setAutoExposure(boolean cameraAutoExposure) {
+        lastAutoExposureActive = cameraAutoExposure;
+        LibCameraJNI.setAutoExposure(cameraAutoExposure);
+    }
+
+    @Override
+    public void setExposure(double exposure) {
+        if (exposure < 0.0 || lastAutoExposureActive) {
+            // Auto-exposure is active right now, don't set anything.
+            return;
+        }
+
+        // HACKS!
+        // If we set exposure too low, libcamera crashes or slows down
+        // Very weird and smelly
+        // For now, band-aid this by just not setting it lower than the "it breaks" limit
+        // Limit is different depending on camera.
+        if (sensorModel == LibCameraJNI.SensorModel.OV9281) {
+            if (exposure < 6.0) {
+                exposure = 6.0;
+            }
+        } else if (sensorModel == LibCameraJNI.SensorModel.OV5647) {
+            if (exposure < 0.7) {
+                exposure = 0.7;
+            }
+        }
+
+        lastManualExposure = exposure;
+        var success = LibCameraJNI.setExposure((int) Math.round(exposure) * 800);
+        if (!success) LibcameraGpuSource.logger.warn("Couldn't set Pi Camera exposure");
+    }
+
+    @Override
+    public void setBrightness(int brightness) {
+        lastBrightness = brightness;
+        double realBrightness = MathUtils.map(brightness, 0.0, 100.0, -1.0, 1.0);
+        var success = LibCameraJNI.setBrightness(realBrightness);
+        if (!success) LibcameraGpuSource.logger.warn("Couldn't set Pi Camera brightness");
+    }
+
+    @Override
+    public void setGain(int gain) {
+        lastGain = gain;
+        // TODO units here seem odd -- 5ish seems legit? So divide by 10
+        var success = LibCameraJNI.setAnalogGain(gain / 10.0);
+        if (!success) LibcameraGpuSource.logger.warn("Couldn't set Pi Camera gain");
+    }
+
+    @Override
+    public void setRedGain(int red) {
+        if (sensorModel != LibCameraJNI.SensorModel.OV9281) {
+            lastAwbGains = Pair.of(red, lastAwbGains.getSecond());
+            setAwbGain(lastAwbGains.getFirst(), lastAwbGains.getSecond());
+        }
+    }
+
+    @Override
+    public void setBlueGain(int blue) {
+        if (sensorModel != LibCameraJNI.SensorModel.OV9281) {
+            lastAwbGains = Pair.of(lastAwbGains.getFirst(), blue);
+            setAwbGain(lastAwbGains.getFirst(), lastAwbGains.getSecond());
+        }
+    }
+
+    public void setAwbGain(int red, int blue) {
+        if (sensorModel != LibCameraJNI.SensorModel.OV9281) {
+            var success = LibCameraJNI.setAwbGain(red / 10.0, blue / 10.0);
+            if (!success) LibcameraGpuSource.logger.warn("Couldn't set Pi Camera AWB gains");
+        }
+    }
+
+    @Override
+    public FPSRatedVideoMode getCurrentVideoMode() {
+        return currentVideoMode;
+    }
+
+    @Override
+    protected void setVideoModeInternal(VideoMode videoMode) {
+        var mode = (FPSRatedVideoMode) videoMode;
+
+        // We need to make sure that other threads don't try to do anything funny while we're recreating
+        // the camera
+        synchronized (LibCameraJNI.CAMERA_LOCK) {
+            boolean success = false;
+            if (m_initialized) {
+                success |= LibCameraJNI.stopCamera();
+                success |= LibCameraJNI.destroyCamera();
+            }
+
+            // if (!success) {
+            //     throw new RuntimeException(
+            //             "Couldn't destroy a zero copy Pi Camera while switching video modes");
+            // }
+
+            System.out.println("Starting camera");
+            success |=
+                    LibCameraJNI.createCamera(
+                            mode.width, mode.height, (m_rotationMode == ImageRotationMode.DEG_180 ? 180 : 0));
+            success |= LibCameraJNI.startCamera();
+            if (!success) {
+                throw new RuntimeException(
+                        "Couldn't create a zero copy Pi Camera while switching video modes");
+            }
+            m_initialized = true;
+        }
+
+        // We don't store last settings on the native side, and when you change video mode these get
+        // reset on MMAL's end
+        setExposure(lastManualExposure);
+        setAutoExposure(lastAutoExposureActive);
+        setBrightness(lastBrightness);
+        setGain(lastGain);
+        setAwbGain(lastAwbGains.getFirst(), lastAwbGains.getSecond());
+
+        LibCameraJNI.setFramesToCopy(true, true);
+
+        currentVideoMode = mode;
+    }
+
+    @Override
+    public HashMap<Integer, VideoMode> getAllVideoModes() {
+        return videoModes;
+    }
+}

photon-core/src/main/java/org/photonvision/vision/camera/LibcameraGpuSource.java

@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.vision.camera;
+
+import edu.wpi.first.cscore.VideoMode;
+import org.photonvision.common.configuration.CameraConfiguration;
+import org.photonvision.common.configuration.ConfigManager;
+import org.photonvision.common.logging.LogGroup;
+import org.photonvision.common.logging.Logger;
+import org.photonvision.vision.frame.FrameProvider;
+import org.photonvision.vision.frame.provider.LibcameraGpuFrameProvider;
+import org.photonvision.vision.processes.VisionSource;
+import org.photonvision.vision.processes.VisionSourceSettables;
+
+public class LibcameraGpuSource extends VisionSource {
+    static final Logger logger = new Logger(LibcameraGpuSource.class, LogGroup.Camera);
+
+    private final LibcameraGpuSettables settables;
+    private final LibcameraGpuFrameProvider frameProvider;
+
+    public LibcameraGpuSource(CameraConfiguration configuration) {
+        super(configuration);
+        if (configuration.cameraType != CameraType.ZeroCopyPicam) {
+            throw new IllegalArgumentException(
+                    "GPUAcceleratedPicamSource only accepts CameraConfigurations with type Picam");
+        }
+
+        settables = new LibcameraGpuSettables(configuration);
+        frameProvider = new LibcameraGpuFrameProvider(settables);
+    }
+
+    @Override
+    public FrameProvider getFrameProvider() {
+        return frameProvider;
+    }
+
+    @Override
+    public VisionSourceSettables getSettables() {
+        return settables;
+    }
+
+    /**
+     * On the OV5649 the actual FPS we want to request from the GPU can be higher than the FPS that we
+     * can do after processing. On the IMX219 these FPSes match pretty closely, except for the
+     * 1280x720 mode. We use this to present a rated FPS to the user that's lower than the actual FPS
+     * we request from the GPU. This is important for setting user expectations, and is also used by
+     * the frontend to detect and explain FPS drops. This class should ONLY be used by Picam video
+     * modes! This is to make sure it shows up nice in the frontend
+     */
+    public static class FPSRatedVideoMode extends VideoMode {
+        public final int fpsActual;
+        public final double fovMultiplier;
+
+        public FPSRatedVideoMode(
+                PixelFormat pixelFormat,
+                int width,
+                int height,
+                int ratedFPS,
+                int actualFPS,
+                double fovMultiplier) {
+            super(pixelFormat, width, height, ratedFPS);
+
+            this.fpsActual = actualFPS;
+            this.fovMultiplier = fovMultiplier;
+        }
+    }
+
+    @Override
+    public boolean isVendorCamera() {
+        return ConfigManager.getInstance().getConfig().getHardwareConfig().hasPresetFOV();
+    }
+}

photon-core/src/main/java/org/photonvision/vision/camera/QuirkyCamera.java

@@ -24,8 +24,27 @@ import java.util.Objects;
 public class QuirkyCamera {
     private static final List<QuirkyCamera> quirkyCameras =
             List.of(
+                    new QuirkyCamera(
+                            0x9331,
+                            0x5A3,
+                            CameraQuirk.CompletelyBroken), // Chris's older generic "Logitec HD Webcam"
+                    new QuirkyCamera(0x825, 0x46D, CameraQuirk.CompletelyBroken), // Logitec C270
+                    new QuirkyCamera(
+                            0x0bda,
+                            0x5510,
+                            CameraQuirk.CompletelyBroken), // A laptop internal camera someone found broken
+                    new QuirkyCamera(
+                            -1, -1, "Snap Camera", CameraQuirk.CompletelyBroken), // SnapCamera on Windows
+                    new QuirkyCamera(
+                            -1,
+                            -1,
+                            "FaceTime HD Camera",
+                            CameraQuirk.CompletelyBroken), // Mac Facetime Camera shared into Windows in Bootcamp
                     new QuirkyCamera(0x2000, 0x1415, CameraQuirk.Gain, CameraQuirk.FPSCap100), // PS3Eye
-                    new QuirkyCamera(-1, -1, "mmal service 16.1", CameraQuirk.PiCam) // PiCam (via V4L2)
+                    new QuirkyCamera(
+                            -1, -1, "mmal service 16.1", CameraQuirk.PiCam), // PiCam (via V4L2, not zerocopy)
+                    new QuirkyCamera(-1, -1, "unicam", CameraQuirk.PiCam), // PiCam (via V4L2, not zerocopy)
+                    new QuirkyCamera(0x85B, 0x46D, CameraQuirk.AdjustableFocus) // Logitech C925-e
                     );
 
     public static final QuirkyCamera DefaultCamera = new QuirkyCamera(0, 0, "");
@@ -35,7 +54,8 @@ public class QuirkyCamera {
                     -1,
                     "mmal service 16.1",
                     CameraQuirk.PiCam,
-                    CameraQuirk.Gain); // PiCam (special zerocopy version)
+                    CameraQuirk.Gain,
+                    CameraQuirk.AWBGain); // PiCam (special zerocopy version)
 
     public final String baseName;
     public final int usbVid;

photon-core/src/main/java/org/photonvision/vision/camera/USBCameraSource.java

@@ -18,10 +18,7 @@
 package org.photonvision.vision.camera;
 
 import edu.wpi.first.cameraserver.CameraServer;
-import edu.wpi.first.cscore.CvSink;
-import edu.wpi.first.cscore.UsbCamera;
-import edu.wpi.first.cscore.VideoException;
-import edu.wpi.first.cscore.VideoMode;
+import edu.wpi.first.cscore.*;
 import java.util.*;
 import java.util.stream.Collectors;
 import org.photonvision.common.configuration.CameraConfiguration;
@@ -44,9 +41,10 @@ public class USBCameraSource extends VisionSource {
 
     public USBCameraSource(CameraConfiguration config) {
         super(config);
+
         logger = new Logger(USBCameraSource.class, config.nickname, LogGroup.Camera);
         camera = new UsbCamera(config.nickname, config.path);
-        cvSink = CameraServer.getInstance().getVideo(this.camera);
+        cvSink = CameraServer.getVideo(this.camera);
 
         cameraQuirks =
                 QuirkyCamera.getQuirkyCamera(
@@ -56,19 +54,34 @@ public class USBCameraSource extends VisionSource {
             logger.info("Quirky camera detected: " + cameraQuirks.baseName);
         }
 
-        usbCameraSettables = new USBCameraSettables(config);
-        usbFrameProvider = new USBFrameProvider(cvSink, usbCameraSettables);
+        if (cameraQuirks.hasQuirk(CameraQuirk.CompletelyBroken)) {
+            // set some defaults, as these should never be used.
+            logger.info("Camera " + cameraQuirks.baseName + " is not supported for PhotonVision");
+            usbCameraSettables = null;
+            usbFrameProvider = null;
+        } else {
+            // Normal init
+            // auto exposure/brightness/gain will be set by the visionmodule later
+            disableAutoFocus();
 
-        if (cameraQuirks.hasQuirk(CameraQuirk.PiCam)) {
-            // Pick a bunch of reasonable setting defaults for vision processing.
-            camera.getProperty("exposure_dynamic_framerate").set(0);
-            camera.getProperty("auto_exposure_bias").set(0);
-            camera.getProperty("image_stabilization").set(0);
-            camera.getProperty("iso_sensitivity").set(0);
-            camera.getProperty("iso_sensitivity_auto").set(0);
-            camera.getProperty("exposure_metering_mode").set(0);
-            camera.getProperty("scene_mode").set(0);
-            camera.getProperty("power_line_frequency").set(2);
+            usbCameraSettables = new USBCameraSettables(config);
+            if (usbCameraSettables.getAllVideoModes().isEmpty()) {
+                logger.info("Camera " + camera.getPath() + " has no video modes supported by PhotonVision");
+                usbFrameProvider = null;
+            } else {
+                usbFrameProvider = new USBFrameProvider(cvSink, usbCameraSettables);
+            }
+        }
+    }
+
+    void disableAutoFocus() {
+        if (cameraQuirks.hasQuirk(CameraQuirk.AdjustableFocus)) {
+            try {
+                camera.getProperty("focus_auto").set(0);
+                camera.getProperty("focus_absolute").set(0); // Focus into infinity
+            } catch (VideoException e) {
+                logger.error("Unable to disable autofocus!", e);
+            }
         }
     }
 
@@ -89,43 +102,98 @@ public class USBCameraSource extends VisionSource {
             setVideoMode(videoModes.get(0));
         }
 
-        private int timeToPiCamV2RawExposure(double time_us) {
+        public void setAutoExposure(boolean cameraAutoExposure) {
+            logger.debug("Setting auto exposure to " + cameraAutoExposure);
+
+            if (cameraQuirks.hasQuirk(CameraQuirk.PiCam)) {
+                // Case, we know this is a picam. Go through v4l2-ctl interface directly
+
+                // Common settings
+                camera
+                        .getProperty("image_stabilization")
+                        .set(0); // No image stabilization, as this will throw off odometry
+                camera.getProperty("power_line_frequency").set(2); // Assume 60Hz USA
+                camera.getProperty("scene_mode").set(0); // no presets
+                camera.getProperty("exposure_metering_mode").set(0);
+                camera.getProperty("exposure_dynamic_framerate").set(0);
+
+                if (!cameraAutoExposure) {
+                    // Pick a bunch of reasonable setting defaults for vision processing retroreflective
+                    camera.getProperty("auto_exposure_bias").set(0);
+                    camera.getProperty("iso_sensitivity_auto").set(0); // Disable auto ISO adjustement
+                    camera.getProperty("iso_sensitivity").set(0); // Manual ISO adjustement
+                    camera.getProperty("white_balance_auto_preset").set(2); // Auto white-balance disabled
+                    camera.getProperty("auto_exposure").set(1); // auto exposure disabled
+                } else {
+                    // Pick a bunch of reasonable setting defaults for driver, fiducials, or otherwise
+                    // nice-for-humans
+                    camera.getProperty("auto_exposure_bias").set(12);
+                    camera.getProperty("iso_sensitivity_auto").set(1);
+                    camera.getProperty("iso_sensitivity").set(1); // Manual ISO adjustement by default
+                    camera.getProperty("white_balance_auto_preset").set(1); // Auto white-balance enabled
+                    camera.getProperty("auto_exposure").set(0); // auto exposure enabled
+                }
+
+            } else {
+                // Case - this is some other USB cam. Default to wpilib's implementation
+
+                var canSetWhiteBalance = !cameraQuirks.hasQuirk(CameraQuirk.Gain);
+
+                if (!cameraAutoExposure) {
+                    // Pick a bunch of reasonable setting defaults for vision processing retroreflective
+                    if (canSetWhiteBalance) {
+                        camera.setWhiteBalanceManual(4000); // Auto white-balance disabled, 4000K preset
+                    }
+                } else {
+                    // Pick a bunch of reasonable setting defaults for driver, fiducials, or otherwise
+                    // nice-for-humans
+                    if (canSetWhiteBalance) {
+                        camera.setWhiteBalanceAuto(); // Auto white-balance enabled
+                    }
+                    camera.setExposureAuto(); // auto exposure enabled
+                }
+            }
+        }
+
+        private int timeToPiCamRawExposure(double time_us) {
             int retVal =
-                    (int) Math.round(time_us / 100.0); // PiCamV2 needs exposure time in units of 100us/bit
+                    (int)
+                            Math.round(
+                                    time_us / 100.0); // Pi Cam's (both v1 and v2) need exposure time in units of
+            // 100us/bit
             return Math.min(Math.max(retVal, 1), 10000); // Cap to allowable range for parameter
         }
 
         private double pctToExposureTimeUs(double pct_in) {
             // Mirror the photonvision raspicam driver's algorithm for picking an exposure time
             // from a 0-100% input
-            final double PADDING_LOW_US = 100;
-            final double PADDING_HIGH_US = 200;
+            final double PADDING_LOW_US = 10;
+            final double PADDING_HIGH_US = 10;
             return PADDING_LOW_US
                     + (pct_in / 100.0) * ((1e6 / (double) camera.getVideoMode().fps) - PADDING_HIGH_US);
         }
 
         @Override
         public void setExposure(double exposure) {
-            try {
-                int scaledExposure = 1;
-                if (cameraQuirks.hasQuirk(CameraQuirk.PiCam)) {
-                    camera.getProperty("white_balance_auto_preset").set(2); // Auto white-balance off
-                    camera.getProperty("auto_exposure").set(1); // auto exposure off
+            if (exposure >= 0.0) {
+                try {
+                    int scaledExposure = 1;
+                    if (cameraQuirks.hasQuirk(CameraQuirk.PiCam)) {
+                        scaledExposure =
+                                (int) Math.round(timeToPiCamRawExposure(pctToExposureTimeUs(exposure)));
+                        logger.debug("Setting camera raw exposure to " + Integer.toString(scaledExposure));
+                        camera.getProperty("raw_exposure_time_absolute").set(scaledExposure);
+                        camera.getProperty("raw_exposure_time_absolute").set(scaledExposure);
 
-                    scaledExposure =
-                            (int) Math.round(timeToPiCamV2RawExposure(pctToExposureTimeUs(exposure)));
-                    logger.debug("Setting camera raw exposure to " + Integer.toString(scaledExposure));
-                    camera.getProperty("raw_exposure_time_absolute").set(scaledExposure);
-                    camera.getProperty("raw_exposure_time_absolute").set(scaledExposure);
-
-                } else {
-                    scaledExposure = (int) Math.round(exposure);
-                    logger.debug("Setting camera exposure to " + Integer.toString(scaledExposure));
-                    camera.setExposureManual(scaledExposure);
-                    camera.setExposureManual(scaledExposure);
+                    } else {
+                        scaledExposure = (int) Math.round(exposure);
+                        logger.debug("Setting camera exposure to " + Integer.toString(scaledExposure));
+                        camera.setExposureManual(scaledExposure);
+                        camera.setExposureManual(scaledExposure);
+                    }
+                } catch (VideoException e) {
+                    logger.error("Failed to set camera exposure!", e);
                 }
-            } catch (VideoException e) {
-                logger.error("Failed to set camera exposure!", e);
             }
         }
 
@@ -180,8 +248,16 @@ public class USBCameraSource extends VisionSource {
                         modes =
                                 new VideoMode[] {
                                     new VideoMode(VideoMode.PixelFormat.kBGR, 320, 240, 90),
+                                    new VideoMode(VideoMode.PixelFormat.kBGR, 320, 240, 30),
+                                    new VideoMode(VideoMode.PixelFormat.kBGR, 320, 240, 15),
+                                    new VideoMode(VideoMode.PixelFormat.kBGR, 320, 240, 10),
                                     new VideoMode(VideoMode.PixelFormat.kBGR, 640, 480, 90),
+                                    new VideoMode(VideoMode.PixelFormat.kBGR, 640, 480, 45),
+                                    new VideoMode(VideoMode.PixelFormat.kBGR, 640, 480, 30),
+                                    new VideoMode(VideoMode.PixelFormat.kBGR, 640, 480, 15),
+                                    new VideoMode(VideoMode.PixelFormat.kBGR, 640, 480, 10),
                                     new VideoMode(VideoMode.PixelFormat.kBGR, 960, 720, 60),
+                                    new VideoMode(VideoMode.PixelFormat.kBGR, 960, 720, 10),
                                     new VideoMode(VideoMode.PixelFormat.kBGR, 1280, 720, 45),
                                     new VideoMode(VideoMode.PixelFormat.kBGR, 1920, 1080, 20),
                                 };
@@ -212,21 +288,24 @@ public class USBCameraSource extends VisionSource {
 
                         videoModesList.add(videoMode);
 
+                        // TODO - do we want to trim down FPS modes? in cases where the camera has no gain
+                        // control,
+                        // lower FPS might be needed to ensure total exposure is acceptable.
                         // We look for modes with the same height/width/pixelformat as this mode
                         // and remove all the ones that are slower. This is sorted low to high.
                         // So we remove the last element (the fastest FPS) from the duplicate list,
                         // and remove all remaining elements from the final list
-                        var duplicateModes =
-                                videoModesList.stream()
-                                        .filter(
-                                                it ->
-                                                        it.height == videoMode.height
-                                                                && it.width == videoMode.width
-                                                                && it.pixelFormat == videoMode.pixelFormat)
-                                        .sorted(Comparator.comparingDouble(it -> it.fps))
-                                        .collect(Collectors.toList());
-                        duplicateModes.remove(duplicateModes.size() - 1);
-                        videoModesList.removeAll(duplicateModes);
+                        // var duplicateModes =
+                        //         videoModesList.stream()
+                        //                 .filter(
+                        //                         it ->
+                        //                                 it.height == videoMode.height
+                        //                                         && it.width == videoMode.width
+                        //                                         && it.pixelFormat == videoMode.pixelFormat)
+                        //                 .sorted(Comparator.comparingDouble(it -> it.fps))
+                        //                 .collect(Collectors.toList());
+                        // duplicateModes.remove(duplicateModes.size() - 1);
+                        // videoModesList.removeAll(duplicateModes);
                     }
                 } catch (Exception e) {
                     logger.error("Exception while enumerating video modes!", e);

photon-core/src/main/java/org/photonvision/vision/camera/ZeroCopyPicamSource.java

@@ -1,213 +0,0 @@
-/*
- * Copyright (C) Photon Vision.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package org.photonvision.vision.camera;
-
-import edu.wpi.first.cscore.VideoMode;
-import edu.wpi.first.math.Pair;
-import java.util.HashMap;
-import org.photonvision.common.configuration.CameraConfiguration;
-import org.photonvision.common.configuration.ConfigManager;
-import org.photonvision.common.logging.LogGroup;
-import org.photonvision.common.logging.Logger;
-import org.photonvision.raspi.PicamJNI;
-import org.photonvision.vision.frame.FrameProvider;
-import org.photonvision.vision.frame.provider.AcceleratedPicamFrameProvider;
-import org.photonvision.vision.processes.VisionSource;
-import org.photonvision.vision.processes.VisionSourceSettables;
-
-public class ZeroCopyPicamSource extends VisionSource {
-    private static final Logger logger = new Logger(ZeroCopyPicamSource.class, LogGroup.Camera);
-
-    private final VisionSourceSettables settables;
-    private final AcceleratedPicamFrameProvider frameProvider;
-
-    public ZeroCopyPicamSource(CameraConfiguration configuration) {
-        super(configuration);
-        if (configuration.cameraType != CameraType.ZeroCopyPicam) {
-            throw new IllegalArgumentException(
-                    "GPUAcceleratedPicamSource only accepts CameraConfigurations with type Picam");
-        }
-
-        settables = new PicamSettables(configuration);
-        frameProvider = new AcceleratedPicamFrameProvider(settables);
-    }
-
-    @Override
-    public FrameProvider getFrameProvider() {
-        return frameProvider;
-    }
-
-    @Override
-    public VisionSourceSettables getSettables() {
-        return settables;
-    }
-
-    /**
-     * On the OV5649 the actual FPS we want to request from the GPU can be higher than the FPS that we
-     * can do after processing. On the IMX219 these FPSes match pretty closely, except for the
-     * 1280x720 mode. We use this to present a rated FPS to the user that's lower than the actual FPS
-     * we request from the GPU. This is important for setting user expectations, and is also used by
-     * the frontend to detect and explain FPS drops. This class should ONLY be used by Picam video
-     * modes! This is to make sure it shows up nice in the frontend
-     */
-    public static class FPSRatedVideoMode extends VideoMode {
-        public final int fpsActual;
-        public final double fovMultiplier;
-
-        public FPSRatedVideoMode(
-                PixelFormat pixelFormat,
-                int width,
-                int height,
-                int ratedFPS,
-                int actualFPS,
-                double fovMultiplier) {
-            super(pixelFormat, width, height, ratedFPS);
-
-            this.fpsActual = actualFPS;
-            this.fovMultiplier = fovMultiplier;
-        }
-    }
-
-    public static class PicamSettables extends VisionSourceSettables {
-        private FPSRatedVideoMode currentVideoMode;
-        private double lastExposure = 50;
-        private int lastBrightness = 50;
-        private int lastGain = 50;
-        private Pair<Integer, Integer> lastAwbGains = new Pair(18, 18);
-
-        public PicamSettables(CameraConfiguration configuration) {
-            super(configuration);
-
-            videoModes = new HashMap<>();
-            PicamJNI.SensorModel sensorModel = PicamJNI.getSensorModel();
-
-            if (sensorModel == PicamJNI.SensorModel.IMX219) {
-                // Settings for the IMX219 sensor, which is used on the Pi Camera Module v2
-                videoModes.put(
-                        0, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 320, 240, 120, 120, .39));
-                videoModes.put(
-                        1, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 640, 480, 65, 90, .39));
-                // TODO: fix 1280x720 in the native code and re-add it
-                videoModes.put(
-                        3, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1920, 1080, 15, 20, .53));
-            } else {
-                if (sensorModel == PicamJNI.SensorModel.IMX477) {
-                    logger.warn(
-                            "It appears you are using a Pi HQ Camera. This camera is not officially supported. You will have to set your camera FOV differently based on resolution.");
-                } else if (sensorModel == PicamJNI.SensorModel.Unknown) {
-                    logger.warn(
-                            "You have an unknown sensor connected to your Pi over CSI! This is likely a bug. If it is not, then you will have to set your camera FOV differently based on resolution.");
-                }
-
-                // Settings for the OV5647 sensor, which is used by the Pi Camera Module v1
-                videoModes.put(
-                        0, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 320, 240, 90, 90, 1));
-                videoModes.put(
-                        1, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 640, 480, 85, 90, 1));
-                videoModes.put(
-                        2, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 960, 720, 45, 49, 0.74));
-                videoModes.put(
-                        3, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1280, 720, 30, 45, 0.91));
-                videoModes.put(
-                        4, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1920, 1080, 15, 20, 0.72));
-            }
-
-            currentVideoMode = (FPSRatedVideoMode) videoModes.get(0);
-        }
-
-        @Override
-        public double getFOV() {
-            return getCurrentVideoMode().fovMultiplier * getConfiguration().FOV;
-        }
-
-        @Override
-        public void setExposure(double exposure) {
-            lastExposure = exposure;
-            var failure = PicamJNI.setExposure((int) Math.round(exposure));
-            if (failure) logger.warn("Couldn't set Pi Camera exposure");
-        }
-
-        @Override
-        public void setBrightness(int brightness) {
-            lastBrightness = brightness;
-            var failure = PicamJNI.setBrightness(brightness);
-            if (failure) logger.warn("Couldn't set Pi Camera brightness");
-        }
-
-        @Override
-        public void setGain(int gain) {
-            lastGain = gain;
-            var failure = PicamJNI.setGain(gain);
-            if (failure) logger.warn("Couldn't set Pi Camera gain");
-        }
-
-        @Override
-        public void setRedGain(int red) {
-            lastAwbGains = Pair.of(red, lastAwbGains.getSecond());
-            setAwbGain(lastAwbGains.getFirst(), lastAwbGains.getSecond());
-        }
-
-        @Override
-        public void setBlueGain(int blue) {
-            lastAwbGains = Pair.of(lastAwbGains.getFirst(), blue);
-            setAwbGain(lastAwbGains.getFirst(), lastAwbGains.getSecond());
-        }
-
-        public void setAwbGain(int red, int blue) {
-            var failure = PicamJNI.setAwbGain(red, blue);
-            if (failure) logger.warn("Couldn't set Pi Camera AWB gains");
-        }
-
-        @Override
-        public FPSRatedVideoMode getCurrentVideoMode() {
-            return currentVideoMode;
-        }
-
-        @Override
-        protected void setVideoModeInternal(VideoMode videoMode) {
-            var mode = (FPSRatedVideoMode) videoMode;
-            var failure = PicamJNI.destroyCamera();
-            if (failure)
-                throw new RuntimeException(
-                        "Couldn't destroy a zero copy Pi Camera while switching video modes");
-            failure = PicamJNI.createCamera(mode.width, mode.height, mode.fpsActual);
-            if (failure)
-                throw new RuntimeException(
-                        "Couldn't create a zero copy Pi Camera while switching video modes");
-
-            // We don't store last settings on the native side, and when you change video mode these get
-            // reset on MMAL's end
-            setExposure(lastExposure);
-            setBrightness(lastBrightness);
-            setGain(lastGain);
-            setAwbGain(lastAwbGains.getFirst(), lastAwbGains.getSecond());
-
-            currentVideoMode = mode;
-        }
-
-        @Override
-        public HashMap<Integer, VideoMode> getAllVideoModes() {
-            return videoModes;
-        }
-    }
-
-    @Override
-    public boolean isVendorCamera() {
-        return ConfigManager.getInstance().getConfig().getHardwareConfig().hasPresetFOV();
-    }
-}

photon-core/src/main/java/org/photonvision/vision/frame/Frame.java

@@ -17,46 +17,58 @@
 
 package org.photonvision.vision.frame;
 
-import edu.wpi.first.math.geometry.Rotation2d;
 import org.photonvision.common.util.math.MathUtils;
 import org.photonvision.vision.opencv.CVMat;
 import org.photonvision.vision.opencv.Releasable;
 
 public class Frame implements Releasable {
     public final long timestampNanos;
-    public final CVMat image;
+
+    // Frame should at _least_ contain the thresholded frame, and sometimes the color image
+    public final CVMat colorImage;
+    public final CVMat processedImage;
+    public final FrameThresholdType type;
+
     public final FrameStaticProperties frameStaticProperties;
 
-    public Frame(CVMat image, long timestampNanos, FrameStaticProperties frameStaticProperties) {
-        this.image = image;
+    public Frame(
+            CVMat color,
+            CVMat processed,
+            FrameThresholdType type,
+            long timestampNanos,
+            FrameStaticProperties frameStaticProperties) {
+        this.colorImage = color;
+        this.processedImage = processed;
+        this.type = type;
         this.timestampNanos = timestampNanos;
         this.frameStaticProperties = frameStaticProperties;
     }
 
-    public Frame(CVMat image, FrameStaticProperties frameStaticProperties) {
-        this(image, MathUtils.wpiNanoTime(), frameStaticProperties);
+    public Frame(
+            CVMat color,
+            CVMat processed,
+            FrameThresholdType processType,
+            FrameStaticProperties frameStaticProperties) {
+        this(color, processed, processType, MathUtils.wpiNanoTime(), frameStaticProperties);
     }
 
     public Frame() {
         this(
                 new CVMat(),
+                new CVMat(),
+                FrameThresholdType.NONE,
                 MathUtils.wpiNanoTime(),
-                new FrameStaticProperties(0, 0, 0, new Rotation2d(), null));
+                new FrameStaticProperties(0, 0, 0, null));
     }
 
     public void copyTo(Frame destFrame) {
-        image.getMat().copyTo(destFrame.image.getMat());
-    }
-
-    public static Frame copyFromAndRelease(Frame frame) {
-        var mat = new CVMat();
-        frame.image.copyTo(mat);
-        frame.release();
-        return new Frame(mat, frame.timestampNanos, frame.frameStaticProperties);
+        colorImage.getMat().copyTo(destFrame.colorImage.getMat());
+        processedImage.getMat().copyTo(destFrame.processedImage.getMat());
     }
 
     @Override
     public void release() {
-        image.release();
+        colorImage.release();
+        processedImage.release();
     }
 }

photon-core/src/main/java/org/photonvision/vision/frame/FrameProvider.java

@@ -18,7 +18,21 @@
 package org.photonvision.vision.frame;
 
 import java.util.function.Supplier;
+import org.photonvision.vision.opencv.ImageRotationMode;
+import org.photonvision.vision.pipe.impl.HSVPipe;
 
 public interface FrameProvider extends Supplier<Frame> {
     String getName();
+
+    /** Ask the camera to produce a certain kind of processed image (eg HSV or greyscale) */
+    public void requestFrameThresholdType(FrameThresholdType type);
+
+    /** Ask the camera to rotate frames it outputs */
+    public void requestFrameRotation(ImageRotationMode rotationMode);
+
+    /** Ask the camera to provide either the input, output, or both frames. */
+    public void requestFrameCopies(boolean copyInput, boolean copyOutput);
+
+    /** Ask the camera to rotate frames it outputs */
+    public void requestHsvSettings(HSVPipe.HSVParams params);
 }

photon-core/src/main/java/org/photonvision/vision/frame/FrameStaticProperties.java

@@ -18,7 +18,6 @@
 package org.photonvision.vision.frame;
 
 import edu.wpi.first.cscore.VideoMode;
-import edu.wpi.first.math.geometry.Rotation2d;
 import org.opencv.core.Point;
 import org.photonvision.common.util.numbers.DoubleCouple;
 import org.photonvision.vision.calibration.CameraCalibrationCoefficients;
@@ -34,7 +33,6 @@ public class FrameStaticProperties {
     public final Point centerPoint;
     public final double horizontalFocalLength;
     public final double verticalFocalLength;
-    public final Rotation2d cameraPitch;
     public CameraCalibrationCoefficients cameraCalibration;
 
     /**
@@ -43,9 +41,8 @@ public class FrameStaticProperties {
      * @param mode The Video Mode of the camera.
      * @param fov The fov of the image.
      */
-    public FrameStaticProperties(
-            VideoMode mode, double fov, Rotation2d cameraPitch, CameraCalibrationCoefficients cal) {
-        this(mode != null ? mode.width : 1, mode != null ? mode.height : 1, fov, cameraPitch, cal);
+    public FrameStaticProperties(VideoMode mode, double fov, CameraCalibrationCoefficients cal) {
+        this(mode != null ? mode.width : 1, mode != null ? mode.height : 1, fov, cal);
     }
 
     /**
@@ -56,15 +53,10 @@ public class FrameStaticProperties {
      * @param fov The fov of the image.
      */
     public FrameStaticProperties(
-            int imageWidth,
-            int imageHeight,
-            double fov,
-            Rotation2d cameraPitch,
-            CameraCalibrationCoefficients cal) {
+            int imageWidth, int imageHeight, double fov, CameraCalibrationCoefficients cal) {
         this.imageWidth = imageWidth;
         this.imageHeight = imageHeight;
         this.fov = fov;
-        this.cameraPitch = cameraPitch;
         this.cameraCalibration = cal;
 
         imageArea = this.imageWidth * this.imageHeight;

photon-core/src/main/java/org/photonvision/vision/frame/FrameThresholdType.java

@@ -15,11 +15,10 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package org.photonvision.common.hardware.metrics;
+package org.photonvision.vision.frame;
 
-public class DiskMetrics extends MetricsBase {
-    public String getUsedDiskPct() {
-        if (diskUsageCommand.isEmpty()) return "";
-        return execute(diskUsageCommand);
-    }
+public enum FrameThresholdType {
+    NONE,
+    HSV,
+    GREYSCALE,
 }

photon-core/src/main/java/org/photonvision/vision/frame/consumer/FileSaveFrameConsumer.java

@@ -17,23 +17,21 @@
 
 package org.photonvision.vision.frame.consumer;
 
+import edu.wpi.first.networktables.IntegerEntry;
 import edu.wpi.first.networktables.NetworkTable;
-import edu.wpi.first.networktables.NetworkTableEntry;
 import java.io.File;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.Date;
-import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.Consumer;
 import org.opencv.imgcodecs.Imgcodecs;
 import org.photonvision.common.configuration.ConfigManager;
 import org.photonvision.common.dataflow.networktables.NetworkTablesManager;
 import org.photonvision.common.logging.LogGroup;
 import org.photonvision.common.logging.Logger;
-import org.photonvision.common.util.TimedTaskManager;
-import org.photonvision.vision.frame.Frame;
+import org.photonvision.vision.opencv.CVMat;
 
-public class FileSaveFrameConsumer implements Consumer<Frame> {
+public class FileSaveFrameConsumer implements Consumer<CVMat> {
     // Formatters to generate unique, timestamped file names
     private static String FILE_PATH = ConfigManager.getInstance().getImageSavePath().toString();
     private static String FILE_EXTENSION = ".jpg";
@@ -44,30 +42,27 @@ public class FileSaveFrameConsumer implements Consumer<Frame> {
     private NetworkTable subTable;
     private final NetworkTable rootTable;
     private final Logger logger;
-    private boolean prevCommand = false;
+    private long imgSaveCountInternal = 0;
     private String camNickname;
     private String fnamePrefix;
-    private final long CMD_RESET_TIME_MS = 500;
-    private final NetworkTableEntry entry;
-    // Helps prevent race conditions between user set & auto-reset logic
-    private ReentrantLock lock;
+    private IntegerEntry entry;
 
     public FileSaveFrameConsumer(String camNickname, String streamPrefix) {
-        this.lock = new ReentrantLock();
         this.fnamePrefix = camNickname + "_" + streamPrefix;
         this.ntEntryName = streamPrefix + NT_SUFFIX;
         this.rootTable = NetworkTablesManager.getInstance().kRootTable;
         updateCameraNickname(camNickname);
-        entry = subTable.getEntry(ntEntryName);
-        entry.forceSetBoolean(false);
         this.logger = new Logger(FileSaveFrameConsumer.class, this.camNickname, LogGroup.General);
     }
 
-    public void accept(Frame frame) {
-        if (frame != null && !frame.image.getMat().empty()) {
-            if (lock.tryLock()) {
-                boolean curCommand = entry.getBoolean(false);
-                if (curCommand && !prevCommand) {
+    public void accept(CVMat image) {
+        if (image != null && image.getMat() != null && !image.getMat().empty()) {
+            var curCommand = entry.get(); // default to just our current count
+            if (curCommand >= 0) {
+                // Only do something if we got a valid current command
+                if (imgSaveCountInternal < curCommand) {
+                    // Save one frame.
+                    // Create the filename
                     Date now = new Date();
                     String savefile =
                             FILE_PATH
@@ -79,42 +74,32 @@ public class FileSaveFrameConsumer implements Consumer<Frame> {
                                     + tf.format(now)
                                     + FILE_EXTENSION;
 
-                    Imgcodecs.imwrite(savefile, frame.image.getMat());
-
-                    // Help the user a bit - set the NT entry back to false after 500ms
-                    TimedTaskManager.getInstance().addOneShotTask(this::resetCommand, CMD_RESET_TIME_MS);
+                    // write to file
+                    Imgcodecs.imwrite(savefile, image.getMat());
 
+                    // Count one more image saved
+                    imgSaveCountInternal++;
                     logger.info("Saved new image at " + savefile);
-                } else if (!curCommand) {
-                    // If the entry is currently false, set it again. This will make sure it shows up on the
-                    // dashboard.
-                    entry.forceSetBoolean(false);
+
+                } else if (imgSaveCountInternal > curCommand) {
+                    imgSaveCountInternal = curCommand;
                 }
-
-                prevCommand = curCommand;
-                lock.unlock();
-            }
-        }
-    }
-
-    private void resetCommand() {
-        lock.lock();
-        this.subTable.getEntry(ntEntryName).setBoolean(false);
-        lock.unlock();
-    }
-
-    private void removeEntries() {
-        if (this.subTable != null) {
-            if (this.subTable.containsKey(ntEntryName)) {
-                this.subTable.delete(ntEntryName);
             }
         }
     }
 
     public void updateCameraNickname(String newCameraNickname) {
-        removeEntries();
+        // Remove existing entries
+        if (this.subTable != null) {
+            if (this.subTable.containsKey(ntEntryName)) {
+                this.subTable.getEntry(ntEntryName).close();
+            }
+        }
+
+        // Recreate and re-init network tables structure
         this.camNickname = newCameraNickname;
         this.subTable = rootTable.getSubTable(this.camNickname);
-        resetCommand();
+        this.subTable.getEntry(ntEntryName).setInteger(imgSaveCountInternal);
+        this.entry = subTable.getIntegerTopic(ntEntryName).getEntry(-1); // Default negative
     }
 }

photon-core/src/main/java/org/photonvision/vision/frame/consumer/MJPGFrameConsumer.java

@@ -28,7 +28,7 @@ import org.opencv.core.Point;
 import org.opencv.core.Rect;
 import org.opencv.imgproc.Imgproc;
 import org.photonvision.common.util.ColorHelper;
-import org.photonvision.vision.frame.Frame;
+import org.photonvision.vision.opencv.CVMat;
 
 public class MJPGFrameConsumer {
     public static final Mat EMPTY_MAT = new Mat(60, 15 * 7, CvType.CV_8UC3);
@@ -119,6 +119,7 @@ public class MJPGFrameConsumer {
 
         this.mjpegServer = new MjpegServer("serve_" + cvSource.getName(), port);
         mjpegServer.setSource(cvSource);
+        mjpegServer.setCompression(75);
 
         listener =
                 new VideoListener(
@@ -166,9 +167,9 @@ public class MJPGFrameConsumer {
         this(name, 320, 240, port);
     }
 
-    public void accept(Frame frame) {
-        if (frame != null && !frame.image.getMat().empty()) {
-            cvSource.putFrame(frame.image.getMat());
+    public void accept(CVMat image) {
+        if (image != null && !image.getMat().empty()) {
+            cvSource.putFrame(image.getMat());
 
             // Make sure our disabled framerate limiting doesn't get confused
             isDisabled = false;

photon-core/src/main/java/org/photonvision/vision/frame/provider/AcceleratedPicamFrameProvider.java

@@ -1,60 +0,0 @@
-/*
- * Copyright (C) Photon Vision.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package org.photonvision.vision.frame.provider;
-
-import org.opencv.core.Mat;
-import org.photonvision.common.util.math.MathUtils;
-import org.photonvision.raspi.PicamJNI;
-import org.photonvision.vision.frame.Frame;
-import org.photonvision.vision.frame.FrameProvider;
-import org.photonvision.vision.opencv.CVMat;
-import org.photonvision.vision.processes.VisionSourceSettables;
-
-public class AcceleratedPicamFrameProvider implements FrameProvider {
-    private final VisionSourceSettables settables;
-
-    private CVMat mat;
-
-    public AcceleratedPicamFrameProvider(VisionSourceSettables visionSettables) {
-        this.settables = visionSettables;
-
-        var vidMode = settables.getCurrentVideoMode();
-        var failure = PicamJNI.createCamera(vidMode.width, vidMode.height, vidMode.fps);
-        if (failure) {
-            failure = PicamJNI.destroyCamera();
-            if (failure) throw new RuntimeException("Couldn't destroy Pi camera after init failure!");
-            throw new RuntimeException(
-                    "Couldn't initialize zero copy Pi camera; check stdout for native code logs");
-        }
-    }
-
-    @Override
-    public String getName() {
-        return "AcceleratedPicamFrameProvider";
-    }
-
-    @Override
-    public Frame get() {
-        long matHandle = PicamJNI.grabFrame(false);
-        mat = new CVMat(new Mat(matHandle));
-        return new Frame(
-                mat,
-                MathUtils.wpiNanoTime() - PicamJNI.getFrameLatency(),
-                settables.getFrameStaticProperties());
-    }
-}

photon-core/src/main/java/org/photonvision/vision/frame/provider/CpuImageProcessor.java

@@ -0,0 +1,124 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.vision.frame.provider;
+
+import org.photonvision.common.util.numbers.IntegerCouple;
+import org.photonvision.vision.frame.Frame;
+import org.photonvision.vision.frame.FrameProvider;
+import org.photonvision.vision.frame.FrameStaticProperties;
+import org.photonvision.vision.frame.FrameThresholdType;
+import org.photonvision.vision.opencv.CVMat;
+import org.photonvision.vision.opencv.ImageRotationMode;
+import org.photonvision.vision.pipe.CVPipe.CVPipeResult;
+import org.photonvision.vision.pipe.impl.GrayscalePipe;
+import org.photonvision.vision.pipe.impl.HSVPipe;
+import org.photonvision.vision.pipe.impl.RotateImagePipe;
+
+public abstract class CpuImageProcessor implements FrameProvider {
+    protected class CapturedFrame {
+        CVMat colorImage;
+        FrameStaticProperties staticProps;
+        long captureTimestamp;
+
+        public CapturedFrame(
+                CVMat colorImage, FrameStaticProperties staticProps, long captureTimestampNanos) {
+            this.colorImage = colorImage;
+            this.staticProps = staticProps;
+            this.captureTimestamp = captureTimestampNanos;
+        }
+    }
+
+    private final HSVPipe m_hsvPipe = new HSVPipe();
+    private final RotateImagePipe m_rImagePipe = new RotateImagePipe();
+    private final GrayscalePipe m_grayPipe = new GrayscalePipe();
+    FrameThresholdType m_processType;
+
+    private final Object m_mutex = new Object();
+
+    abstract CapturedFrame getInputMat();
+
+    public CpuImageProcessor() {
+        m_hsvPipe.setParams(
+                new HSVPipe.HSVParams(
+                        new IntegerCouple(0, 180),
+                        new IntegerCouple(0, 255),
+                        new IntegerCouple(0, 255),
+                        false));
+    }
+
+    @Override
+    public final Frame get() {
+        // TODO Auto-generated method stub
+        var input = getInputMat();
+
+        CVMat outputMat = null;
+        long sumNanos = 0;
+
+        {
+            CVPipeResult<Void> out = m_rImagePipe.run(input.colorImage.getMat());
+            sumNanos += out.nanosElapsed;
+        }
+
+        if (!input.colorImage.getMat().empty()) {
+            if (m_processType == FrameThresholdType.HSV) {
+                var hsvResult = m_hsvPipe.run(input.colorImage.getMat());
+                outputMat = new CVMat(hsvResult.output);
+                sumNanos += hsvResult.nanosElapsed;
+            } else if (m_processType == FrameThresholdType.GREYSCALE) {
+                var result = m_grayPipe.run(input.colorImage.getMat());
+                outputMat = new CVMat(result.output);
+                sumNanos += result.nanosElapsed;
+            } else {
+                outputMat = new CVMat();
+            }
+        } else {
+            System.out.println("Input was empty!");
+            outputMat = new CVMat();
+        }
+
+        return new Frame(
+                input.colorImage, outputMat, m_processType, input.captureTimestamp, input.staticProps);
+    }
+
+    @Override
+    public void requestFrameThresholdType(FrameThresholdType type) {
+        synchronized (m_mutex) {
+            this.m_processType = type;
+        }
+    }
+
+    @Override
+    public void requestFrameRotation(ImageRotationMode rotationMode) {
+        synchronized (m_mutex) {
+            m_rImagePipe.setParams(new RotateImagePipe.RotateImageParams(rotationMode));
+        }
+    }
+
+    /** Ask the camera to rotate frames it outputs */
+    public void requestHsvSettings(HSVPipe.HSVParams params) {
+        synchronized (m_mutex) {
+            m_hsvPipe.setParams(params);
+        }
+    }
+
+    @Override
+    public void requestFrameCopies(boolean copyInput, boolean copyOutput) {
+        // We don't actually do zero-copy, so this method is a no-op
+        return;
+    }
+}

photon-core/src/main/java/org/photonvision/vision/frame/provider/FileFrameProvider.java

@@ -17,14 +17,13 @@
 
 package org.photonvision.vision.frame.provider;
 
-import edu.wpi.first.math.geometry.Rotation2d;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import org.opencv.core.Mat;
 import org.opencv.imgcodecs.Imgcodecs;
+import org.photonvision.common.util.math.MathUtils;
 import org.photonvision.vision.calibration.CameraCalibrationCoefficients;
-import org.photonvision.vision.frame.Frame;
 import org.photonvision.vision.frame.FrameProvider;
 import org.photonvision.vision.frame.FrameStaticProperties;
 import org.photonvision.vision.opencv.CVMat;
@@ -33,14 +32,14 @@ import org.photonvision.vision.opencv.CVMat;
  * A {@link FrameProvider} that will read and provide an image from a {@link java.nio.file.Path
  * path}.
  */
-public class FileFrameProvider implements FrameProvider {
-    public static final int MAX_FPS = 10;
+public class FileFrameProvider extends CpuImageProcessor {
+    public static final int MAX_FPS = 5;
     private static int count = 0;
 
     private final int thisIndex = count++;
     private final Path path;
     private final int millisDelay;
-    private final Frame originalFrame;
+    private final CVMat originalFrame;
 
     private final FrameStaticProperties properties;
 
@@ -54,20 +53,15 @@ public class FileFrameProvider implements FrameProvider {
      * @param maxFPS The max framerate to provide the image at.
      */
     public FileFrameProvider(Path path, double fov, int maxFPS) {
-        this(path, fov, maxFPS, null, null);
+        this(path, fov, maxFPS, null);
+    }
+
+    public FileFrameProvider(Path path, double fov, CameraCalibrationCoefficients calibration) {
+        this(path, fov, MAX_FPS, calibration);
     }
 
     public FileFrameProvider(
-            Path path, double fov, Rotation2d pitch, CameraCalibrationCoefficients calibration) {
-        this(path, fov, MAX_FPS, pitch, calibration);
-    }
-
-    public FileFrameProvider(
-            Path path,
-            double fov,
-            int maxFPS,
-            Rotation2d pitch,
-            CameraCalibrationCoefficients calibration) {
+            Path path, double fov, int maxFPS, CameraCalibrationCoefficients calibration) {
         if (!Files.exists(path))
             throw new RuntimeException("Invalid path for image: " + path.toAbsolutePath().toString());
         this.path = path;
@@ -75,9 +69,8 @@ public class FileFrameProvider implements FrameProvider {
 
         Mat rawImage = Imgcodecs.imread(path.toString());
         if (rawImage.cols() > 0 && rawImage.rows() > 0) {
-            properties =
-                    new FrameStaticProperties(rawImage.width(), rawImage.height(), fov, pitch, calibration);
-            originalFrame = new Frame(new CVMat(rawImage), properties);
+            properties = new FrameStaticProperties(rawImage.width(), rawImage.height(), fov, calibration);
+            originalFrame = new CVMat(rawImage);
         } else {
             throw new RuntimeException("Image loading failed!");
         }
@@ -104,9 +97,9 @@ public class FileFrameProvider implements FrameProvider {
     }
 
     @Override
-    public Frame get() {
-        Frame outputFrame = new Frame(new CVMat(), properties);
-        originalFrame.copyTo(outputFrame);
+    public CapturedFrame getInputMat() {
+        var out = new CVMat();
+        out.copyTo(originalFrame);
 
         // block to keep FPS at a defined rate
         if (System.currentTimeMillis() - lastGetMillis < millisDelay) {
@@ -118,7 +111,7 @@ public class FileFrameProvider implements FrameProvider {
         }
 
         lastGetMillis = System.currentTimeMillis();
-        return outputFrame;
+        return new CapturedFrame(out, properties, MathUtils.wpiNanoTime());
     }
 
     @Override

photon-core/src/main/java/org/photonvision/vision/frame/provider/LibcameraGpuFrameProvider.java

@@ -0,0 +1,114 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.vision.frame.provider;
+
+import org.opencv.core.Mat;
+import org.photonvision.common.util.math.MathUtils;
+import org.photonvision.raspi.LibCameraJNI;
+import org.photonvision.vision.camera.LibcameraGpuSettables;
+import org.photonvision.vision.frame.Frame;
+import org.photonvision.vision.frame.FrameProvider;
+import org.photonvision.vision.frame.FrameThresholdType;
+import org.photonvision.vision.opencv.CVMat;
+import org.photonvision.vision.opencv.ImageRotationMode;
+import org.photonvision.vision.pipe.impl.HSVPipe.HSVParams;
+
+public class LibcameraGpuFrameProvider implements FrameProvider {
+    private final LibcameraGpuSettables settables;
+
+    public LibcameraGpuFrameProvider(LibcameraGpuSettables visionSettables) {
+        this.settables = visionSettables;
+
+        var vidMode = settables.getCurrentVideoMode();
+        settables.setVideoMode(vidMode);
+    }
+
+    @Override
+    public String getName() {
+        return "AcceleratedPicamFrameProvider";
+    }
+
+    int i = 0;
+
+    @Override
+    public Frame get() {
+        // We need to make sure that other threads don't try to change video modes while we're waiting
+        // for a frame
+        // System.out.println("GET!");
+        synchronized (LibCameraJNI.CAMERA_LOCK) {
+            var success = LibCameraJNI.awaitNewFrame();
+
+            if (!success) {
+                System.out.println("No new frame");
+                return new Frame();
+            }
+
+            var colorMat = new CVMat(new Mat(LibCameraJNI.takeColorFrame()));
+            var processedMat = new CVMat(new Mat(LibCameraJNI.takeProcessedFrame()));
+
+            // System.out.println("Color mat: " + colorMat.getMat().size());
+
+            // Imgcodecs.imwrite("color" + i + ".jpg", colorMat.getMat());
+            // Imgcodecs.imwrite("processed" + (i) + ".jpg", processedMat.getMat());
+
+            int itype = LibCameraJNI.getGpuProcessType();
+            FrameThresholdType type = FrameThresholdType.NONE;
+            if (itype < FrameThresholdType.values().length && itype >= 0) {
+                type = FrameThresholdType.values()[itype];
+            }
+
+            var now = LibCameraJNI.getLibcameraTimestamp();
+            var capture = LibCameraJNI.getFrameCaptureTime();
+            var latency = (now - capture);
+
+            return new Frame(
+                    colorMat,
+                    processedMat,
+                    type,
+                    MathUtils.wpiNanoTime() - latency,
+                    settables.getFrameStaticProperties());
+        }
+    }
+
+    @Override
+    public void requestFrameThresholdType(FrameThresholdType type) {
+        LibCameraJNI.setGpuProcessType(type.ordinal());
+    }
+
+    @Override
+    public void requestFrameRotation(ImageRotationMode rotationMode) {
+        this.settables.setRotation(rotationMode);
+    }
+
+    @Override
+    public void requestHsvSettings(HSVParams params) {
+        LibCameraJNI.setThresholds(
+                params.getHsvLower().val[0] / 180.0,
+                params.getHsvLower().val[1] / 255.0,
+                params.getHsvLower().val[2] / 255.0,
+                params.getHsvUpper().val[0] / 180.0,
+                params.getHsvUpper().val[1] / 255.0,
+                params.getHsvUpper().val[2] / 255.0,
+                params.getHueInverted());
+    }
+
+    @Override
+    public void requestFrameCopies(boolean copyInput, boolean copyOutput) {
+        LibCameraJNI.setFramesToCopy(copyInput, copyOutput);
+    }
+}

photon-core/src/main/java/org/photonvision/vision/frame/provider/USBFrameProvider.java

@@ -19,12 +19,10 @@ package org.photonvision.vision.frame.provider;
 
 import edu.wpi.first.cscore.CvSink;
 import org.photonvision.common.util.math.MathUtils;
-import org.photonvision.vision.frame.Frame;
-import org.photonvision.vision.frame.FrameProvider;
 import org.photonvision.vision.opencv.CVMat;
 import org.photonvision.vision.processes.VisionSourceSettables;
 
-public class USBFrameProvider implements FrameProvider {
+public class USBFrameProvider extends CpuImageProcessor {
     private final CvSink cvSink;
 
     @SuppressWarnings("SpellCheckingInspection")
@@ -38,18 +36,19 @@ public class USBFrameProvider implements FrameProvider {
     }
 
     @Override
-    public Frame get() {
+    public CapturedFrame getInputMat() {
         var mat = new CVMat(); // We do this so that we don't fill a Mat in use by another thread
         // This is from wpi::Now, or WPIUtilJNI.now()
         long time =
-                cvSink.grabFrame(
-                        mat.getMat()); // Units are microseconds, epoch is the same as the Unix epoch
+                cvSink.grabFrame(mat.getMat())
+                        * 1000; // Units are microseconds, epoch is the same as the Unix epoch
 
         // Sometimes CSCore gives us a zero frametime.
         if (time <= 1e-6) {
             time = MathUtils.wpiNanoTime();
         }
-        return new Frame(mat, MathUtils.microsToNanos(time), settables.getFrameStaticProperties());
+
+        return new CapturedFrame(mat, settables.getFrameStaticProperties(), time);
     }
 
     @Override

photon-core/src/main/java/org/photonvision/vision/pipe/impl/AprilTagDetectionPipe.java

@@ -0,0 +1,69 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.vision.pipe.impl;
+
+import edu.wpi.first.apriltag.AprilTagDetection;
+import edu.wpi.first.apriltag.AprilTagDetector;
+import java.util.List;
+import org.photonvision.vision.opencv.CVMat;
+import org.photonvision.vision.pipe.CVPipe;
+
+public class AprilTagDetectionPipe
+        extends CVPipe<CVMat, List<AprilTagDetection>, AprilTagDetectionPipeParams> {
+    private final AprilTagDetector m_detector = new AprilTagDetector();
+
+    boolean useNativePoseEst;
+
+    public AprilTagDetectionPipe() {
+        super();
+
+        m_detector.addFamily("tag16h5");
+        m_detector.addFamily("tag36h11");
+    }
+
+    @Override
+    protected List<AprilTagDetection> process(CVMat in) {
+        if (in.getMat().empty()) {
+            return List.of();
+        }
+
+        var ret = m_detector.detect(in.getMat());
+
+        if (ret == null) {
+            return List.of();
+        }
+
+        return List.of(ret);
+    }
+
+    @Override
+    public void setParams(AprilTagDetectionPipeParams newParams) {
+        if (this.params == null || !this.params.equals(newParams)) {
+            m_detector.setConfig(newParams.detectorParams);
+
+            m_detector.clearFamilies();
+            m_detector.addFamily(newParams.family.getNativeName());
+        }
+
+        super.setParams(newParams);
+    }
+
+    public void setNativePoseEstimationEnabled(boolean enabled) {
+        this.useNativePoseEst = enabled;
+    }
+}

photon-core/src/main/java/org/photonvision/vision/pipe/impl/AprilTagDetectionPipeParams.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.vision.pipe.impl;
+
+import edu.wpi.first.apriltag.AprilTagDetector;
+import org.photonvision.vision.apriltag.AprilTagFamily;
+
+public class AprilTagDetectionPipeParams {
+    public final AprilTagFamily family;
+    public final AprilTagDetector.Config detectorParams;
+
+    public AprilTagDetectionPipeParams(AprilTagFamily tagFamily, AprilTagDetector.Config config) {
+        this.family = tagFamily;
+        this.detectorParams = config;
+    }
+
+    @Override
+    public int hashCode() {
+        final int prime = 31;
+        int result = 1;
+        result = prime * result + ((family == null) ? 0 : family.hashCode());
+        result = prime * result + ((detectorParams == null) ? 0 : detectorParams.hashCode());
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null) return false;
+        if (getClass() != obj.getClass()) return false;
+        AprilTagDetectionPipeParams other = (AprilTagDetectionPipeParams) obj;
+        if (family != other.family) return false;
+        if (detectorParams == null) {
+            if (other.detectorParams != null) return false;
+        } else if (!detectorParams.equals(other.detectorParams)) return false;
+        return true;
+    }
+}