Reduce initial connection bandwidth (#1200)

Reduces bandwidth requirements by being much lazier about how much calibration data is sent to the UI.
This commit is contained in:
Matt
2024-02-01 21:42:54 -05:00
committed by GitHub
parent c7508fea46
commit 62112cd2fd
15 changed files with 275 additions and 88 deletions

View File

@@ -25,15 +25,8 @@ const getUniqueVideoFormatsByResolution = (): VideoFormat[] => {
const calib = useCameraSettingsStore().getCalibrationCoeffs(format.resolution);
if (calib !== undefined) {
// Is this the right formula for RMS error? who knows! not me!
const perViewSumSquareReprojectionError = calib.observations.flatMap((it) =>
it.reprojectionErrors.flatMap((it2) => [it2.x, it2.y])
);
// For each error, square it, sum the squares, and divide by total points N
format.mean = Math.sqrt(
perViewSumSquareReprojectionError.map((it) => Math.pow(it, 2)).reduce((a, b) => a + b, 0) /
perViewSumSquareReprojectionError.length
);
format.mean = calib.meanErrors.reduce((a, b) => a + b) / calib.meanErrors.length;
format.horizontalFOV =
2 * Math.atan2(format.resolution.width / 2, calib.cameraIntrinsics.data[0]) * (180 / Math.PI);

View File

@@ -1,51 +1,19 @@
<script setup lang="ts">
import type { BoardObservation, CameraCalibrationResult, VideoFormat } from "@/types/SettingTypes";
import type { CameraCalibrationResult, VideoFormat } from "@/types/SettingTypes";
import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
import { useStateStore } from "@/stores/StateStore";
import { ref } from "vue";
import loadingImage from "@/assets/images/loading.svg";
import { computed, inject, ref } from "vue";
import { getResolutionString, parseJsonFile } from "@/lib/PhotonUtils";
const props = defineProps<{
videoFormat: VideoFormat;
}>();
const getMeanFromView = (o: BoardObservation) => {
// Is this the right formula for RMS error? who knows! not me!
const perViewSumSquareReprojectionError = o.reprojectionErrors.flatMap((it2) => [it2.x, it2.y]);
// For each error, square it, sum the squares, and divide by total points N
return Math.sqrt(
perViewSumSquareReprojectionError.map((it) => Math.pow(it, 2)).reduce((a, b) => a + b, 0) /
perViewSumSquareReprojectionError.length
);
const exportCalibration = ref();
const openExportCalibrationPrompt = () => {
exportCalibration.value.click();
};
// Import and export functions
const downloadCalibration = () => {
const calibData = useCameraSettingsStore().getCalibrationCoeffs(props.videoFormat.resolution);
if (calibData === undefined) {
useStateStore().showSnackbarMessage({
color: "error",
message:
"Calibration data isn't available for the requested resolution, please calibrate the requested resolution first"
});
return;
}
const camUniqueName = useCameraSettingsStore().currentCameraSettings.uniqueName;
const filename = `photon_calibration_${camUniqueName}_${calibData.resolution.width}x${calibData.resolution.height}.json`;
const fileData = JSON.stringify(calibData);
const element = document.createElement("a");
element.style.display = "none";
element.setAttribute("href", "data:text/plain;charset=utf-8," + encodeURIComponent(fileData));
element.setAttribute("download", filename);
document.body.appendChild(element);
element.click();
document.body.removeChild(element);
};
const importCalibrationFromPhotonJson = ref();
const openUploadPhotonCalibJsonPrompt = () => {
importCalibrationFromPhotonJson.value.click();
@@ -97,19 +65,28 @@ const importCalibration = async () => {
};
interface ObservationDetails {
snapshotSrc: any;
mean: number;
index: number;
}
const currentCalibrationCoeffs = computed<CameraCalibrationResult | undefined>(() =>
useCameraSettingsStore().getCalibrationCoeffs(props.videoFormat.resolution)
);
const getObservationDetails = (): ObservationDetails[] | undefined => {
return useCameraSettingsStore()
.getCalibrationCoeffs(props.videoFormat.resolution)
?.observations.map((o, i) => ({
index: i,
mean: parseFloat(getMeanFromView(o).toFixed(2)),
snapshotSrc: o.includeObservationInCalibration ? "data:image/png;base64," + o.snapshotData.data : loadingImage
}));
const coefficients = currentCalibrationCoeffs.value;
return coefficients?.meanErrors.map((m, i) => ({
index: i,
mean: parseFloat(m.toFixed(2))
}));
};
const exportCalibrationURL = computed<string>(() =>
useCameraSettingsStore().getCalJSONUrl(inject("backendHost") as string, props.videoFormat.resolution)
);
const calibrationImageURL = (index: number) =>
useCameraSettingsStore().getCalImageUrl(inject<string>("backendHost") as string, props.videoFormat.resolution, index);
</script>
<template>
@@ -140,19 +117,22 @@ const getObservationDetails = (): ObservationDetails[] | undefined => {
<v-btn
color="secondary"
class="mt-4"
:disabled="useCameraSettingsStore().getCalibrationCoeffs(props.videoFormat.resolution) === undefined"
:disabled="!currentCalibrationCoeffs"
style="width: 100%"
@click="downloadCalibration"
@click="openExportCalibrationPrompt"
>
<v-icon left>mdi-export</v-icon>
<span>Export</span>
</v-btn>
<a
ref="exportCalibration"
style="color: black; text-decoration: none; display: none"
:href="exportCalibrationURL"
target="_blank"
/>
</v-col>
</v-row>
<v-row
v-if="useCameraSettingsStore().getCalibrationCoeffs(props.videoFormat.resolution) !== undefined"
class="pt-2"
>
<v-row v-if="!currentCalibrationCoeffs" class="pt-2">
<v-card-subtitle>Calibration Details</v-card-subtitle>
<v-simple-table dense style="width: 100%" class="pl-2 pr-2">
<template #default>
@@ -231,7 +211,9 @@ const getObservationDetails = (): ObservationDetails[] | undefined => {
</tr>
<tr>
<td>Horizontal FOV</td>
<td>{{ videoFormat.horizontalFOV !== undefined ? videoFormat.horizontalFOV.toFixed(2) + "°" : "-" }}</td>
<td>
{{ videoFormat.horizontalFOV !== undefined ? videoFormat.horizontalFOV.toFixed(2) + "°" : "-" }}
</td>
</tr>
<tr>
<td>Vertical FOV</td>
@@ -242,11 +224,7 @@ const getObservationDetails = (): ObservationDetails[] | undefined => {
<td>{{ videoFormat.diagonalFOV !== undefined ? videoFormat.diagonalFOV.toFixed(2) + "°" : "-" }}</td>
</tr>
<!-- Board warp, only shown for mrcal-calibrated cameras -->
<tr
v-if="
useCameraSettingsStore().getCalibrationCoeffs(props.videoFormat.resolution)?.calobjectWarp?.length === 2
"
>
<tr v-if="currentCalibrationCoeffs?.calobjectWarp?.length === 2">
<td>Board warp, X/Y</td>
<td>
{{
@@ -278,7 +256,7 @@ const getObservationDetails = (): ObservationDetails[] | undefined => {
<template #expanded-item="{ headers, item }">
<td :colspan="headers.length">
<div style="display: flex; justify-content: center; width: 100%">
<img :src="item.snapshotSrc" alt="observation image" class="snapshot-preview pt-2 pb-2" />
<img :src="calibrationImageURL(item.index)" alt="observation image" class="snapshot-preview pt-2 pb-2" />
</div>
</td>
</template>

View File

@@ -416,6 +416,23 @@ export const useCameraSettingsStore = defineStore("cameraSettings", {
cameraIndex: number = useStateStore().currentCameraIndex
): CameraCalibrationResult | undefined {
return this.cameras[cameraIndex].completeCalibrations.find((v) => resolutionsAreEqual(v.resolution, resolution));
},
getCalImageUrl(host: string, resolution: Resolution, idx: number, cameraIdx = useStateStore().currentCameraIndex) {
const url = new URL(`http://${host}/api/utils/getCalSnapshot`);
url.searchParams.set("width", Math.round(resolution.width).toFixed(0));
url.searchParams.set("height", Math.round(resolution.height).toFixed(0));
url.searchParams.set("snapshotIdx", Math.round(idx).toFixed(0));
url.searchParams.set("cameraIdx", Math.round(cameraIdx).toFixed(0));
return url.href;
},
getCalJSONUrl(host: string, resolution: Resolution, cameraIdx = useStateStore().currentCameraIndex) {
const url = new URL(`http://${host}/api/utils/getCalibrationJSON`);
url.searchParams.set("width", Math.round(resolution.width).toFixed(0));
url.searchParams.set("height", Math.round(resolution.height).toFixed(0));
url.searchParams.set("cameraIdx", Math.round(cameraIdx).toFixed(0));
return url.href;
}
}
});

View File

@@ -138,6 +138,9 @@ export interface CameraCalibrationResult {
distCoeffs: JsonMatOfDouble;
observations: BoardObservation[];
calobjectWarp?: number[];
// We might have to omit observations for bandwith, so backend will send us this
numSnapshots: number;
meanErrors: number[];
}
export enum ValidQuirks {
@@ -255,7 +258,9 @@ export const PlaceholderCameraSettings: CameraSettings = {
snapshotName: "img0.png",
snapshotData: { rows: 480, cols: 640, type: CvType.CV_8U, data: "" }
}
]
],
numSnapshots: 1,
meanErrors: [123.45]
}
],
pipelineNicknames: ["Placeholder Pipeline"],

View File

@@ -31,7 +31,7 @@ import org.photonvision.common.util.SerializationUtils;
import org.photonvision.jni.RknnDetectorJNI;
import org.photonvision.mrcal.MrCalJNILoader;
import org.photonvision.raspi.LibCameraJNILoader;
import org.photonvision.vision.calibration.CameraCalibrationCoefficients;
import org.photonvision.vision.calibration.UICameraCalibrationCoefficients;
import org.photonvision.vision.camera.QuirkyCamera;
import org.photonvision.vision.processes.VisionModule;
import org.photonvision.vision.processes.VisionModuleManager;
@@ -126,13 +126,6 @@ public class PhotonConfiguration {
settingsSubmap.put("networkSettings", netConfigMap);
map.put(
"cameraSettings",
VisionModuleManager.getInstance().getModules().stream()
.map(VisionModule::toUICameraConfig)
.map(SerializationUtils::objectToHashMap)
.collect(Collectors.toList()));
var lightingConfig = new UILightingConfig();
lightingConfig.brightness = hardwareSettings.ledBrightnessPercentage;
lightingConfig.supported = !hardwareConfig.ledPins.isEmpty();
@@ -181,7 +174,7 @@ public class PhotonConfiguration {
public HashMap<Integer, HashMap<String, Object>> videoFormatList;
public int outputStreamPort;
public int inputStreamPort;
public List<CameraCalibrationCoefficients> calibrations;
public List<UICameraCalibrationCoefficients> calibrations;
public boolean isFovConfigurable = true;
public QuirkyCamera cameraQuirks;
public boolean isCSICamera;

View File

@@ -26,6 +26,7 @@ import org.photonvision.common.logging.LogGroup;
import org.photonvision.common.logging.Logger;
import org.photonvision.common.util.SerializationUtils;
import org.photonvision.vision.pipeline.result.CVPipelineResult;
import org.photonvision.vision.pipeline.result.CalibrationPipelineResult;
public class UIDataPublisher implements CVPipelineResultConsumer {
private static final Logger logger = new Logger(UIDataPublisher.class, LogGroup.VisionModule);
@@ -41,16 +42,22 @@ public class UIDataPublisher implements CVPipelineResultConsumer {
public void accept(CVPipelineResult result) {
long now = System.currentTimeMillis();
// only update the UI at 15hz
// only update the UI at 10hz
if (lastUIResultUpdateTime + 1000.0 / 10.0 > now) return;
var dataMap = new HashMap<String, Object>();
dataMap.put("fps", result.fps);
dataMap.put("latency", result.getLatencyMillis());
var uiTargets = new ArrayList<HashMap<String, Object>>(result.targets.size());
for (var t : result.targets) {
uiTargets.add(t.toHashMap());
// We don't actually need to send targets during calibration and it can take up a lot (up to
// 1.2Mbps for 60 snapshots) of target results with no pitch/yaw/etc set
if (!(result instanceof CalibrationPipelineResult)) {
for (var t : result.targets) {
uiTargets.add(t.toHashMap());
}
}
dataMap.put("targets", uiTargets);
dataMap.put("classNames", result.objectDetectionClassNames);

View File

@@ -24,7 +24,7 @@ import java.util.List;
import org.opencv.core.Point;
import org.opencv.core.Point3;
public final class BoardObservation {
public final class BoardObservation implements Cloneable {
// Expected feature 3d location in the camera frame
@JsonProperty("locationInObjectSpace")
public List<Point3> locationInObjectSpace;
@@ -68,4 +68,33 @@ public final class BoardObservation {
this.snapshotName = snapshotName;
this.snapshotData = snapshotData;
}
@Override
public String toString() {
return "BoardObservation [locationInObjectSpace="
+ locationInObjectSpace
+ ", locationInImageSpace="
+ locationInImageSpace
+ ", reprojectionErrors="
+ reprojectionErrors
+ ", optimisedCameraToObject="
+ optimisedCameraToObject
+ ", includeObservationInCalibration="
+ includeObservationInCalibration
+ ", snapshotName="
+ snapshotName
+ ", snapshotData="
+ snapshotData
+ "]";
}
@Override
public BoardObservation clone() {
try {
return (BoardObservation) super.clone();
} catch (CloneNotSupportedException e) {
System.err.println("Guhhh clone buh");
return null;
}
}
}

View File

@@ -191,8 +191,8 @@ public class CameraCalibrationCoefficients implements Releasable {
+ cameraIntrinsics
+ ", distCoeffs="
+ distCoeffs
+ ", observations="
+ observations
+ ", observationslen="
+ observations.size()
+ ", calobjectWarp="
+ Arrays.toString(calobjectWarp)
+ ", intrinsicsArr="
@@ -201,4 +201,16 @@ public class CameraCalibrationCoefficients implements Releasable {
+ Arrays.toString(distCoeffsArr)
+ "]";
}
public UICameraCalibrationCoefficients cloneWithoutObservations() {
return new UICameraCalibrationCoefficients(
resolution,
cameraIntrinsics,
distCoeffs,
calobjectWarp,
observations,
calobjectSize,
calobjectSpacing,
lensmodel);
}
}

View File

@@ -76,4 +76,17 @@ public class JsonImageMat implements Releasable {
public void release() {
if (wrappedMat != null) wrappedMat.release();
}
@Override
public String toString() {
return "JsonImageMat [rows="
+ rows
+ ", cols="
+ cols
+ ", type="
+ type
+ ", datalen="
+ data.length()
+ "]";
}
}

View File

@@ -40,7 +40,7 @@ public class JsonMatOfDouble implements Releasable {
@JsonIgnore private Mat wrappedMat = null;
@JsonIgnore private Matrix wpilibMat = null;
private MatOfDouble wrappedMatOfDouble;
@JsonIgnore private MatOfDouble wrappedMatOfDouble;
public JsonMatOfDouble(int rows, int cols, double[] data) {
this(rows, cols, CvType.CV_64FC1, data);

View File

@@ -0,0 +1,59 @@
/*
* Copyright (C) Photon Vision.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.photonvision.vision.calibration;
import java.util.List;
import java.util.stream.Collectors;
import org.opencv.core.Size;
public class UICameraCalibrationCoefficients extends CameraCalibrationCoefficients {
public int numSnapshots;
public List<Double> meanErrors;
public UICameraCalibrationCoefficients(
Size resolution,
JsonMatOfDouble cameraIntrinsics,
JsonMatOfDouble distCoeffs,
double[] calobjectWarp,
List<BoardObservation> observations,
Size calobjectSize,
double calobjectSpacing,
CameraLensModel lensmodel) {
// yeet observations, keep all else
super(
resolution,
cameraIntrinsics,
distCoeffs,
calobjectWarp,
List.of(),
calobjectSize,
calobjectSpacing,
lensmodel);
this.numSnapshots = observations.size();
this.meanErrors =
observations.stream()
.map(
it2 ->
it2.reprojectionErrors.stream()
.mapToDouble(it -> Math.sqrt(it.x * it.x + it.y * it.y))
.average()
.orElse(0))
.collect(Collectors.toList());
}
}

View File

@@ -26,6 +26,7 @@ import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.function.BiConsumer;
import java.util.stream.Collectors;
import org.opencv.core.Size;
import org.photonvision.common.configuration.CameraConfiguration;
import org.photonvision.common.configuration.ConfigManager;
@@ -536,7 +537,10 @@ public class VisionModule {
ret.outputStreamPort = this.outputStreamPort;
ret.inputStreamPort = this.inputStreamPort;
ret.calibrations = visionSource.getSettables().getConfiguration().calibrations;
ret.calibrations =
visionSource.getSettables().getConfiguration().calibrations.stream()
.map(CameraCalibrationCoefficients::cloneWithoutObservations)
.collect(Collectors.toList());
ret.isFovConfigurable =
!(ConfigManager.getInstance().getConfig().getHardwareConfig().hasPresetFOV()

View File

@@ -350,8 +350,7 @@ public class DataSocketHandler {
}
}
private void sendMessage(Object message, WsContext user) throws JsonProcessingException {
ByteBuffer b = ByteBuffer.wrap(objectMapper.writeValueAsBytes(message));
private void sendMessage(ByteBuffer b, WsContext user) throws JsonProcessingException {
if (user.session.isOpen()) {
user.send(b);
}
@@ -359,16 +358,18 @@ public class DataSocketHandler {
public void broadcastMessage(Object message, WsContext userToSkip)
throws JsonProcessingException {
ByteBuffer b = ByteBuffer.wrap(objectMapper.writeValueAsBytes(message));
if (userToSkip == null) {
for (WsContext user : users) {
sendMessage(message, user);
sendMessage(b, user);
}
} else {
var skipUserPort = ((InetSocketAddress) userToSkip.session.getRemoteAddress()).getPort();
for (WsContext user : users) {
var userPort = ((InetSocketAddress) user.session.getRemoteAddress()).getPort();
if (userPort != skipUserPort) {
sendMessage(message, user);
sendMessage(b, user);
}
}
}

View File

@@ -31,6 +31,9 @@ import java.util.HashMap;
import java.util.Optional;
import javax.imageio.ImageIO;
import org.apache.commons.io.FileUtils;
import org.opencv.core.MatOfByte;
import org.opencv.core.MatOfInt;
import org.opencv.imgcodecs.Imgcodecs;
import org.photonvision.common.configuration.ConfigManager;
import org.photonvision.common.configuration.NetworkConfig;
import org.photonvision.common.dataflow.DataChangeDestination;
@@ -580,6 +583,77 @@ public class RequestHandler {
ctx.status(204);
}
public static void onCalibrationSnapshotRequest(Context ctx) {
logger.info(ctx.queryString().toString());
int idx = Integer.parseInt(ctx.queryParam("cameraIdx"));
var width = Integer.parseInt(ctx.queryParam("width"));
var height = Integer.parseInt(ctx.queryParam("height"));
var observationIdx = Integer.parseInt(ctx.queryParam("snapshotIdx"));
CameraCalibrationCoefficients calList =
VisionModuleManager.getInstance()
.getModule(idx)
.getStateAsCameraConfig()
.calibrations
.stream()
.filter(
it ->
Math.abs(it.resolution.width - width) < 1e-4
&& Math.abs(it.resolution.height - height) < 1e-4)
.findFirst()
.orElse(null);
if (calList == null || calList.observations.size() < observationIdx) {
ctx.status(404);
return;
}
// encode as jpeg to save even more space. reduces size of a 1280p image from 300k to 25k
var jpegBytes = new MatOfByte();
Imgcodecs.imencode(
".jpg",
calList.observations.get(observationIdx).snapshotData.getAsMat(),
jpegBytes,
new MatOfInt(Imgcodecs.IMWRITE_JPEG_QUALITY, 60));
ctx.result(jpegBytes.toArray());
jpegBytes.release();
ctx.status(200);
}
public static void onCalibrationExportRequest(Context ctx) {
logger.info(ctx.queryString().toString());
int idx = Integer.parseInt(ctx.queryParam("cameraIdx"));
var width = Integer.parseInt(ctx.queryParam("width"));
var height = Integer.parseInt(ctx.queryParam("height"));
var cc = VisionModuleManager.getInstance().getModule(idx).getStateAsCameraConfig();
CameraCalibrationCoefficients calList =
cc.calibrations.stream()
.filter(
it ->
Math.abs(it.resolution.width - width) < 1e-4
&& Math.abs(it.resolution.height - height) < 1e-4)
.findFirst()
.orElse(null);
if (calList == null) {
ctx.status(404);
return;
}
var filename = "photon_calibration_" + cc.uniqueName + "_" + width + "x" + height + ".json";
ctx.contentType("application/zip");
ctx.header("Content-Disposition", "attachment; filename=\"" + filename + "\"");
ctx.json(calList);
ctx.status(200);
}
public static void onImageSnapshotsRequest(Context ctx) {
var snapshots = new ArrayList<HashMap<String, Object>>();
var cameraDirs = ConfigManager.getInstance().getImageSavePath().toFile().listFiles();

View File

@@ -130,6 +130,8 @@ public class Server {
app.post("/api/utils/restartDevice", RequestHandler::onDeviceRestartRequest);
app.post("/api/utils/publishMetrics", RequestHandler::onMetricsPublishRequest);
app.get("/api/utils/getImageSnapshots", RequestHandler::onImageSnapshotsRequest);
app.get("/api/utils/getCalSnapshot", RequestHandler::onCalibrationSnapshotRequest);
app.get("/api/utils/getCalibrationJSON", RequestHandler::onCalibrationExportRequest);
// Calibration
app.post("/api/calibration/end", RequestHandler::onCalibrationEndRequest);