Team2890/PhotonVision
4
0
Fork 0
mirror of https://github.com/PhotonVision/photonvision synced 2026-06-19 00:41:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add Websocket Camera Streaming (#529)

Browse Source 
* WIP adding second websocket handling for cameras

* just more WIP

* even more wip. Most java-side framework completed, but not yet debugged

* IT LIVES. Still needs lots of cleanup. But we're transferring and displaying data!

* moved down an architecture layer. Improved multiple-camera handling

* Additional WIP to help improve smoothness and performance, though not yet tested

* bugfixes galore

* tweak compression

* spotless

* more tweaks for handling slow/intermittent streams

* wpilibformat maybe?

* clang-format maybe?

* WIP - adding thinclient. I don't like it yet, it should be more auto-generated than it is.

* thinclient formatting fixups

* Reduced amount of empty send data by limiting to only one stream per client (which is all we really need). Framerate is up slightly, overhead is down.

* bugfixes, faster streaming, better mjpeg compression settings, thinclient working

* spotless and formatting

* cmon wpiformat....

* re-added mjpg streams

* added a loading GIF to imporve the feeling of responsiveness

* formatting

* urlparams and built-in thinclient

* wpiformat

* prevent wpiformat complaints

* Removed uint8 array and base64 conversion from client side

* Synced up js implementations for ws streaming

* formatting/spotless
This commit is contained in:
Chris Gerth
2022-10-30 13:16:17 -05:00
committed by GitHub
parent b68b0ca5f6
commit aaac6a4fbb
23 changed files with 890 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
photon-server/src/main/java/org/photonvision/server/CameraSocketHandler.java Normal file
View File

@@ -0,0 +1,140 @@
/*
* Copyright (C) Photon Vision.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.photonvision.server;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.javalin.websocket.WsBinaryMessageContext;
import io.javalin.websocket.WsCloseContext;
import io.javalin.websocket.WsConnectContext;
import io.javalin.websocket.WsContext;
import io.javalin.websocket.WsMessageContext;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import org.photonvision.common.logging.LogGroup;
import org.photonvision.common.logging.Logger;
import org.photonvision.vision.videoStream.SocketVideoStreamManager;
public class CameraSocketHandler {
private final Logger logger = new Logger(CameraSocketHandler.class, LogGroup.WebServer);
private final List<WsContext> users = new CopyOnWriteArrayList<>();
private final SocketVideoStreamManager svsManager = SocketVideoStreamManager.getInstance();
private Thread cameraBroadcastThread;
public static class UIMap extends HashMap<String, Object> {}
private static class ThreadSafeSingleton {
private static final CameraSocketHandler INSTANCE = new CameraSocketHandler();
}
public static CameraSocketHandler getInstance() {
return CameraSocketHandler.ThreadSafeSingleton.INSTANCE;
}
private CameraSocketHandler() {
cameraBroadcastThread = new Thread(this::broadcastFramesTask);
cameraBroadcastThread.setPriority(2); // fairly low priority
cameraBroadcastThread.start();
}
public void onConnect(WsConnectContext context) {
context.session.setIdleTimeout(Long.MAX_VALUE); // TODO: determine better value
var insa = context.session.getRemote().getInetSocketAddress();
var host = insa.getAddress().toString() + ":" + insa.getPort();
logger.info("New camera websocket connection from " + host);
users.add(context);
}
protected void onClose(WsCloseContext context) {
var insa = context.session.getRemote().getInetSocketAddress();
var host = insa.getAddress().toString() + ":" + insa.getPort();
var reason = context.reason() != null ? context.reason() : "Connection closed by client";
logger.info("Closing camera websocket connection from " + host + " for reason: " + reason);
svsManager.removeSubscription(context);
users.remove(context);
}
@SuppressWarnings({"unchecked"})
public void onMessage(WsMessageContext context) {
var messageStr = context.message();
ObjectMapper mapper = new ObjectMapper();
try {
JsonNode actualObj = mapper.readTree(messageStr);
try {
var entryCmd = actualObj.get("cmd").asText();
var socketMessageType = CameraSocketMessageType.fromEntryKey(entryCmd);
logger.trace(() -> "Got Camera WS message: [" + socketMessageType + "]");
if (socketMessageType == null) {
logger.warn("Got unknown socket message command: " + entryCmd);
}
switch (socketMessageType) {
case CSMT_SUBSCRIBE:
{
int portId = actualObj.get("port").asInt();
svsManager.addSubscription(context, portId);
break;
}
case CSMT_UNSUBSCRIBE:
{
svsManager.removeSubscription(context);
break;
}
}
} catch (Exception e) {
logger.error("Failed to parse message!", e);
}
} catch (JsonProcessingException e) {
logger.warn("Could not parse message \"" + messageStr + "\"");
e.printStackTrace();
return;
}
}
@SuppressWarnings({"unchecked"})
public void onBinaryMessage(WsBinaryMessageContext context) {
return; // ignoring binary messages for now
}
private void broadcastFramesTask() {
// Background camera image broadcasting thread
while (!Thread.currentThread().isInterrupted()) {
svsManager.allStreamConvertNextFrame();
try {
Thread.sleep(1);
} catch (InterruptedException e) {
logger.error("Exception waiting for camera stream broadcast semaphore", e);
}
for (var user : users) {
var sendBytes = svsManager.getSendFrame(user);
if (sendBytes != null) {
user.send(sendBytes);
}
}
}
}
}

46
photon-server/src/main/java/org/photonvision/server/CameraSocketMessageType.java Normal file
View File

@@ -0,0 +1,46 @@
/*
* Copyright (C) Photon Vision.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.photonvision.server;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Map;
@SuppressWarnings("unused")
public enum CameraSocketMessageType {
CSMT_SUBSCRIBE("subscribe"),
CSMT_UNSUBSCRIBE("unsubscribe");
public final String entryKey;
CameraSocketMessageType(String entryKey) {
this.entryKey = entryKey;
}
private static final Map<String, CameraSocketMessageType> entryKeyToValueMap = new HashMap<>();
static {
for (var value : EnumSet.allOf(CameraSocketMessageType.class)) {
entryKeyToValueMap.put(value.entryKey, value);
}
}
public static CameraSocketMessageType fromEntryKey(String entryKey) {
return entryKeyToValueMap.get(entryKey);
}
}

27
photon-server/src/main/java/org/photonvision/server/SocketHandler.java → photon-server/src/main/java/org/photonvision/server/DataSocketHandler.java
View File

@@ -42,8 +42,8 @@ import org.photonvision.common.logging.Logger;
import org.photonvision.vision.pipeline.PipelineType;
@SuppressWarnings("rawtypes")
public class SocketHandler {
private final Logger logger = new Logger(SocketHandler.class, LogGroup.WebServer);
public class DataSocketHandler {
private final Logger logger = new Logger(DataSocketHandler.class, LogGroup.WebServer);
private final List<WsContext> users = new CopyOnWriteArrayList<>();
private final ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory());
private final DataChangeService dcService = DataChangeService.getInstance();
@@ -54,14 +54,14 @@ public class SocketHandler {
public static class UIMap extends HashMap<String, Object> {}
private static class ThreadSafeSingleton {
private static final SocketHandler INSTANCE = new SocketHandler();
private static final DataSocketHandler INSTANCE = new DataSocketHandler();
}
public static SocketHandler getInstance() {
return SocketHandler.ThreadSafeSingleton.INSTANCE;
public static DataSocketHandler getInstance() {
return DataSocketHandler.ThreadSafeSingleton.INSTANCE;
}
private SocketHandler() {
private DataSocketHandler() {
dcService.addSubscribers(
uiOutboundSubscriber,
new UIInboundSubscriber()); // Subscribe outgoing messages to the data change service
@@ -84,19 +84,6 @@ public class SocketHandler {
var reason = context.reason() != null ? context.reason() : "Connection closed by client";
logger.info("Closing websocket connection from " + host + " for reason: " + reason);
users.remove(context);
if (users.size() == 0) {
logger.info("All websocket connections are closed. Setting inputShouldShow to false.");
// cameraIndex -1 means the event is received by all cameras
dcService.publishEvent(
new IncomingWebSocketEvent<>(
DataChangeDestination.DCD_ACTIVEPIPELINESETTINGS,
"inputShouldShow",
false,
-1,
null));
}
}
@SuppressWarnings({"unchecked"})
@@ -117,7 +104,7 @@ public class SocketHandler {
try {
var entryKey = entry.getKey();
var entryValue = entry.getValue();
var socketMessageType = SocketMessageType.fromEntryKey(entryKey);
var socketMessageType = DataSocketMessageType.fromEntryKey(entryKey);
logger.trace(
() ->

10
photon-server/src/main/java/org/photonvision/server/SocketMessageType.java → photon-server/src/main/java/org/photonvision/server/DataSocketMessageType.java
View File

@@ -22,7 +22,7 @@ import java.util.HashMap;
import java.util.Map;
@SuppressWarnings("unused")
public enum SocketMessageType {
public enum DataSocketMessageType {
SMT_DRIVERMODE("driverMode"),
SMT_CHANGECAMERANAME("changeCameraName"),
SMT_CHANGEPIPELINENAME("changePipelineName"),
@@ -40,19 +40,19 @@ public enum SocketMessageType {
public final String entryKey;
SocketMessageType(String entryKey) {
DataSocketMessageType(String entryKey) {
this.entryKey = entryKey;
}
private static final Map<String, SocketMessageType> entryKeyToValueMap = new HashMap<>();
private static final Map<String, DataSocketMessageType> entryKeyToValueMap = new HashMap<>();
static {
for (var value : EnumSet.allOf(SocketMessageType.class)) {
for (var value : EnumSet.allOf(DataSocketMessageType.class)) {
entryKeyToValueMap.put(value.entryKey, value);
}
}
public static SocketMessageType fromEntryKey(String entryKey) {
public static DataSocketMessageType fromEntryKey(String entryKey) {
return entryKeyToValueMap.get(entryKey);
}
}

23
photon-server/src/main/java/org/photonvision/server/Server.java
View File

@@ -61,15 +61,24 @@ public class Server {
})));
});
var socketHandler = SocketHandler.getInstance();
/*Web Socket Events */
/*Web Socket Events for Data Exchage */
var dsHandler = DataSocketHandler.getInstance();
app.ws(
"/websocket",
"/websocket_data",
ws -> {
ws.onConnect(socketHandler::onConnect);
ws.onClose(socketHandler::onClose);
ws.onBinaryMessage(socketHandler::onBinaryMessage);
ws.onConnect(dsHandler::onConnect);
ws.onClose(dsHandler::onClose);
ws.onBinaryMessage(dsHandler::onBinaryMessage);
});
/*Web Socket Events for Camera Streaming */
var camDsHandler = CameraSocketHandler.getInstance();
app.ws(
"/websocket_cameras",
ws -> {
ws.onConnect(camDsHandler::onConnect);
ws.onClose(camDsHandler::onClose);
ws.onBinaryMessage(camDsHandler::onBinaryMessage);
ws.onMessage(camDsHandler::onMessage);
});
/*API Events*/
app.post("/api/settings/import", RequestHandler::onSettingUpload);

4
photon-server/src/main/java/org/photonvision/server/UIOutboundSubscriber.java
View File

@@ -35,9 +35,9 @@ import org.photonvision.common.logging.Logger;
class UIOutboundSubscriber extends DataChangeSubscriber {
Logger logger = new Logger(UIOutboundSubscriber.class, LogGroup.WebServer);
private final SocketHandler socketHandler;
private final DataSocketHandler socketHandler;
public UIOutboundSubscriber(SocketHandler socketHandler) {
public UIOutboundSubscriber(DataSocketHandler socketHandler) {
super(DataChangeSource.AllSources, Collections.singletonList(DataChangeDestination.DCD_UI));
this.socketHandler = socketHandler;
}