Add Websocket Camera Streaming (#529)

* WIP adding second websocket handling for cameras

* just more WIP

* even more wip. Most java-side framework completed, but not yet debugged

* IT LIVES. Still needs lots of cleanup. But we're transferring and displaying data!

* moved down an architecture layer. Improved multiple-camera handling

* Additional WIP to help improve smoothness and performance, though not yet tested

* bugfixes galore

* tweak compression

* spotless

* more tweaks for handling slow/intermittent streams

* wpilibformat maybe?

* clang-format maybe?

* WIP - adding thinclient. I don't like it yet, it should be more auto-generated than it is.

* thinclient formatting fixups

* Reduced amount of empty send data by limiting to only one stream per client (which is all we really need). Framerate is up slightly, overhead is down.

* bugfixes, faster streaming, better mjpeg compression settings, thinclient working

* spotless and formatting

* cmon wpiformat....

* re-added mjpg streams

* added a loading GIF to imporve the feeling of responsiveness

* formatting

* urlparams and built-in thinclient

* wpiformat

* prevent wpiformat complaints

* Removed uint8 array and base64 conversion from client side

* Synced up js implementations for ws streaming

* formatting/spotless

photon-client/src/components/common/cv-image.vue

@@ -5,7 +5,7 @@
     :style="styleObject"
     :src="src"
     alt=""
-    @click="e => $emit('click', e)"
+    @click="e => {this.openThinclientStream(e)}"
   >
 </template>
 
@@ -13,7 +13,7 @@
     export default {
         name: "CvImage",
         // eslint-disable-next-line vue/require-prop-types
-        props: ['address', 'scale', 'maxHeight', 'maxHeightMd', 'maxHeightLg', 'maxHeightXl', 'colorPicking', 'id', 'disconnected'],
+        props: ['idx', 'scale', 'maxHeight', 'maxHeightMd', 'maxHeightLg', 'maxHeightXl', 'colorPicking', 'id', 'disconnected'],
         data() {
             return {
                 seed: 1.0,
@@ -46,18 +46,48 @@
                     return ret;
                 }
             },
-            src: {
+            port: {
               get() {
-                return this.disconnected ? require("../../assets/noStream.jpg") : this.address + "?" + this.seed // This prevents caching
-              },
-            },
+                if(this.idx == 0){
+                  return this.$store.state.cameraSettings[this.$store.state.currentCameraIndex].inputStreamPort;
+                } else {
+                  return this.$store.state.cameraSettings[this.$store.state.currentCameraIndex].outputStreamPort;
+                }
+              }
+            }
+        },
+        watch : {
+          port(newPort, oldPort){
+            newPort;
+            oldPort;
+            this.reload();
+          },
+          disconnected(newVal, oldVal){
+            oldVal;
+            if(newVal){
+              this.wsStream.stopStream();
+            } else {
+              this.wsStream.startStream();
+            }
+          }
         },
         mounted() {
-            this.reload(); // Force reload image on creation
+          var wsvs = require('../../plugins/WebsocketVideoStream');
+          this.wsStream = new wsvs.WebsocketVideoStream(this.id, this.port, window.location.host);
+        },
+        unmounted() {
+          this.wsStream.stopStream();
+          this.wsStream.ws_close();
         },
         methods: {
             reload() {
-                this.seed = new Date().getTime();
+              console.log("Reloading " + this.id + " with port " + String(this.port));
+              this.wsStream.setPort(this.port);
+            },
+            openThinclientStream(e){
+              e;
+              var URL = "/thinclient.html?port=" + String(this.port) + "&host=" + window.location.hostname;
+              window.open(URL, '_blank');
             }
         },
     }

photon-client/src/main.js

@@ -15,11 +15,11 @@ if (process.env.NODE_ENV === "production") {
     Vue.prototype.$address = location.hostname + ":5800";
 }
 
-const wsURL = '//' + Vue.prototype.$address + '/websocket';
+const wsDataURL = '//' + Vue.prototype.$address + '/websocket_data';
 
 import VueNativeSock from 'vue-native-websocket';
 
-Vue.use(VueNativeSock, wsURL, {
+Vue.use(VueNativeSock, wsDataURL, {
     reconnection: true,
     reconnectionDelay: 100,
     connectManually: true,

photon-client/src/plugins/ColorPicker.js

@@ -5,7 +5,7 @@ function initColorPicker() {
     if (!canvas)
         canvas = document.createElement('canvas');
 
-    image = document.querySelector('#normal-stream');
+    image = document.querySelector('#raw-stream');
     if (image !== null) {
       canvas.width = image.width;
       canvas.height = image.height;

photon-client/src/plugins/WebsocketVideoStream.js

@@ -0,0 +1,148 @@
+
+
+export class WebsocketVideoStream{
+
+
+    constructor(drawDiv, streamPort, host) {
+
+        this.drawDiv = drawDiv;
+        this.image = document.getElementById(this.drawDiv);
+        this.streamPort = streamPort;
+        this.serverAddr = "ws://" + host + "/websocket_cameras";
+        this.noStream = false;
+        this.noStreamPrev = false;
+        this.setNoStream();
+        this.ws_connect();
+        this.imgData = null;
+        this.imgDataTime = -1;
+        this.imgObjURL = null;
+        this.frameRxCount = 0;
+
+        requestAnimationFrame(()=>this.animationLoop());
+
+    }
+
+    animationLoop(){
+        var now = window.performance.now();
+
+        if((now - this.imgDataTime) > 2500 && this.imgData != null){
+            //Handle websocket send timeouts by restarting
+            this.setNoStream();
+            this.stopStream();
+            setTimeout(this.startStream.bind(this), 1000); //restart stream one second later
+        } else {
+            if(this.streamPort == null){
+                this.setNoStream();
+            } else if (this.imgData != null) {
+                //From https://stackoverflow.com/questions/67507616/set-image-src-from-image-blob/67507685#67507685
+                if(this.imgObjURL != null){
+                    URL.revokeObjectURL(this.imgObjURL)
+                }
+                this.imgObjURL = URL.createObjectURL(this.imgData);
+
+                //Update the image with the new mimetype and image
+                this.image.src = this.imgObjURL;
+                this.noStream = false;
+
+            } else {
+                //Nothing, hold previous image while waiting for next frame
+            }
+        }
+
+
+        requestAnimationFrame(()=>this.animationLoop());
+    }
+
+    setNoStream() {
+        this.noStreamPrev = this.noStream;
+        this.noStream = true;
+        if(this.noStreamPrev == false && this.noStream == true){
+            //One-shot background change to preserve animation
+            this.image.src = require("../assets/loading.gif");
+        }
+    }
+
+    startStream() {
+        if(this.serverConnectionActive == true && this.streamPort > 0){
+            this.ws.send(JSON.stringify({"cmd": "subscribe", "port":this.streamPort}));
+            this.noStream = false;
+        }
+    }
+
+    stopStream() {
+        if(this.serverConnectionActive == true && this.streamPort > 0){
+            this.ws.send(JSON.stringify({"cmd": "unsubscribe"}));
+            this.noStream = true;
+        }
+    }
+
+    setPort(streamPort){
+        this.stopStream();
+        this.frameRxCount = 0;
+        this.streamPort = streamPort;
+        this.startStream();
+    }
+
+    ws_onOpen() {
+        // Set the flag allowing general server communication
+        this.serverConnectionActive = true;
+        console.log("Connected!");
+        this.startStream();
+    }
+
+    ws_onClose(e) {
+        this.setNoStream();
+
+        //Clear flags to stop server communication
+        this.ws = null;
+        this.serverConnectionActive = false;
+
+        console.log('Camera Socket is closed. Reconnect will be attempted in 0.5 second.', e.reason);
+        setTimeout(this.ws_connect.bind(this), 500);
+
+        if(!e.wasClean){
+            console.error('Socket encountered error!');
+        }
+
+    }
+
+    ws_onError(e){
+        e; //prevent unused failure
+        this.ws.close();
+    }
+
+    ws_onMessage(e){
+        if(typeof e.data === 'string'){
+            //string data from host
+            //TODO - anything to receive info here? Maybe "available streams?"
+        } else {
+            if(e.data.size > 0){
+                //binary data - a frame
+                this.imgData = e.data;
+                this.imgDataTime = window.performance.now();
+                this.frameRxCount++;
+            } else {
+                //TODO - server is sending empty frames?
+            }
+        }
+
+    }
+
+    ws_connect() {
+        this.ws = new WebSocket(this.serverAddr);
+        this.ws.binaryType = "blob";
+        this.ws.onopen = this.ws_onOpen.bind(this);
+        this.ws.onmessage = this.ws_onMessage.bind(this);
+        this.ws.onclose = this.ws_onClose.bind(this);
+        this.ws.onerror = this.ws_onError.bind(this);
+        console.log("Connecting to server " + this.serverAddr);
+    }
+
+    ws_close(){
+        this.ws.close();
+    }
+
+}
+
+
+export default {WebsocketVideoStream}

photon-client/src/store/index.js

@@ -35,8 +35,8 @@ export default new Vuex.Store({
                 tiltDegrees: 0.0,
                 currentPipelineIndex: 0,
                 pipelineNicknames: ["Unknown"],
-                outputStreamPort: 1181,
-                inputStreamPort: 1182,
+                outputStreamPort: 0,
+                inputStreamPort: 0,
                 nickname: "Unknown",
                 videoFormatList: [
                     {

photon-client/src/views/CamerasView.vue

@@ -291,7 +291,8 @@
       >
         <template>
           <CVimage
-            :address="$store.getters.streamAddress[1]"
+            :id="cameras-cal"
+            :idx=1
             :disconnected="!$store.state.backendConnected"
             scale="100"
             style="border-radius: 5px;"

photon-client/src/views/PipelineView.vue

@@ -58,16 +58,16 @@
               >
                 <div style="position: relative; width: 100%; height: 100%;">
                   <cv-image
-                    :id="idx === 0 ? 'normal-stream' : ''"
+                    :id="idx === 0 ? 'raw-stream' : 'processed-stream'"
                     ref="streams"
-                    :address="$store.getters.streamAddress[idx]"
+                    :idx=idx
                     :disconnected="!$store.state.backendConnected"
                     scale="100"
                     :max-height="$store.getters.isDriverMode ? '40vh' : '300px'"
                     :max-height-md="$store.getters.isDriverMode ? '50vh' : '380px'"
                     :max-height-lg="$store.getters.isDriverMode ? '55vh' : '390px'"
                     :max-height-xl="$store.getters.isDriverMode ? '60vh' : '450px'"
-                    :alt="'Stream' + idx"
+                    :alt="'Stream ' + idx"
                     :color-picking="$store.state.colorPicking && idx === 0"
                     @click="onImageClick"
                   />
@@ -85,7 +85,7 @@
           <v-card
             color="primary"
           >
-            <camera-and-pipeline-select @camera-name-changed="reloadStreams" />
+            <camera-and-pipeline-select />
           </v-card>
           <v-card
             :disabled="$store.getters.isDriverMode || $store.state.colorPicking"