From 678961e4f2a1ce587f041977ec32dda2241d4baa Mon Sep 17 00:00:00 2001
From: Chris Gerth <gerth2@users.noreply.github.com>
Date: Thu, 1 Dec 2022 13:42:21 -0500
Subject: [PATCH] Websocket Stream Stats & Robustness Improvements (#605)

* wip support for a stats overlay

* WIP adding stream stats. But.... eeeh. Corporate.

* kbits over mbytes

* A ton more tweaks:

- Increased thread priority for streaming to reduce "stutter/slow" issues
- revised client side URL creation order to prevent the possibility of repeat-identical URL's
- Improved overlay to only be visible on mouseover, and fully centered in the screen

* wpiformat on js
---
 .../src/components/common/cv-image.vue        |   1 +
 .../src/components/common/cv-input.vue        |   2 +-
 .../src/plugins/WebsocketVideoStream.js       | 171 +++++++++++++++---
 photon-client/src/views/PipelineView.vue      |   2 +-
 .../vision/videoStream/SocketVideoStream.java |  11 --
 .../server/CameraSocketHandler.java           |   2 +-
 6 files changed, 147 insertions(+), 42 deletions(-)

diff --git a/photon-client/src/components/common/cv-image.vue b/photon-client/src/components/common/cv-image.vue
index b2ccba404..b13ece41f 100644
--- a/photon-client/src/components/common/cv-image.vue
+++ b/photon-client/src/components/common/cv-image.vue
@@ -9,6 +9,7 @@
   >
 </template>
 
+
 <script>
     export default {
         name: "CvImage",
diff --git a/photon-client/src/components/common/cv-input.vue b/photon-client/src/components/common/cv-input.vue
index 256cb9f08..fbc340db4 100644
--- a/photon-client/src/components/common/cv-input.vue
+++ b/photon-client/src/components/common/cv-input.vue
@@ -26,7 +26,7 @@
     </v-row>
   </div>
 </template>
-s
+
 <script>
     import TooltippedLabel from "./cv-tooltipped-label";
 
diff --git a/photon-client/src/plugins/WebsocketVideoStream.js b/photon-client/src/plugins/WebsocketVideoStream.js
index 65912a306..5d2ea8b1b 100644
--- a/photon-client/src/plugins/WebsocketVideoStream.js
+++ b/photon-client/src/plugins/WebsocketVideoStream.js
@@ -1,8 +1,62 @@
+// Circular buffer storage. Externally-apparent 'length' increases indefinitely
+// while any items with indexes below length-n will be forgotten (undefined
+// will be returned if you try to get them, trying to set is an exception).
+// n represents the initial length of the array, not a maximum
+class StatsHistoryBuffer{
+    constructor (){
+        this.windowLen = 10;
+        this._array= new Array(this.windowLen);
+        this.headPtr = 0;
+        this.frameCount = 0;
+        this.bitAvgAccum = 0;
+
+        //calculated vals
+        this.bitRate_Mbps = 0;
+        this.framerate_fps = 0;
+    }
+
+    putAndPop(v){
+        this.headPtr++;
+        var idx = (this.headPtr)%this._array.length;
+        var poppedVal = this._array[idx];
+        this._array[idx] = v;
+        return poppedVal;
+    }
+
+    addSample(time, frameSize_bits, dispFrame_count) {
+        var oldVal = this.putAndPop([time, frameSize_bits, dispFrame_count]);
+
+        this.bitAvgAccum += frameSize_bits;
+
+        if(oldVal !=null){
+            var oldTime = oldVal[0];
+            var oldFrameSize = oldVal[1];
+            var oldFrameCount = oldVal[2];
+
+            var deltaTime_s = (time - oldTime);
+
+            this.bitAvgAccum -= oldFrameSize;
+
+            //bitrate - total bits transferred over the time period, divided by the period length
+            // converted to mbps
+            this.bitRate_Mbps = ( this.bitAvgAccum / deltaTime_s ) * (1.0/1048576.0);
+
+            //framerate - total frames displayed over the time period, divided by the period length
+            this.framerate_fps = (dispFrame_count - oldFrameCount) / deltaTime_s;
+        }
+
+
+    }
+
+    getText(){
+        return "Streaming @ " + this.framerate_fps.toFixed(1) + "FPS  " + this.bitRate_Mbps.toFixed(1) + "Mbps";
+    }
+
+}
 
 
 export class WebsocketVideoStream{
 
-
     constructor(drawDiv, streamPort, host) {
         console.log("host " + host + " port " + streamPort)
 
@@ -11,39 +65,81 @@ export class WebsocketVideoStream{
         this.streamPort = streamPort;
         this.newStreamPortReq = null;
         this.serverAddr = "ws://" + host + "/websocket_cameras";
-        this.dispNoStream();
-        this.ws_connect();
         this.imgData = null;
         this.imgDataTime = -1;
+        this.prevImgDataTime = -1;
         this.imgObjURL = null;
         this.frameRxCount = 0;
+        this.dispFrameCount = 0;
+        this.stats = null;
 
-        //Display state machine
-        this.DSM_DISCONNECTED = "DISCONNECTED";
-        this.DSM_WAIT_FOR_VALID_PORT = "WAIT_FOR_VALID_PORT";
-        this.DSM_SUBSCRIBE = "SUBSCRIBE";
-        this.DSM_WAIT_FOR_FIRST_FRAME = "WAIT_FOR_FIRST_FRAME";
-        this.DSM_SHOWING = "SHOWING";
-        this.DSM_RESTART_UNSUBSCRIBE = "UNSUBSCRIBE";
-        this.DSM_RESTART_WAIT = "WAIT_BEFORE_SUBSCRIBE";
+        //Set up div for stream stats info provided for users
+        this.statsTextDiv = this.image.parentNode.appendChild(document.createElement("div"));
+
+        //Centered over the image
+        this.statsTextDiv.style.position = "absolute";
+        this.statsTextDiv.style.left = "50%";
+        this.statsTextDiv.style.top = "50%";
+        this.statsTextDiv.style.transform = "translate(-50%, -50%)";
+
+        // Big enough for a line or two of text, with centered text
+        this.statsTextDiv.style.padding = "0.5em"
+        this.statsTextDiv.style.overflow = "hidden";
+        this.statsTextDiv.style.textAlign = "center";
+        this.statsTextDiv.style.verticalAlign = "middle";
+
+        // Styled to be black with grey text
+        this.statsTextDiv.style.backgroundColor = "black";
+        this.statsTextDiv.style.color = "#9E9E9E";
+        this.statsTextDiv.style.borderRadius = "3px";
+
+        //Default no text
+        this.statsTextDiv.innerHTML = "";
+
+        // Only show on mouseover, with opacity fade-in/fade-out
+        this.statsTextDiv.style.opacity = "0.0";
+        this.statsTextDiv.style.transition = "opacity 0.25s ease 0.25s";
+        this.statsTextDiv.style.transitionDelay = "opacity 0.5s";
+        this.image.addEventListener('mouseover', () => {this.statsTextDiv.style.opacity = "0.6";});
+        this.statsTextDiv.addEventListener('mouseover', () => {this.statsTextDiv.style.opacity = "0.6";});
+        this.image.addEventListener('mouseout', () => {this.statsTextDiv.style.opacity = "0.0";});
+
+        //Display state machine descriptions
+        this.DSM_DISCONNECTED = "Disconnected";
+        this.DSM_WAIT_FOR_VALID_PORT = "Waiting for valid port ID";
+        this.DSM_SUBSCRIBE = "Subscribing";
+        this.DSM_WAIT_FOR_FIRST_FRAME = "Waiting for frame data";
+        this.DSM_SHOWING = "Showing Frames";
+        this.DSM_RESTART_UNSUBSCRIBE = "Unsubscribing";
+        this.DSM_RESTART_WAIT = "Waiting before resubscribe";
 
         this.dsm_cur_state = this.DSM_DISCONNECTED;
         this.dsm_prev_state = this.DSM_DISCONNECTED;
         this.dsm_restart_start_time = window.performance.now();
 
+        this.dispNoStream();
+        this.ws_connect();
+
         requestAnimationFrame(()=>this.animationLoop());
 
     }
 
     dispImageData(){
-        //From https://stackoverflow.com/questions/67507616/set-image-src-from-image-blob/67507685#67507685
-        if(this.imgObjURL != null){
-            URL.revokeObjectURL(this.imgObjURL)
-        }
-        this.imgObjURL = URL.createObjectURL(this.imgData);
+        if(this.prevImgDataTime != this.imgDataTime){
+            //From https://stackoverflow.com/questions/67507616/set-image-src-from-image-blob/67507685#67507685
+            //Ensure uniqueness by making the new one before revoking the old one.
+            var oldURL = this.imgObjURL
+            this.imgObjURL = URL.createObjectURL(this.imgData);
+            if(oldURL != null){
+                URL.revokeObjectURL(oldURL)
+            }
 
-        //Update the image with the new mimetype and image
-        this.image.src = this.imgObjURL;
+            //Update the image with the new mimetype and image
+            this.image.src = this.imgObjURL;
+
+            this.dispFrameCount++;
+            this.prevImgDataTime = this.imgDataTime;
+        } // else no new image, don't update anything
     }
 
     dispNoStream() {
@@ -52,8 +148,8 @@ export class WebsocketVideoStream{
 
     animationLoop(){
         // Update time metrics
-        var now = window.performance.now();
-        var timeInState  = now - this.dsm_restart_start_time;
+        var curTime_s = window.performance.now() / 1000.0;
+        var timeInState  = curTime_s - this.dsm_restart_start_time;
 
         // Save previous state
         this.dsm_prev_state = this.dsm_cur_state;
@@ -93,7 +189,7 @@ export class WebsocketVideoStream{
                     }
                     break;
                 case this.DSM_SHOWING:
-                    if((now - this.imgDataTime) > 2500){
+                    if((curTime_s - this.imgDataTime) > 2.5){
                         //timeout, begin the restart sequence
                         this.dsm_cur_state = this.DSM_RESTART_UNSUBSCRIBE;
                     } else if (this.newStreamPortReq != null){
@@ -109,7 +205,7 @@ export class WebsocketVideoStream{
                     this.dsm_cur_state = this.DSM_RESTART_WAIT;
                     break;
                 case this.DSM_RESTART_WAIT:
-                    if (timeInState > 250) {
+                    if (timeInState > 0.25) {
                         //we've waited long enough, go to try to re-subscribe
                         this.dsm_cur_state = this.DSM_WAIT_FOR_VALID_PORT;
                     } else {
@@ -132,7 +228,12 @@ export class WebsocketVideoStream{
 
         if(this.dsm_cur_state == this.DSM_SHOWING){
             // Currently in SHOWING
+            // Show image and update status text
             this.dispImageData();
+            this.statsTextDiv.innerHTML = this.stats.getText();
+        } else {
+            //Just show the state for debug
+            this.statsTextDiv.innerHTML = this.dsm_cur_state;
         }
 
         if(this.dsm_cur_state != this.DSM_SHOWING && this.dsm_prev_state == this.DSM_SHOWING ){
@@ -143,13 +244,13 @@ export class WebsocketVideoStream{
         if(this.dsm_cur_state == this.DSM_RESTART_UNSUBSCRIBE){
             // Currently in UNSUBSCRIBE, do the unsubscribe actions
             this.stopStream();
-            this.dsm_restart_start_time = now;
+            this.dsm_restart_start_time = curTime_s;
         }
 
         if(this.dsm_cur_state == this.DSM_SUBSCRIBE){
             // Currently in SUBSCRIBE, do the subscribe actions
             this.startStream();
-            this.dsm_restart_start_time = now;
+            this.dsm_restart_start_time = curTime_s;
         }
 
         if(this.dsm_cur_state == this.DSM_WAIT_FOR_VALID_PORT){
@@ -160,6 +261,8 @@ export class WebsocketVideoStream{
             }
         }
 
+        //Update status text
+
         requestAnimationFrame(()=>this.animationLoop());
     }
 
@@ -184,6 +287,11 @@ export class WebsocketVideoStream{
         // Set the flag allowing general server communication
         this.serverConnectionActive = true;
         console.log("Camera Websockets Connected!");
+
+        // New websocket connection, reset stats
+        this.frameRxCount = 0;
+        this.dispFrameCount = 0;
+        this.stats = new StatsHistoryBuffer();
     }
 
     ws_onClose(e) {
@@ -206,18 +314,25 @@ export class WebsocketVideoStream{
     }
 
     ws_onMessage(e){
-        console.log("Got message from " + this.serverAddr)
+        //console.log("Got message from " + this.serverAddr)
+        var msgTime_s = window.performance.now() / 1000.0;
         if(typeof e.data === 'string'){
             //string data from host
             //TODO - anything to receive info here? Maybe "available streams?"
         } else {
             if(e.data.size > 0){
-                //binary data - a frame
+                //binary data - a frame!
+                //Save frame data for display in the next animation thread
                 this.imgData = e.data;
-                this.imgDataTime = window.performance.now();
+                this.imgDataTime = msgTime_s;
+
+                //Count the incoming frame
                 this.frameRxCount++;
+
+                //keep the stats up to date
+                this.stats.addSample(msgTime_s,this.imgData.size * 8,this.dispFrameCount);
             } else {
-                //TODO - server is sending empty frames?
+                console.log("WS Stream Error: Server sent empty frame!");
             }
         }
 
diff --git a/photon-client/src/views/PipelineView.vue b/photon-client/src/views/PipelineView.vue
index 1cb9bcf2c..b665f64b0 100644
--- a/photon-client/src/views/PipelineView.vue
+++ b/photon-client/src/views/PipelineView.vue
@@ -33,7 +33,7 @@
                 :color="fpsTooLow ? 'error' : 'transparent'"
                 :text-color="fpsTooLow ? 'white' : 'grey'"
               >
-                <span class="pr-1">{{ Math.round($store.state.pipelineResults.fps) }}&nbsp;FPS &ndash;</span>
+                <span class="pr-1">Processing @ {{ Math.round($store.state.pipelineResults.fps) }}&nbsp;FPS &ndash;</span>
                 <span v-if="!fpsTooLow">{{ Math.min(Math.round($store.state.pipelineResults.latency), 9999) }} ms latency</span>
                 <span v-else-if="!$store.getters.currentPipelineSettings.inputShouldShow">HSV thresholds are too broad; narrow them for better performance</span>
                 <span v-else>stop viewing the raw stream for better performance</span>
diff --git a/photon-core/src/main/java/org/photonvision/vision/videoStream/SocketVideoStream.java b/photon-core/src/main/java/org/photonvision/vision/videoStream/SocketVideoStream.java
index cf2784c94..63c3809a1 100644
--- a/photon-core/src/main/java/org/photonvision/vision/videoStream/SocketVideoStream.java
+++ b/photon-core/src/main/java/org/photonvision/vision/videoStream/SocketVideoStream.java
@@ -18,7 +18,6 @@
 package org.photonvision.vision.videoStream;
 
 import java.nio.ByteBuffer;
-import java.util.Base64;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.Consumer;
@@ -77,16 +76,6 @@ public class SocketVideoStream implements Consumer<Frame> {
         oldSchoolServer.accept(frame);
     }
 
-    public String getJPEGBase64EncodedStr() {
-        String sendStr = null;
-        jpegBytesLock.lock();
-        if (jpegBytes != null) {
-            sendStr = Base64.getEncoder().encodeToString(jpegBytes.toArray());
-        }
-        jpegBytesLock.unlock();
-        return sendStr;
-    }
-
     public ByteBuffer getJPEGByteBuffer() {
         ByteBuffer sendStr = null;
         jpegBytesLock.lock();
diff --git a/photon-server/src/main/java/org/photonvision/server/CameraSocketHandler.java b/photon-server/src/main/java/org/photonvision/server/CameraSocketHandler.java
index 3dc3cb940..940ea050c 100644
--- a/photon-server/src/main/java/org/photonvision/server/CameraSocketHandler.java
+++ b/photon-server/src/main/java/org/photonvision/server/CameraSocketHandler.java
@@ -51,7 +51,7 @@ public class CameraSocketHandler {
 
     private CameraSocketHandler() {
         cameraBroadcastThread = new Thread(this::broadcastFramesTask);
-        cameraBroadcastThread.setPriority(2); // fairly low priority
+        cameraBroadcastThread.setPriority(Thread.MAX_PRIORITY - 3); // fairly high priority
         cameraBroadcastThread.start();
     }