From 1971744589e86eb69f46dc0a8d8ecb14a98e48e0 Mon Sep 17 00:00:00 2001
From: Chris Gerth <gerth2@users.noreply.github.com>
Date: Sun, 25 Dec 2022 08:55:12 -0600
Subject: [PATCH] Revert to mjpeg (#645)

* Reverted to front end using MJPEG streams. Added FPS limiting to the stream.

* formatting

* fixup - got handlers getting called on error to reload

* revised architecture to let a click open a new tab

Co-authored-by: Matt <matthew.morley.ca@gmail.com>
---
 .../src/components/common/cv-image.vue        | 93 +++++++++++--------
 .../vision/videoStream/SocketVideoStream.java | 18 +++-
 2 files changed, 68 insertions(+), 43 deletions(-)

diff --git a/photon-client/src/components/common/cv-image.vue b/photon-client/src/components/common/cv-image.vue
index b13ece41f..32c466143 100644
--- a/photon-client/src/components/common/cv-image.vue
+++ b/photon-client/src/components/common/cv-image.vue
@@ -5,20 +5,19 @@
     :style="styleObject"
     :src="src"
     alt=""
-    @click="e => {this.openThinclientStream(e)}"
-  >
+    @click="clickHandler"
+    @error="loadErrHandler"
+  />
 </template>
 
-
 <script>
     export default {
         name: "CvImage",
         // eslint-disable-next-line vue/require-prop-types
-        props: ['idx', 'scale', 'maxHeight', 'maxHeightMd', 'maxHeightLg', 'maxHeightXl', 'colorPicking', 'id', 'disconnected'],
+        props: ['address', 'scale', 'maxHeight', 'maxHeightMd', 'maxHeightLg', 'maxHeightXl', 'colorPicking', 'id', 'disconnected'],
         data() {
             return {
                 seed: 1.0,
-                src: ""
             }
         },
         computed: {
@@ -28,13 +27,14 @@
                       "border-radius": "3px",
                       "display": "block",
                       "object-fit": "contain",
+                      "background-size:": "contain",
                       "object-position": "50% 50%",
                       "max-width": "100%",
                       "margin-left": "auto",
                       "margin-right": "auto",
                       "max-height": this.maxHeight,
                       height: `${this.scale}%`,
-                      cursor: (this.colorPicking ? `url(${require("../../assets/eyedropper.svg")}),` : "") + "default",
+                      cursor: (this.colorPicking ? `url(${require("../../assets/eyedropper.svg")}),` : "pointer") + "default",
                     };
 
                     if (this.$vuetify.breakpoint.xl) {
@@ -48,47 +48,62 @@
                     return ret;
                 }
             },
-            port: {
+            src: {
               get() {
-                if(this.idx == 0){
-                  return this.$store.state.cameraSettings[this.$store.state.currentCameraIndex].inputStreamPort;
+                var port = this.getCurPort();
+                if(port <= 0){
+                  //Invalid port, keep it spinny
+                  return require("../../assets/loading.gif");
                 } else {
-                  return this.$store.state.cameraSettings[this.$store.state.currentCameraIndex].outputStreamPort;
+                  //Valid port, connect
+                  return this.getSrcURLFromPort(port);
                 }
-              }
-            }
-        },
-        watch : {
-          port(newPort, oldPort){
-            newPort;
-            oldPort;
-            this.reload();
-          },
-          disconnected(newVal, oldVal){
-            oldVal;
-            if(newVal){
-              this.wsStream.setPort(0);
-            } else {
-              this.wsStream.setPort(this.port);
-            }
-          }
+              },
+            },
         },
         mounted() {
-          var wsvs = require('../../plugins/WebsocketVideoStream');
-          this.wsStream = new wsvs.WebsocketVideoStream(this.id, this.port, this.$address);
-        },
-        unmounted() {
-          this.wsStream.setPort(0);
+            this.reload(); // Force reload image on creation
         },
         methods: {
-            reload() {
-              console.log("Reloading " + this.id + " with port " + String(this.port));
-              this.wsStream.setPort(this.port);
+            getCurPort(){
+              var port = -1;
+              if(this.disconnected){
+                //Disconnected, port is unknown.
+                port = -1;
+              } else {
+                //Connected - get the port
+                if(this.id == 'raw-stream'){
+                  port = this.$store.state.cameraSettings[this.$store.state.currentCameraIndex].inputStreamPort
+                } else {
+                  port = this.$store.state.cameraSettings[this.$store.state.currentCameraIndex].outputStreamPort
+                }
+              }
+              return port;
             },
-            openThinclientStream(e){
-              e;
-              var URL = "/thinclient.html?port=" + String(this.port) + "&host=" + window.location.hostname;
-              window.open(URL, '_blank');
+            getSrcURLFromPort(port){
+              return "http://" + location.hostname + ":" + port + "/stream.mjpg" + "?" + this.seed;
+            },
+            loadErrHandler(event) {
+                console.log(event);
+                console.log("Error loading image, attempting to do it again...");
+                this.reload();
+            },
+            clickHandler(event) {
+              if(this.colorPicking){
+                this.$emit('click', event);
+              } else {
+                var port = this.getCurPort();
+                if(port <= 0){
+                  console.log("No valid port, ignoring click.");
+                } else {
+                  //Valid port, connect
+                  window.open(this.getSrcURLFromPort(port), '_blank');
+                }
+              }
+
+            },
+            reload() {
+                this.seed = new Date().getTime();
             }
         },
     }
diff --git a/photon-core/src/main/java/org/photonvision/vision/videoStream/SocketVideoStream.java b/photon-core/src/main/java/org/photonvision/vision/videoStream/SocketVideoStream.java
index 63c3809a1..de72a8cf3 100644
--- a/photon-core/src/main/java/org/photonvision/vision/videoStream/SocketVideoStream.java
+++ b/photon-core/src/main/java/org/photonvision/vision/videoStream/SocketVideoStream.java
@@ -24,6 +24,7 @@ import java.util.function.Consumer;
 import org.opencv.core.MatOfByte;
 import org.opencv.core.MatOfInt;
 import org.opencv.imgcodecs.Imgcodecs;
+import org.photonvision.common.util.math.MathUtils;
 import org.photonvision.vision.frame.Frame;
 import org.photonvision.vision.frame.consumer.MJPGFrameConsumer;
 
@@ -39,11 +40,14 @@ public class SocketVideoStream implements Consumer<Frame> {
 
     // Synclock around manipulating the jpeg bytes from multiple threads
     Lock jpegBytesLock = new ReentrantLock();
-
-    MJPGFrameConsumer oldSchoolServer;
-
     private int userCount = 0;
 
+    // FPS-limited MJPEG sender
+    private final double FPS_MAX = 30.0;
+    private final long minFramePeriodNanos = Math.round(1000000000.0 / FPS_MAX);
+    private long nextFrameSendTime = MathUtils.wpiNanoTime() + minFramePeriodNanos;
+    MJPGFrameConsumer oldSchoolServer;
+
     public SocketVideoStream(int portID) {
         this.portID = portID;
         oldSchoolServer =
@@ -73,7 +77,13 @@ public class SocketVideoStream implements Consumer<Frame> {
                 }
             }
         }
-        oldSchoolServer.accept(frame);
+
+        // Send the frame in an FPS-limited fashion
+        var now = MathUtils.wpiNanoTime();
+        if (now > nextFrameSendTime) {
+            oldSchoolServer.accept(frame);
+            nextFrameSendTime = now + minFramePeriodNanos;
+        }
     }
 
     public ByteBuffer getJPEGByteBuffer() {