Team2890/PhotonVision
4
0
Fork 0
mirror of https://github.com/PhotonVision/photonvision synced 2026-06-24 01:31:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Websocket Stream Stats & Robustness Improvements (#605)

Browse Source 
* wip support for a stats overlay

* WIP adding stream stats. But.... eeeh. Corporate.

* kbits over mbytes

* A ton more tweaks:

- Increased thread priority for streaming to reduce "stutter/slow" issues
- revised client side URL creation order to prevent the possibility of repeat-identical URL's
- Improved overlay to only be visible on mouseover, and fully centered in the screen

* wpiformat on js
This commit is contained in:
Chris Gerth
2022-12-01 13:42:21 -05:00
committed by GitHub
parent 4c004fc780
commit 678961e4f2
6 changed files with 147 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

171
photon-client/src/plugins/WebsocketVideoStream.js
View File

@@ -1,8 +1,62 @@
// Circular buffer storage. Externally-apparent 'length' increases indefinitely
// while any items with indexes below length-n will be forgotten (undefined
// will be returned if you try to get them, trying to set is an exception).
// n represents the initial length of the array, not a maximum
class StatsHistoryBuffer{
constructor (){
this.windowLen = 10;
this._array= new Array(this.windowLen);
this.headPtr = 0;
this.frameCount = 0;
this.bitAvgAccum = 0;
//calculated vals
this.bitRate_Mbps = 0;
this.framerate_fps = 0;
}
putAndPop(v){
this.headPtr++;
var idx = (this.headPtr)%this._array.length;
var poppedVal = this._array[idx];
this._array[idx] = v;
return poppedVal;
}
addSample(time, frameSize_bits, dispFrame_count) {
var oldVal = this.putAndPop([time, frameSize_bits, dispFrame_count]);
this.bitAvgAccum += frameSize_bits;
if(oldVal !=null){
var oldTime = oldVal[0];
var oldFrameSize = oldVal[1];
var oldFrameCount = oldVal[2];
var deltaTime_s = (time - oldTime);
this.bitAvgAccum -= oldFrameSize;
//bitrate - total bits transferred over the time period, divided by the period length
// converted to mbps
this.bitRate_Mbps = ( this.bitAvgAccum / deltaTime_s ) * (1.0/1048576.0);
//framerate - total frames displayed over the time period, divided by the period length
this.framerate_fps = (dispFrame_count - oldFrameCount) / deltaTime_s;
}
}
getText(){
return "Streaming @ " + this.framerate_fps.toFixed(1) + "FPS " + this.bitRate_Mbps.toFixed(1) + "Mbps";
}
}
export class WebsocketVideoStream{
constructor(drawDiv, streamPort, host) {
console.log("host " + host + " port " + streamPort)
@@ -11,39 +65,81 @@ export class WebsocketVideoStream{
this.streamPort = streamPort;
this.newStreamPortReq = null;
this.serverAddr = "ws://" + host + "/websocket_cameras";
this.dispNoStream();
this.ws_connect();
this.imgData = null;
this.imgDataTime = -1;
this.prevImgDataTime = -1;
this.imgObjURL = null;
this.frameRxCount = 0;
this.dispFrameCount = 0;
this.stats = null;
//Display state machine
this.DSM_DISCONNECTED = "DISCONNECTED";
this.DSM_WAIT_FOR_VALID_PORT = "WAIT_FOR_VALID_PORT";
this.DSM_SUBSCRIBE = "SUBSCRIBE";
this.DSM_WAIT_FOR_FIRST_FRAME = "WAIT_FOR_FIRST_FRAME";
this.DSM_SHOWING = "SHOWING";
this.DSM_RESTART_UNSUBSCRIBE = "UNSUBSCRIBE";
this.DSM_RESTART_WAIT = "WAIT_BEFORE_SUBSCRIBE";
//Set up div for stream stats info provided for users
this.statsTextDiv = this.image.parentNode.appendChild(document.createElement("div"));
//Centered over the image
this.statsTextDiv.style.position = "absolute";
this.statsTextDiv.style.left = "50%";
this.statsTextDiv.style.top = "50%";
this.statsTextDiv.style.transform = "translate(-50%, -50%)";
// Big enough for a line or two of text, with centered text
this.statsTextDiv.style.padding = "0.5em"
this.statsTextDiv.style.overflow = "hidden";
this.statsTextDiv.style.textAlign = "center";
this.statsTextDiv.style.verticalAlign = "middle";
// Styled to be black with grey text
this.statsTextDiv.style.backgroundColor = "black";
this.statsTextDiv.style.color = "#9E9E9E";
this.statsTextDiv.style.borderRadius = "3px";
//Default no text
this.statsTextDiv.innerHTML = "";
// Only show on mouseover, with opacity fade-in/fade-out
this.statsTextDiv.style.opacity = "0.0";
this.statsTextDiv.style.transition = "opacity 0.25s ease 0.25s";
this.statsTextDiv.style.transitionDelay = "opacity 0.5s";
this.image.addEventListener('mouseover', () => {this.statsTextDiv.style.opacity = "0.6";});
this.statsTextDiv.addEventListener('mouseover', () => {this.statsTextDiv.style.opacity = "0.6";});
this.image.addEventListener('mouseout', () => {this.statsTextDiv.style.opacity = "0.0";});
//Display state machine descriptions
this.DSM_DISCONNECTED = "Disconnected";
this.DSM_WAIT_FOR_VALID_PORT = "Waiting for valid port ID";
this.DSM_SUBSCRIBE = "Subscribing";
this.DSM_WAIT_FOR_FIRST_FRAME = "Waiting for frame data";
this.DSM_SHOWING = "Showing Frames";
this.DSM_RESTART_UNSUBSCRIBE = "Unsubscribing";
this.DSM_RESTART_WAIT = "Waiting before resubscribe";
this.dsm_cur_state = this.DSM_DISCONNECTED;
this.dsm_prev_state = this.DSM_DISCONNECTED;
this.dsm_restart_start_time = window.performance.now();
this.dispNoStream();
this.ws_connect();
requestAnimationFrame(()=>this.animationLoop());
}
dispImageData(){
//From https://stackoverflow.com/questions/67507616/set-image-src-from-image-blob/67507685#67507685
if(this.imgObjURL != null){
URL.revokeObjectURL(this.imgObjURL)
}
this.imgObjURL = URL.createObjectURL(this.imgData);
if(this.prevImgDataTime != this.imgDataTime){
//From https://stackoverflow.com/questions/67507616/set-image-src-from-image-blob/67507685#67507685
//Ensure uniqueness by making the new one before revoking the old one.
var oldURL = this.imgObjURL
this.imgObjURL = URL.createObjectURL(this.imgData);
if(oldURL != null){
URL.revokeObjectURL(oldURL)
}
//Update the image with the new mimetype and image
this.image.src = this.imgObjURL;
//Update the image with the new mimetype and image
this.image.src = this.imgObjURL;
this.dispFrameCount++;
this.prevImgDataTime = this.imgDataTime;
} // else no new image, don't update anything
}
dispNoStream() {
@@ -52,8 +148,8 @@ export class WebsocketVideoStream{
animationLoop(){
// Update time metrics
var now = window.performance.now();
var timeInState = now - this.dsm_restart_start_time;
var curTime_s = window.performance.now() / 1000.0;
var timeInState = curTime_s - this.dsm_restart_start_time;
// Save previous state
this.dsm_prev_state = this.dsm_cur_state;
@@ -93,7 +189,7 @@ export class WebsocketVideoStream{
}
break;
case this.DSM_SHOWING:
if((now - this.imgDataTime) > 2500){
if((curTime_s - this.imgDataTime) > 2.5){
//timeout, begin the restart sequence
this.dsm_cur_state = this.DSM_RESTART_UNSUBSCRIBE;
} else if (this.newStreamPortReq != null){
@@ -109,7 +205,7 @@ export class WebsocketVideoStream{
this.dsm_cur_state = this.DSM_RESTART_WAIT;
break;
case this.DSM_RESTART_WAIT:
if (timeInState > 250) {
if (timeInState > 0.25) {
//we've waited long enough, go to try to re-subscribe
this.dsm_cur_state = this.DSM_WAIT_FOR_VALID_PORT;
} else {
@@ -132,7 +228,12 @@ export class WebsocketVideoStream{
if(this.dsm_cur_state == this.DSM_SHOWING){
// Currently in SHOWING
// Show image and update status text
this.dispImageData();
this.statsTextDiv.innerHTML = this.stats.getText();
} else {
//Just show the state for debug
this.statsTextDiv.innerHTML = this.dsm_cur_state;
}
if(this.dsm_cur_state != this.DSM_SHOWING && this.dsm_prev_state == this.DSM_SHOWING ){
@@ -143,13 +244,13 @@ export class WebsocketVideoStream{
if(this.dsm_cur_state == this.DSM_RESTART_UNSUBSCRIBE){
// Currently in UNSUBSCRIBE, do the unsubscribe actions
this.stopStream();
this.dsm_restart_start_time = now;
this.dsm_restart_start_time = curTime_s;
}
if(this.dsm_cur_state == this.DSM_SUBSCRIBE){
// Currently in SUBSCRIBE, do the subscribe actions
this.startStream();
this.dsm_restart_start_time = now;
this.dsm_restart_start_time = curTime_s;
}
if(this.dsm_cur_state == this.DSM_WAIT_FOR_VALID_PORT){
@@ -160,6 +261,8 @@ export class WebsocketVideoStream{
}
}
//Update status text
requestAnimationFrame(()=>this.animationLoop());
}
@@ -184,6 +287,11 @@ export class WebsocketVideoStream{
// Set the flag allowing general server communication
this.serverConnectionActive = true;
console.log("Camera Websockets Connected!");
// New websocket connection, reset stats
this.frameRxCount = 0;
this.dispFrameCount = 0;
this.stats = new StatsHistoryBuffer();
}
ws_onClose(e) {
@@ -206,18 +314,25 @@ export class WebsocketVideoStream{
}
ws_onMessage(e){
console.log("Got message from " + this.serverAddr)
//console.log("Got message from " + this.serverAddr)
var msgTime_s = window.performance.now() / 1000.0;
if(typeof e.data === 'string'){
//string data from host
//TODO - anything to receive info here? Maybe "available streams?"
} else {
if(e.data.size > 0){
//binary data - a frame
//binary data - a frame!
//Save frame data for display in the next animation thread
this.imgData = e.data;
this.imgDataTime = window.performance.now();
this.imgDataTime = msgTime_s;
//Count the incoming frame
this.frameRxCount++;
//keep the stats up to date
this.stats.addSample(msgTime_s,this.imgData.size * 8,this.dispFrameCount);
} else {
//TODO - server is sending empty frames?
console.log("WS Stream Error: Server sent empty frame!");
}
}