From 6120325121b1eefb959612ff6a2bdc671f632e26 Mon Sep 17 00:00:00 2001 From: ori Date: Wed, 14 Aug 2019 09:52:43 -0700 Subject: [PATCH 01/15] all camera sever calls are from self --- backend/app/handlers/VisionHandler.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/backend/app/handlers/VisionHandler.py b/backend/app/handlers/VisionHandler.py index 2d5a1f6af..48eac6d05 100644 --- a/backend/app/handlers/VisionHandler.py +++ b/backend/app/handlers/VisionHandler.py @@ -19,6 +19,8 @@ from ..handlers.SocketHandler import send_all_async class VisionHandler(metaclass=Singleton): def __init__(self): self.kernel = numpy.ones((5, 5), numpy.uint8) + self.cs = CameraServer.getInstance() + self.settings_manager = SettingsManager() def _hsv_threshold(self, hue: list, saturation: list, value: list, img: numpy.ndarray, is_erode: bool, is_dilate: bool): @@ -248,14 +250,14 @@ class VisionHandler(metaclass=Singleton): NetworkTables.startClientTeam(team=SettingsManager.general_settings.get("team_number", 1577)) # NetworkTables.initialize("localhost") - cs = CameraServer.getInstance() + port = 5550 for cam_name in SettingsManager().usb_cameras: - threading.Thread(target=self.thread_proc, args=(cs, cam_name, port)).start() + threading.Thread(target=self.thread_proc, args=(cam_name, port)).start() port += 1 - def thread_proc(self, cs, cam_name, port=5557): + def thread_proc(self, cam_name, port=5557): asyncio.set_event_loop(asyncio.new_event_loop()) SettingsManager.cams_curr_pipeline[cam_name] = "pipeline0" pipeline = SettingsManager().cams[cam_name]["pipelines"][SettingsManager.cams_curr_pipeline[cam_name]] @@ -289,7 +291,7 @@ class VisionHandler(metaclass=Singleton): table.addEntryListenerEx(mode_listener, key="Driver_Mode", flags=networktables.NetworkTablesInstance.NotifyFlags.UPDATE) #gettings video from curent camera - cv_sink = cs.getVideo(camera=SettingsManager.usb_cameras[cam_name]) + cv_sink = self.cs.getVideo(camera=SettingsManager.usb_cameras[cam_name]) width = SettingsManager().cams[cam_name]["video_mode"]["width"] height = SettingsManager().cams[cam_name]["video_mode"]["height"] @@ -297,9 +299,9 @@ class VisionHandler(metaclass=Singleton): image = numpy.zeros(shape=(width, height, 3), dtype=numpy.uint8) #setting up a video server for camera - cv_publish = cs.putVideo(name=cam_name, width=width, height=height) + cv_publish = self.cs.putVideo(name=cam_name, width=width, height=height) # saving camera port in cam name dict for usage in client - SettingsManager().cams_port[cam_name] = cs._sinks['serve_'+cam_name].getPort() + SettingsManager().cams_port[cam_name] = self.cs._sinks['serve_'+cam_name].getPort() #setting up a zmq connection to the opencv subprocess context = zmq.Context() From fec97959d82efdf3aefaf1af94dcad25dbd712c9 Mon Sep 17 00:00:00 2001 From: ori Date: Wed, 14 Aug 2019 10:15:44 -0700 Subject: [PATCH 02/15] all settings manager are now in self --- backend/app/handlers/VisionHandler.py | 47 +++++++++++++-------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/backend/app/handlers/VisionHandler.py b/backend/app/handlers/VisionHandler.py index 48eac6d05..a1977f5fe 100644 --- a/backend/app/handlers/VisionHandler.py +++ b/backend/app/handlers/VisionHandler.py @@ -247,35 +247,34 @@ class VisionHandler(metaclass=Singleton): return yaw def run(self): - NetworkTables.startClientTeam(team=SettingsManager.general_settings.get("team_number", 1577)) + NetworkTables.startClientTeam(team=self.settings_manager.general_settings.get("team_number", 1577)) # NetworkTables.initialize("localhost") - port = 5550 - for cam_name in SettingsManager().usb_cameras: + for cam_name in self.settings_manager.usb_cameras: threading.Thread(target=self.thread_proc, args=(cam_name, port)).start() port += 1 def thread_proc(self, cam_name, port=5557): asyncio.set_event_loop(asyncio.new_event_loop()) - SettingsManager.cams_curr_pipeline[cam_name] = "pipeline0" - pipeline = SettingsManager().cams[cam_name]["pipelines"][SettingsManager.cams_curr_pipeline[cam_name]] - FOV = SettingsManager().cams[cam_name]["FOV"] + self.settings_manager.cams_curr_pipeline[cam_name] = "pipeline0" + pipeline = self.settings_manager.cams[cam_name]["pipelines"][self.settings_manager.cams_curr_pipeline[cam_name]] + FOV = self.settings_manager.cams[cam_name]["FOV"] def change_camera_values(pipline): - SettingsManager.usb_cameras[cam_name].setBrightness(pipeline['brightness']) - SettingsManager.usb_cameras[cam_name].setExposureManual(pipeline['exposure']) - SettingsManager.usb_cameras[cam_name].setWhiteBalanceAuto() + self.settings_manager.usb_cameras[cam_name].setBrightness(pipeline['brightness']) + self.settings_manager.usb_cameras[cam_name].setExposureManual(pipeline['exposure']) + self.settings_manager.usb_cameras[cam_name].setWhiteBalanceAuto() def pipeline_listener(table, key, value, is_new): asyncio.set_event_loop(asyncio.new_event_loop()) - SettingsManager.cams_curr_pipeline[cam_name] = value + self.settings_manager.cams_curr_pipeline[cam_name] = value change_camera_values(pipeline) - if cam_name == SettingsManager().general_settings['curr_camera']: - SettingsManager().general_settings['curr_pipeline'] = value - update_settings = SettingsManager().get_curr_pipeline() - update_settings['curr_pipeline'] = SettingsManager().general_settings["curr_pipeline"] + if cam_name == self.settings_manager.general_settings['curr_camera']: + self.settings_manager.general_settings['curr_pipeline'] = value + update_settings = self.settings_manager.get_curr_pipeline() + update_settings['curr_pipeline'] = self.settings_manager.general_settings["curr_pipeline"] send_all_async(update_settings) def mode_listener(table, key, value, is_new): @@ -285,23 +284,23 @@ class VisionHandler(metaclass=Singleton): }) table = NetworkTables.getTable("/Chameleon-Vision/" + cam_name) - table.putString('Pipeline', SettingsManager.cams_curr_pipeline[cam_name]) + table.putString('Pipeline', self.settings_manager.cams_curr_pipeline[cam_name]) table.addEntryListenerEx(pipeline_listener, key="Pipeline", flags=networktables.NetworkTablesInstance.NotifyFlags.UPDATE) table.addEntryListenerEx(mode_listener, key="Driver_Mode", flags=networktables.NetworkTablesInstance.NotifyFlags.UPDATE) #gettings video from curent camera - cv_sink = self.cs.getVideo(camera=SettingsManager.usb_cameras[cam_name]) + cv_sink = self.cs.getVideo(camera=self.settings_manager.usb_cameras[cam_name]) - width = SettingsManager().cams[cam_name]["video_mode"]["width"] - height = SettingsManager().cams[cam_name]["video_mode"]["height"] + width = self.settings_manager.cams[cam_name]["video_mode"]["width"] + height = self.settings_manager.cams[cam_name]["video_mode"]["height"] image = numpy.zeros(shape=(width, height, 3), dtype=numpy.uint8) #setting up a video server for camera cv_publish = self.cs.putVideo(name=cam_name, width=width, height=height) # saving camera port in cam name dict for usage in client - SettingsManager().cams_port[cam_name] = self.cs._sinks['serve_'+cam_name].getPort() + self.settings_manager.cams_port[cam_name] = self.cs._sinks['serve_'+cam_name].getPort() #setting up a zmq connection to the opencv subprocess context = zmq.Context() @@ -315,7 +314,7 @@ class VisionHandler(metaclass=Singleton): change_camera_values(pipeline) while True: - pipeline = SettingsManager().cams[cam_name]["pipelines"][SettingsManager.cams_curr_pipeline[cam_name]] + pipeline = self.settings_manager.cams[cam_name]["pipelines"][self.settings_manager.cams_curr_pipeline[cam_name]] _, image = cv_sink.grabFrame(image) socket.send_json(dict( pipeline=pipeline @@ -335,7 +334,7 @@ class VisionHandler(metaclass=Singleton): table.putNumber('yaw', nt_data['yaw']) #if the selected camera in ui is this cam send the point to the ui - if SettingsManager().general_settings['curr_camera'] == cam_name: + if self.settings_manager.general_settings['curr_camera'] == cam_name: try: if nt_data['raw_point'] is not None: send_all_async({ @@ -349,7 +348,7 @@ class VisionHandler(metaclass=Singleton): except Exception as e: print(e) #send the image to the camera server - + # print(nt_data['fps']) cv_publish.putFrame(p_image) def camera_process(self, cam_name, port, FOV): @@ -357,8 +356,8 @@ class VisionHandler(metaclass=Singleton): diagonalView = math.radians(FOV) #needs to be implemented in client - width = SettingsManager().cams[cam_name]["video_mode"]["width"] - height = SettingsManager().cams[cam_name]["video_mode"]["height"] + width = self.settings_manager.cams[cam_name]["video_mode"]["width"] + height = self.settings_manager.cams[cam_name]["video_mode"]["height"] centerX = (width / 2) - .5 centerY = (height / 2) - .5 cam_area = width * height From 4b279f3a0f3ef5d7bbd3b1382fdedb4a50ff2405 Mon Sep 17 00:00:00 2001 From: ori Date: Wed, 14 Aug 2019 10:25:47 -0700 Subject: [PATCH 03/15] threded camera fps is at about 50 --- backend/app/handlers/VisionHandler.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/backend/app/handlers/VisionHandler.py b/backend/app/handlers/VisionHandler.py index a1977f5fe..aebdea2a0 100644 --- a/backend/app/handlers/VisionHandler.py +++ b/backend/app/handlers/VisionHandler.py @@ -295,8 +295,6 @@ class VisionHandler(metaclass=Singleton): width = self.settings_manager.cams[cam_name]["video_mode"]["width"] height = self.settings_manager.cams[cam_name]["video_mode"]["height"] - image = numpy.zeros(shape=(width, height, 3), dtype=numpy.uint8) - #setting up a video server for camera cv_publish = self.cs.putVideo(name=cam_name, width=width, height=height) # saving camera port in cam name dict for usage in client @@ -313,9 +311,15 @@ class VisionHandler(metaclass=Singleton): change_camera_values(pipeline) + def _thread(): + global image + image = numpy.zeros(shape=(width, height, 3), dtype=numpy.uint8) + while True: + _, image = cv_sink.grabFrame(image) + + threading.Thread(target=_thread).start() while True: pipeline = self.settings_manager.cams[cam_name]["pipelines"][self.settings_manager.cams_curr_pipeline[cam_name]] - _, image = cv_sink.grabFrame(image) socket.send_json(dict( pipeline=pipeline ), zmq.SNDMORE) @@ -362,7 +366,7 @@ class VisionHandler(metaclass=Singleton): centerY = (height / 2) - .5 cam_area = width * height - aspect_fraction = Fraction(width,height) + aspect_fraction = Fraction(width, height) horizontal_ratio = aspect_fraction.numerator vertical_ratio = aspect_fraction.denominator From a3138d8e4f7fb8853737ae85caf75fb2f53c5453 Mon Sep 17 00:00:00 2001 From: ori Date: Wed, 14 Aug 2019 12:59:30 -0700 Subject: [PATCH 04/15] added camera server publish thread --- backend/app/classes/Exceptions.py | 1 - backend/app/handlers/SocketHandler.py | 1 + backend/app/handlers/VisionHandler.py | 25 +++++++++++++------ .../settings/cams/USB Camera-B4.09.24.1.json | 2 +- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/backend/app/classes/Exceptions.py b/backend/app/classes/Exceptions.py index 59a8363e3..67493d2aa 100644 --- a/backend/app/classes/Exceptions.py +++ b/backend/app/classes/Exceptions.py @@ -1,4 +1,3 @@ - class PipelineAlreadyExistsException(Exception): def __init__(self, pipe_name): diff --git a/backend/app/handlers/SocketHandler.py b/backend/app/handlers/SocketHandler.py index 35c5750ee..75096dfd6 100644 --- a/backend/app/handlers/SocketHandler.py +++ b/backend/app/handlers/SocketHandler.py @@ -18,6 +18,7 @@ def send_all_async(message): class ChameleonWebSocket(tornado.websocket.WebSocketHandler): + actions = {} set_this_camera_settings = ["exposure", "brightness"] diff --git a/backend/app/handlers/VisionHandler.py b/backend/app/handlers/VisionHandler.py index aebdea2a0..881f03bb3 100644 --- a/backend/app/handlers/VisionHandler.py +++ b/backend/app/handlers/VisionHandler.py @@ -257,6 +257,10 @@ class VisionHandler(metaclass=Singleton): port += 1 def thread_proc(self, cam_name, port=5557): + global p_image + global nt_data + global table + nt_data = {'valid': False} asyncio.set_event_loop(asyncio.new_event_loop()) self.settings_manager.cams_curr_pipeline[cam_name] = "pipeline0" pipeline = self.settings_manager.cams[cam_name]["pipelines"][self.settings_manager.cams_curr_pipeline[cam_name]] @@ -311,13 +315,25 @@ class VisionHandler(metaclass=Singleton): change_camera_values(pipeline) - def _thread(): + def _image_thread(): global image + global p_image image = numpy.zeros(shape=(width, height, 3), dtype=numpy.uint8) + p_image = image while True: _, image = cv_sink.grabFrame(image) - threading.Thread(target=_thread).start() + def _publish_thread(): + while True: + try: + cv_publish.putFrame(p_image) + + except: + pass + + threading.Thread(target=_image_thread).start() + threading.Thread(target=_publish_thread).start() + while True: pipeline = self.settings_manager.cams[cam_name]["pipelines"][self.settings_manager.cams_curr_pipeline[cam_name]] socket.send_json(dict( @@ -329,9 +345,6 @@ class VisionHandler(metaclass=Singleton): nt_data = socket.recv_json() table.putBoolean('valid', nt_data['valid']) # check if point is valid - - # print(nt_data['fps']) - if nt_data['valid']: #send the point using network tables table.putNumber('pitch', nt_data['pitch']) @@ -352,8 +365,6 @@ class VisionHandler(metaclass=Singleton): except Exception as e: print(e) #send the image to the camera server - # print(nt_data['fps']) - cv_publish.putFrame(p_image) def camera_process(self, cam_name, port, FOV): from fractions import Fraction diff --git a/backend/settings/cams/USB Camera-B4.09.24.1.json b/backend/settings/cams/USB Camera-B4.09.24.1.json index 2dd7aa24f..91187d8b0 100644 --- a/backend/settings/cams/USB Camera-B4.09.24.1.json +++ b/backend/settings/cams/USB Camera-B4.09.24.1.json @@ -1 +1 @@ -{"pipelines": {"pipeline0": {"exposure": 50, "brightness": 15, "orientation": "Normal", "hue": [0, 100], "saturation": [0, 100], "value": [0, 100], "erode": false, "dilate": false, "area": [0, 100], "ratio": [0, 20], "extent": [0, 100], "is_binary": 0, "sort_mode": "Largest", "target_group": "Single", "target_intersection": "Up", "M": 1, "B": 0}, "pipeline1": {"exposure": 50, "brightness": 50, "orientation": "Normal", "hue": [0, 100], "saturation": [0, 100], "value": [0, 100], "erode": false, "dilate": false, "area": [0, 100], "ratio": [0, 20], "extent": [0, 100], "is_binary": 0, "sort_mode": "Largest", "target_group": "Single", "target_intersection": "Up", "M": 1, "B": 0}, "pipeline2": {"exposure": 50, "brightness": 50, "orientation": "Normal", "hue": [0, 100], "saturation": [0, 100], "value": [0, 100], "erode": false, "dilate": false, "area": [0, 100], "ratio": [0, 20], "extent": [0, 100], "is_binary": "Normal", "sort_mode": "Largest", "target_group": "Single", "target_intersection": "Up", "M": 1, "B": 0}, "pipeline3": {"exposure": 50, "brightness": 50, "orientation": "Normal", "hue": [0, 100], "saturation": [0, 100], "value": [0, 100], "erode": false, "dilate": false, "area": [0, 100], "ratio": [0, 20], "extent": [0, 100], "is_binary": "Normal", "sort_mode": "Largest", "target_group": "Single", "target_intersection": "Up", "M": 1, "B": 0}, "pipeline4": {"exposure": 50, "brightness": 50, "orientation": "Normal", "hue": [0, 100], "saturation": [0, 100], "value": [0, 100], "erode": false, "dilate": false, "area": [0, 100], "ratio": [0, 20], "extent": [0, 100], "is_binary": "Normal", "sort_mode": "Largest", "target_group": "Single", "target_intersection": "Up", "M": 1, "B": 0}, "pipeline5": {"exposure": 50, "brightness": 50, "orientation": "Normal", "hue": [0, 100], "saturation": [0, 100], "value": [0, 100], "erode": false, "dilate": false, "area": [0, 100], "ratio": [0, 20], "extent": [0, 100], "is_binary": "Normal", "sort_mode": "Largest", "target_group": "Single", "target_intersection": "Up", "M": 1, "B": 0}, "pipeline6": {"exposure": 50, "brightness": 50, "orientation": "Normal", "hue": [0, 100], "saturation": [0, 100], "value": [0, 100], "erode": false, "dilate": false, "area": [0, 100], "ratio": [0, 20], "extent": [0, 100], "is_binary": "Normal", "sort_mode": "Largest", "target_group": "Single", "target_intersection": "Up", "M": 1, "B": 0}, "pipeline7": {"exposure": 50, "brightness": 50, "orientation": "Normal", "hue": [0, 100], "saturation": [0, 100], "value": [0, 100], "erode": false, "dilate": false, "area": [0, 100], "ratio": [0, 20], "extent": [0, 100], "is_binary": "Normal", "sort_mode": "Largest", "target_group": "Single", "target_intersection": "Up", "M": 1, "B": 0}, "pipeline8": {"exposure": 50, "brightness": 50, "orientation": "Normal", "hue": [0, 100], "saturation": [0, 100], "value": [0, 100], "erode": false, "dilate": false, "area": [0, 100], "ratio": [0, 20], "extent": [0, 100], "is_binary": "Normal", "sort_mode": "Largest", "target_group": "Single", "target_intersection": "Up", "M": 1, "B": 0}, "pipeline9": {"exposure": 50, "brightness": 50, "orientation": "Normal", "hue": [0, 100], "saturation": [0, 100], "value": [0, 100], "erode": false, "dilate": false, "area": [0, 100], "ratio": [0, 20], "extent": [0, 100], "is_binary": 0, "sort_mode": "Largest", "target_group": "Single", "target_intersection": "Up", "M": 1, "B": 0}}, "path": "/dev/v4l/by-path/pci-0000:02:03.0-usb-0:1:1.0-video-index0", "video_mode": {"fps": 187, "width": 320, "height": 240, "pixel_format": "kYUYV"}, "resolution": 0, "FOV": 60.8} \ No newline at end of file +{"pipelines": {"pipeline0": {"exposure": 50, "brightness": 15, "orientation": "Normal", "hue": [0, 100], "saturation": [0, 100], "value": [0, 100], "erode": false, "dilate": true, "area": [0, 100], "ratio": [0, 20], "extent": [0, 100], "is_binary": 0, "sort_mode": "Largest", "target_group": "Single", "target_intersection": "Up", "M": 1, "B": 0}, "pipeline1": {"exposure": 50, "brightness": 50, "orientation": "Normal", "hue": [0, 100], "saturation": [0, 100], "value": [0, 100], "erode": false, "dilate": false, "area": [0, 100], "ratio": [0, 20], "extent": [0, 100], "is_binary": 0, "sort_mode": "Largest", "target_group": "Single", "target_intersection": "Up", "M": 1, "B": 0}, "pipeline2": {"exposure": 50, "brightness": 50, "orientation": "Normal", "hue": [0, 100], "saturation": [0, 100], "value": [0, 100], "erode": false, "dilate": false, "area": [0, 100], "ratio": [0, 20], "extent": [0, 100], "is_binary": "Normal", "sort_mode": "Largest", "target_group": "Single", "target_intersection": "Up", "M": 1, "B": 0}, "pipeline3": {"exposure": 50, "brightness": 50, "orientation": "Normal", "hue": [0, 100], "saturation": [0, 100], "value": [0, 100], "erode": false, "dilate": false, "area": [0, 100], "ratio": [0, 20], "extent": [0, 100], "is_binary": "Normal", "sort_mode": "Largest", "target_group": "Single", "target_intersection": "Up", "M": 1, "B": 0}, "pipeline4": {"exposure": 50, "brightness": 50, "orientation": "Normal", "hue": [0, 100], "saturation": [0, 100], "value": [0, 100], "erode": false, "dilate": false, "area": [0, 100], "ratio": [0, 20], "extent": [0, 100], "is_binary": "Normal", "sort_mode": "Largest", "target_group": "Single", "target_intersection": "Up", "M": 1, "B": 0}, "pipeline5": {"exposure": 50, "brightness": 50, "orientation": "Normal", "hue": [0, 100], "saturation": [0, 100], "value": [0, 100], "erode": false, "dilate": false, "area": [0, 100], "ratio": [0, 20], "extent": [0, 100], "is_binary": "Normal", "sort_mode": "Largest", "target_group": "Single", "target_intersection": "Up", "M": 1, "B": 0}, "pipeline6": {"exposure": 50, "brightness": 50, "orientation": "Normal", "hue": [0, 100], "saturation": [0, 100], "value": [0, 100], "erode": false, "dilate": false, "area": [0, 100], "ratio": [0, 20], "extent": [0, 100], "is_binary": "Normal", "sort_mode": "Largest", "target_group": "Single", "target_intersection": "Up", "M": 1, "B": 0}, "pipeline7": {"exposure": 50, "brightness": 50, "orientation": "Normal", "hue": [0, 100], "saturation": [0, 100], "value": [0, 100], "erode": false, "dilate": false, "area": [0, 100], "ratio": [0, 20], "extent": [0, 100], "is_binary": "Normal", "sort_mode": "Largest", "target_group": "Single", "target_intersection": "Up", "M": 1, "B": 0}, "pipeline8": {"exposure": 50, "brightness": 50, "orientation": "Normal", "hue": [0, 100], "saturation": [0, 100], "value": [0, 100], "erode": false, "dilate": false, "area": [0, 100], "ratio": [0, 20], "extent": [0, 100], "is_binary": "Normal", "sort_mode": "Largest", "target_group": "Single", "target_intersection": "Up", "M": 1, "B": 0}, "pipeline9": {"exposure": 50, "brightness": 50, "orientation": "Normal", "hue": [0, 100], "saturation": [0, 100], "value": [0, 100], "erode": false, "dilate": false, "area": [0, 100], "ratio": [0, 20], "extent": [0, 100], "is_binary": 0, "sort_mode": "Largest", "target_group": "Single", "target_intersection": "Up", "M": 1, "B": 0}}, "path": "/dev/v4l/by-path/pci-0000:02:03.0-usb-0:1:1.0-video-index0", "video_mode": {"fps": 187, "width": 320, "height": 240, "pixel_format": "kYUYV"}, "resolution": 0, "FOV": 60.8} \ No newline at end of file From b7116baf3f16b77b0f2f2022e28f03b9298bd3cf Mon Sep 17 00:00:00 2001 From: ori Date: Wed, 14 Aug 2019 14:23:32 -0700 Subject: [PATCH 05/15] 100 fps mark --- backend/app/handlers/VisionHandler.py | 44 +++++++++++++-------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/backend/app/handlers/VisionHandler.py b/backend/app/handlers/VisionHandler.py index 881f03bb3..50127e068 100644 --- a/backend/app/handlers/VisionHandler.py +++ b/backend/app/handlers/VisionHandler.py @@ -261,7 +261,7 @@ class VisionHandler(metaclass=Singleton): global nt_data global table nt_data = {'valid': False} - asyncio.set_event_loop(asyncio.new_event_loop()) + # asyncio.set_event_loop(asyncio.new_event_loop()) self.settings_manager.cams_curr_pipeline[cam_name] = "pipeline0" pipeline = self.settings_manager.cams[cam_name]["pipelines"][self.settings_manager.cams_curr_pipeline[cam_name]] FOV = self.settings_manager.cams[cam_name]["FOV"] @@ -324,10 +324,29 @@ class VisionHandler(metaclass=Singleton): _, image = cv_sink.grabFrame(image) def _publish_thread(): + # asyncio.set_event_loop(asyncio.new_event_loop()) while True: try: cv_publish.putFrame(p_image) - + table.putBoolean('valid', nt_data['valid']) + # check if point is valid + if nt_data['valid']: + #send the point using network tables + table.putNumber('pitch', nt_data['pitch']) + table.putNumber('yaw', nt_data['yaw']) + #if the selected camera in ui is this cam send the point to the ui + if self.settings_manager.general_settings['curr_camera'] == cam_name: + try: + send_all_async({ + 'raw_point': nt_data['raw_point'], + 'point': { + 'pitch': nt_data['pitch'], + 'yaw': nt_data['yaw'], + 'fps': nt_data['fps'] + } + }) + except: + pass except: pass @@ -343,27 +362,6 @@ class VisionHandler(metaclass=Singleton): socket.send_pyobj(image) p_image = socket.recv_pyobj() nt_data = socket.recv_json() - table.putBoolean('valid', nt_data['valid']) - # check if point is valid - if nt_data['valid']: - #send the point using network tables - table.putNumber('pitch', nt_data['pitch']) - table.putNumber('yaw', nt_data['yaw']) - #if the selected camera in ui is this cam send the point to the ui - - if self.settings_manager.general_settings['curr_camera'] == cam_name: - try: - if nt_data['raw_point'] is not None: - send_all_async({ - 'raw_point': nt_data['raw_point'], - 'point': { - 'pitch': nt_data['pitch'], - 'yaw': nt_data['yaw'], - 'fps': nt_data['fps'] - } - }) - except Exception as e: - print(e) #send the image to the camera server def camera_process(self, cam_name, port, FOV): From bfb42f31fb645fdf1286276e8c65da89d1a15a45 Mon Sep 17 00:00:00 2001 From: ori agranat Date: Thu, 15 Aug 2019 08:08:03 +0300 Subject: [PATCH 06/15] updated git ignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index b006761b8..1dc29ec9b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ Python/app/__pycache__/ Python/app/handlers/__pycache__/ \.vscode/ + +backend/settings/ From 771958fb71c68b13f15784fde95d1b5aba818958 Mon Sep 17 00:00:00 2001 From: ori agranat Date: Thu, 15 Aug 2019 05:18:20 +0000 Subject: [PATCH 07/15] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a779f1c95..1ccde2e98 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ These instructions will get you a copy of the project up and running on your loc so in order to run this project we will need to install python in order to run the backend and node.js with vue.js in order to run the fronted #### backend -- python 3.6 and above +- python 3.7 and above - opencv 3.4.5 - tornado web framework - robotpy-cscore From e068fe3cb249b53ff8b04e776618ac9dc003f4a2 Mon Sep 17 00:00:00 2001 From: ori Date: Thu, 15 Aug 2019 09:07:52 -0700 Subject: [PATCH 08/15] added time stamp and removed websocket --- backend/app/handlers/SocketHandler.py | 5 +++-- backend/app/handlers/VisionHandler.py | 21 ++++++--------------- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/backend/app/handlers/SocketHandler.py b/backend/app/handlers/SocketHandler.py index 75096dfd6..228cbc30d 100644 --- a/backend/app/handlers/SocketHandler.py +++ b/backend/app/handlers/SocketHandler.py @@ -6,7 +6,7 @@ from ..classes.Exceptions import NoCameraConnectedException from ..classes.SettingsManager import SettingsManager -web_socket_clients = [] +web_socket_clients = set() def send_all_async(message): @@ -17,6 +17,7 @@ def send_all_async(message): pass + class ChameleonWebSocket(tornado.websocket.WebSocketHandler): actions = {} @@ -39,7 +40,7 @@ class ChameleonWebSocket(tornado.websocket.WebSocketHandler): def open(self): self.send_full_settings() if self not in web_socket_clients: - web_socket_clients.append(self) + web_socket_clients.add(self) print("WebSocket opened") diff --git a/backend/app/handlers/VisionHandler.py b/backend/app/handlers/VisionHandler.py index 50127e068..823dc24e2 100644 --- a/backend/app/handlers/VisionHandler.py +++ b/backend/app/handlers/VisionHandler.py @@ -247,8 +247,8 @@ class VisionHandler(metaclass=Singleton): return yaw def run(self): - NetworkTables.startClientTeam(team=self.settings_manager.general_settings.get("team_number", 1577)) - # NetworkTables.initialize("localhost") + # NetworkTables.startClientTeam(team=self.settings_manager.general_settings.get("team_number", 1577)) + NetworkTables.initialize("localhost") port = 5550 @@ -318,10 +318,11 @@ class VisionHandler(metaclass=Singleton): def _image_thread(): global image global p_image + global time_stamp image = numpy.zeros(shape=(width, height, 3), dtype=numpy.uint8) p_image = image while True: - _, image = cv_sink.grabFrame(image) + time_stamp, image = cv_sink.grabFrame(image) def _publish_thread(): # asyncio.set_event_loop(asyncio.new_event_loop()) @@ -334,19 +335,9 @@ class VisionHandler(metaclass=Singleton): #send the point using network tables table.putNumber('pitch', nt_data['pitch']) table.putNumber('yaw', nt_data['yaw']) + table.putNumber('fps', nt_data['fps']) + table.putNumber('time_stamp', time_stamp) #if the selected camera in ui is this cam send the point to the ui - if self.settings_manager.general_settings['curr_camera'] == cam_name: - try: - send_all_async({ - 'raw_point': nt_data['raw_point'], - 'point': { - 'pitch': nt_data['pitch'], - 'yaw': nt_data['yaw'], - 'fps': nt_data['fps'] - } - }) - except: - pass except: pass From c8500cbcf78d563af2d93bd24f8246e0959e9a26 Mon Sep 17 00:00:00 2001 From: ori Date: Thu, 15 Aug 2019 09:40:18 -0700 Subject: [PATCH 09/15] first class setup --- backend/Main.py | 20 ++- backend/app/handlers/CameraHander.py | 223 ++++++++++++++++++++++++++ backend/app/handlers/SocketHandler.py | 3 +- backend/app/handlers/VisionHandler.py | 206 ------------------------ 4 files changed, 239 insertions(+), 213 deletions(-) create mode 100644 backend/app/handlers/CameraHander.py diff --git a/backend/Main.py b/backend/Main.py index e1a47bc3e..45f60d57c 100644 --- a/backend/Main.py +++ b/backend/Main.py @@ -1,3 +1,5 @@ +from datetime import timedelta +from networktables import NetworkTables import tornado.ioloop import logging from app.ChameleonVisionApp import ChameleonApplication @@ -6,6 +8,8 @@ from tornado.options import options from app.handlers.VisionHandler import VisionHandler import threading import asyncio +from app.handlers.SocketHandler import send_all_async +from app.handlers.CameraHander import CameraHandler def run_server(): @@ -15,15 +19,21 @@ def run_server(): print(f"Serving on port {options.port}") app.listen(options.port) tornado.ioloop.IOLoop.current().start() - + tornado.ioloop.IOLoop.instance().add_timeout(timedelta(seconds=1), + send_all_async) +def run(): + # NetworkTables.startClientTeam(team=self.settings_manager.general_settings.get("team_number", 1577)) + NetworkTables.initialize("localhost") + port = 5550 + for cam_name in settings_manager.usb_cameras: + CameraHandler(cam_name, port).run() + port += 1 if __name__ == "__main__": logging.basicConfig(level=logging.DEBUG) - SettingsManager() - - VisionHandler().run() + settings_manager = SettingsManager() server_thread = threading.Thread(target=run_server) server_thread.start() - + run() while True: pass diff --git a/backend/app/handlers/CameraHander.py b/backend/app/handlers/CameraHander.py new file mode 100644 index 000000000..5d114409e --- /dev/null +++ b/backend/app/handlers/CameraHander.py @@ -0,0 +1,223 @@ +import math + +import numpy +from cscore import CameraServer +from app.classes.SettingsManager import SettingsManager +from ..handlers.SocketHandler import send_all_async +from multiprocessing import Process +import threading +import zmq +import asyncio +import time +from networktables import NetworkTables +import networktables +from .VisionHandler import VisionHandler + + +class CameraHandler: + def __init__(self, cam_name, port): + self.cs = CameraServer.getInstance() + self.settings_manager = SettingsManager() + self.vision_handler = VisionHandler() + self.port = port + self.cam_name = cam_name + + def run(self): + threading.Thread(target=self.thread_proc).start() + + def thread_proc(self): + cam_name = self.cam_name + port = self.port + global p_image + global nt_data + global table + nt_data = {'valid': False} + asyncio.set_event_loop(asyncio.new_event_loop()) + self.settings_manager.cams_curr_pipeline[cam_name] = "pipeline0" + pipeline = self.settings_manager.cams[cam_name]["pipelines"][self.settings_manager.cams_curr_pipeline[cam_name]] + FOV = self.settings_manager.cams[cam_name]["FOV"] + + def change_camera_values(pipline): + self.settings_manager.usb_cameras[cam_name].setBrightness(pipeline['brightness']) + self.settings_manager.usb_cameras[cam_name].setExposureManual(pipeline['exposure']) + self.settings_manager.usb_cameras[cam_name].setWhiteBalanceAuto() + + def pipeline_listener(table, key, value, is_new): + asyncio.set_event_loop(asyncio.new_event_loop()) + self.settings_manager.cams_curr_pipeline[cam_name] = value + change_camera_values(pipeline) + if cam_name == self.settings_manager.general_settings['curr_camera']: + self.settings_manager.general_settings['curr_pipeline'] = value + update_settings = self.settings_manager.get_curr_pipeline() + update_settings['curr_pipeline'] = self.settings_manager.general_settings["curr_pipeline"] + send_all_async(update_settings) + + def mode_listener(table, key, value, is_new): + change_camera_values({ + 'brightness': 25, + 'exposure': 15 + }) + + table = NetworkTables.getTable("/Chameleon-Vision/" + cam_name) + table.putString('Pipeline', self.settings_manager.cams_curr_pipeline[cam_name]) + table.addEntryListenerEx(pipeline_listener, key="Pipeline", + flags=networktables.NetworkTablesInstance.NotifyFlags.UPDATE) + table.addEntryListenerEx(mode_listener, key="Driver_Mode", + flags=networktables.NetworkTablesInstance.NotifyFlags.UPDATE) + # gettings video from curent camera + cv_sink = self.cs.getVideo(camera=self.settings_manager.usb_cameras[cam_name]) + + width = self.settings_manager.cams[cam_name]["video_mode"]["width"] + height = self.settings_manager.cams[cam_name]["video_mode"]["height"] + + # setting up a video server for camera + cv_publish = self.cs.putVideo(name=cam_name, width=width, height=height) + # saving camera port in cam name dict for usage in client + self.settings_manager.cams_port[cam_name] = self.cs._sinks['serve_' + cam_name].getPort() + + # setting up a zmq connection to the opencv subprocess + context = zmq.Context() + socket = context.socket(zmq.PAIR) + socket.bind('tcp://*:%s' % str(port)) + + # starting the process with inital values + p = Process(target=self.camera_process, args=(cam_name, port, FOV)) + p.start() + + change_camera_values(pipeline) + + def _image_thread(): + global image + global p_image + global time_stamp + image = numpy.zeros(shape=(width, height, 3), dtype=numpy.uint8) + p_image = image + while True: + time_stamp, image = cv_sink.grabFrame(image) + + def _publish_thread(): + # asyncio.set_event_loop(asyncio.new_event_loop()) + while True: + try: + cv_publish.putFrame(p_image) + table.putBoolean('valid', nt_data['valid']) + # check if point is valid + if nt_data['valid']: + # send the point using network tables + table.putNumber('pitch', nt_data['pitch']) + table.putNumber('yaw', nt_data['yaw']) + table.putNumber('fps', nt_data['fps']) + table.putNumber('time_stamp', time_stamp) + # if the selected camera in ui is this cam send the point to the ui + except: + pass + + threading.Thread(target=_image_thread).start() + threading.Thread(target=_publish_thread).start() + + while True: + pipeline = self.settings_manager.cams[cam_name]["pipelines"][ + self.settings_manager.cams_curr_pipeline[cam_name]] + socket.send_json(dict( + pipeline=pipeline + ), zmq.SNDMORE) + + socket.send_pyobj(image) + p_image = socket.recv_pyobj() + nt_data = socket.recv_json() + if self.settings_manager.general_settings['curr_camera'] == cam_name: + try: + send_all_async({ + 'raw_point': nt_data['raw_point'], + 'point': { + 'pitch': nt_data['pitch'], + 'yaw': nt_data['yaw'], + 'fps': nt_data['fps'] + } + }) + except: + pass + + def camera_process(self, cam_name, port, FOV): + from fractions import Fraction + + diagonalView = math.radians(FOV) # needs to be implemented in client + + width = self.settings_manager.cams[cam_name]["video_mode"]["width"] + height = self.settings_manager.cams[cam_name]["video_mode"]["height"] + centerX = (width / 2) - .5 + centerY = (height / 2) - .5 + cam_area = width * height + + aspect_fraction = Fraction(width, height) + horizontal_ratio = aspect_fraction.numerator + vertical_ratio = aspect_fraction.denominator + + horizontalView = math.atan(math.tan(diagonalView / 2) * (horizontal_ratio / diagonalView)) * 2 + verticalView = math.atan(math.tan(diagonalView / 2) * (vertical_ratio / diagonalView)) * 2 + + H_FOCAL_LENGTH = width / (2 * math.tan((horizontalView / 2))) + V_FOCAL_LENGTH = height / (2 * math.tan((verticalView / 2))) + + context = zmq.Context() + socket = context.socket(zmq.PAIR) + socket.connect('tcp://localhost:%s' % str(port)) + filter_contours = self.vision_handler.Filter_Contours(center_x=centerX, center_y=centerY) + x = 1 + counter = 0 + start_time = time.time() + fps = 0 + while True: + obj = socket.recv_json() + image = socket.recv_pyobj() + curr_pipeline = obj["pipeline"] + if curr_pipeline['orientation'] == "Inverted": + M = cv2.getRotationMatrix2D((width / 2, height / 2), 180, 1) + image = cv2.warpAffine(image, M, (width, height)) + hsv_image = self.vision_handler._hsv_threshold(curr_pipeline["hue"], + curr_pipeline["saturation"], curr_pipeline["value"], + image, curr_pipeline["erode"], curr_pipeline["dilate"]) + # if table.getBoolean("Driver_Mode", False): + contours = self.vision_handler.find_contours(hsv_image) + filtered_contours = filter_contours.filter_contours(input_contours=contours, area=curr_pipeline['area'], + ratio=curr_pipeline['ratio'], + extent=curr_pipeline['extent'], + sort_mode=curr_pipeline['sort_mode'], cam_area=cam_area, + target_grouping=curr_pipeline['target_group'], + target_intersection= + curr_pipeline['target_intersection']) + final_contour = self.vision_handler.output_contour(filtered_contours) + try: + center = final_contour[0] + center_x = (center[1] - curr_pipeline['B']) / curr_pipeline["M"] + center_y = (center[0] * curr_pipeline["M"]) + curr_pipeline["B"] + pitch = self.vision_handler.calculate_pitch(pixel_y=center[1], center_y=center_y, v_focal_length=V_FOCAL_LENGTH) + yaw = self.vision_handler.calculate_yaw(pixel_x=center[0], center_x=center_x, h_focal_length=H_FOCAL_LENGTH) + valid = True + except IndexError: + center = None + pitch = None + yaw = None + valid = False + + if curr_pipeline['is_binary']: + draw_image = hsv_image + else: + draw_image = image + res = self.vision_handler.draw_image(input_image=draw_image, contour=final_contour) + socket.send_pyobj(res) + socket.send_json(dict( + pitch=pitch, + yaw=yaw, + valid=valid, + raw_point=center, + fps=fps + )) + counter += 1 + if (time.time() - start_time) > x: + fps = (counter / (time.time() - start_time)) + counter = 0 + start_time = time.time() + + + diff --git a/backend/app/handlers/SocketHandler.py b/backend/app/handlers/SocketHandler.py index 228cbc30d..6a4eaad41 100644 --- a/backend/app/handlers/SocketHandler.py +++ b/backend/app/handlers/SocketHandler.py @@ -1,4 +1,4 @@ -import asyncio +from datetime import timedelta import tornado.websocket import json @@ -17,7 +17,6 @@ def send_all_async(message): pass - class ChameleonWebSocket(tornado.websocket.WebSocketHandler): actions = {} diff --git a/backend/app/handlers/VisionHandler.py b/backend/app/handlers/VisionHandler.py index 823dc24e2..38a7ca9e4 100644 --- a/backend/app/handlers/VisionHandler.py +++ b/backend/app/handlers/VisionHandler.py @@ -1,26 +1,13 @@ -import asyncio -import time - -from networktables import NetworkTables -import networktables import cv2 import numpy -from cscore import CameraServer -from app.classes.SettingsManager import SettingsManager from ..classes.Singleton import Singleton -from multiprocessing import Process -import threading -import zmq import math from enum import Enum, unique -from ..handlers.SocketHandler import send_all_async class VisionHandler(metaclass=Singleton): def __init__(self): self.kernel = numpy.ones((5, 5), numpy.uint8) - self.cs = CameraServer.getInstance() - self.settings_manager = SettingsManager() def _hsv_threshold(self, hue: list, saturation: list, value: list, img: numpy.ndarray, is_erode: bool, is_dilate: bool): @@ -245,196 +232,3 @@ class VisionHandler(metaclass=Singleton): def calculate_yaw(self, pixel_x, center_x, h_focal_length): yaw = math.degrees(math.atan((pixel_x - center_x) / h_focal_length)) return yaw - - def run(self): - # NetworkTables.startClientTeam(team=self.settings_manager.general_settings.get("team_number", 1577)) - NetworkTables.initialize("localhost") - - port = 5550 - - for cam_name in self.settings_manager.usb_cameras: - threading.Thread(target=self.thread_proc, args=(cam_name, port)).start() - port += 1 - - def thread_proc(self, cam_name, port=5557): - global p_image - global nt_data - global table - nt_data = {'valid': False} - # asyncio.set_event_loop(asyncio.new_event_loop()) - self.settings_manager.cams_curr_pipeline[cam_name] = "pipeline0" - pipeline = self.settings_manager.cams[cam_name]["pipelines"][self.settings_manager.cams_curr_pipeline[cam_name]] - FOV = self.settings_manager.cams[cam_name]["FOV"] - - def change_camera_values(pipline): - self.settings_manager.usb_cameras[cam_name].setBrightness(pipeline['brightness']) - self.settings_manager.usb_cameras[cam_name].setExposureManual(pipeline['exposure']) - self.settings_manager.usb_cameras[cam_name].setWhiteBalanceAuto() - - def pipeline_listener(table, key, value, is_new): - asyncio.set_event_loop(asyncio.new_event_loop()) - self.settings_manager.cams_curr_pipeline[cam_name] = value - change_camera_values(pipeline) - if cam_name == self.settings_manager.general_settings['curr_camera']: - self.settings_manager.general_settings['curr_pipeline'] = value - update_settings = self.settings_manager.get_curr_pipeline() - update_settings['curr_pipeline'] = self.settings_manager.general_settings["curr_pipeline"] - send_all_async(update_settings) - - def mode_listener(table, key, value, is_new): - change_camera_values({ - 'brightness': 25, - 'exposure': 15 - }) - - table = NetworkTables.getTable("/Chameleon-Vision/" + cam_name) - table.putString('Pipeline', self.settings_manager.cams_curr_pipeline[cam_name]) - table.addEntryListenerEx(pipeline_listener, key="Pipeline", - flags=networktables.NetworkTablesInstance.NotifyFlags.UPDATE) - table.addEntryListenerEx(mode_listener, key="Driver_Mode", - flags=networktables.NetworkTablesInstance.NotifyFlags.UPDATE) - #gettings video from curent camera - cv_sink = self.cs.getVideo(camera=self.settings_manager.usb_cameras[cam_name]) - - width = self.settings_manager.cams[cam_name]["video_mode"]["width"] - height = self.settings_manager.cams[cam_name]["video_mode"]["height"] - - #setting up a video server for camera - cv_publish = self.cs.putVideo(name=cam_name, width=width, height=height) - # saving camera port in cam name dict for usage in client - self.settings_manager.cams_port[cam_name] = self.cs._sinks['serve_'+cam_name].getPort() - - #setting up a zmq connection to the opencv subprocess - context = zmq.Context() - socket = context.socket(zmq.PAIR) - socket.bind('tcp://*:%s' % str(port)) - - #starting the process with inital values - p = Process(target=self.camera_process, args=(cam_name, port, FOV)) - p.start() - - change_camera_values(pipeline) - - def _image_thread(): - global image - global p_image - global time_stamp - image = numpy.zeros(shape=(width, height, 3), dtype=numpy.uint8) - p_image = image - while True: - time_stamp, image = cv_sink.grabFrame(image) - - def _publish_thread(): - # asyncio.set_event_loop(asyncio.new_event_loop()) - while True: - try: - cv_publish.putFrame(p_image) - table.putBoolean('valid', nt_data['valid']) - # check if point is valid - if nt_data['valid']: - #send the point using network tables - table.putNumber('pitch', nt_data['pitch']) - table.putNumber('yaw', nt_data['yaw']) - table.putNumber('fps', nt_data['fps']) - table.putNumber('time_stamp', time_stamp) - #if the selected camera in ui is this cam send the point to the ui - except: - pass - - threading.Thread(target=_image_thread).start() - threading.Thread(target=_publish_thread).start() - - while True: - pipeline = self.settings_manager.cams[cam_name]["pipelines"][self.settings_manager.cams_curr_pipeline[cam_name]] - socket.send_json(dict( - pipeline=pipeline - ), zmq.SNDMORE) - - socket.send_pyobj(image) - p_image = socket.recv_pyobj() - nt_data = socket.recv_json() - #send the image to the camera server - - def camera_process(self, cam_name, port, FOV): - from fractions import Fraction - - diagonalView = math.radians(FOV) #needs to be implemented in client - - width = self.settings_manager.cams[cam_name]["video_mode"]["width"] - height = self.settings_manager.cams[cam_name]["video_mode"]["height"] - centerX = (width / 2) - .5 - centerY = (height / 2) - .5 - cam_area = width * height - - aspect_fraction = Fraction(width, height) - horizontal_ratio = aspect_fraction.numerator - vertical_ratio = aspect_fraction.denominator - - horizontalView = math.atan(math.tan(diagonalView/2) * (horizontal_ratio / diagonalView)) * 2 - verticalView = math.atan(math.tan(diagonalView/2) * (vertical_ratio / diagonalView)) * 2 - - H_FOCAL_LENGTH = width / (2*math.tan((horizontalView/2))) - V_FOCAL_LENGTH = height / (2*math.tan((verticalView/2))) - - context = zmq.Context() - socket = context.socket(zmq.PAIR) - socket.connect('tcp://localhost:%s' % str(port)) - filter_contours = self.Filter_Contours(center_x=centerX, center_y=centerY) - x = 1 - counter = 0 - start_time = time.time() - fps = 0 - while True: - obj = socket.recv_json() - image = socket.recv_pyobj() - curr_pipeline = obj["pipeline"] - if curr_pipeline['orientation'] == "Inverted": - M = cv2.getRotationMatrix2D((width / 2, height / 2), 180, 1) - image = cv2.warpAffine(image, M, (width, height)) - hsv_image = self._hsv_threshold(curr_pipeline["hue"], - curr_pipeline["saturation"], curr_pipeline["value"], - image, curr_pipeline["erode"], curr_pipeline["dilate"]) - # if table.getBoolean("Driver_Mode", False): - contours = self.find_contours(hsv_image) - filtered_contours = filter_contours.filter_contours(input_contours=contours, area=curr_pipeline['area'], - ratio=curr_pipeline['ratio'], - extent=curr_pipeline['extent'], - sort_mode=curr_pipeline['sort_mode'], cam_area=cam_area, - target_grouping=curr_pipeline['target_group'], - target_intersection= - curr_pipeline['target_intersection']) - final_contour = self.output_contour(filtered_contours) - try: - center = final_contour[0] - center_x = (center[1] - curr_pipeline['B']) / curr_pipeline["M"] - center_y = (center[0] * curr_pipeline["M"]) + curr_pipeline["B"] - pitch = self.calculate_pitch(pixel_y=center[1], center_y=center_y, v_focal_length=V_FOCAL_LENGTH) - yaw = self.calculate_yaw(pixel_x=center[0], center_x=center_x, h_focal_length=H_FOCAL_LENGTH) - valid = True - except IndexError: - center = None - pitch = None - yaw = None - valid = False - - if curr_pipeline['is_binary']: - draw_image = hsv_image - else: - draw_image = image - res = self.draw_image(input_image=draw_image, contour=final_contour) - socket.send_pyobj(res) - socket.send_json(dict( - pitch=pitch, - yaw=yaw, - valid=valid, - raw_point=center, - fps=fps - )) - counter += 1 - if (time.time() - start_time) > x: - fps = (counter / (time.time() - start_time)) - counter = 0 - start_time = time.time() - - - From f5f8a7376b180bf63b856aa690941f6ce6423153 Mon Sep 17 00:00:00 2001 From: ori Date: Thu, 15 Aug 2019 10:43:01 -0700 Subject: [PATCH 10/15] test on odroid --- backend/app/handlers/CameraHander.py | 57 ++++++++++++++-------------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/backend/app/handlers/CameraHander.py b/backend/app/handlers/CameraHander.py index 5d114409e..7873adb15 100644 --- a/backend/app/handlers/CameraHander.py +++ b/backend/app/handlers/CameraHander.py @@ -1,5 +1,5 @@ import math - +import cv2 import numpy from cscore import CameraServer from app.classes.SettingsManager import SettingsManager @@ -21,6 +21,11 @@ class CameraHandler: self.vision_handler = VisionHandler() self.port = port self.cam_name = cam_name + self.image = None + self.p_image = None + self.table = None + self.nt_data = {'valid': False} + self.time_stamp = 0 def run(self): threading.Thread(target=self.thread_proc).start() @@ -28,10 +33,7 @@ class CameraHandler: def thread_proc(self): cam_name = self.cam_name port = self.port - global p_image - global nt_data - global table - nt_data = {'valid': False} + asyncio.set_event_loop(asyncio.new_event_loop()) self.settings_manager.cams_curr_pipeline[cam_name] = "pipeline0" pipeline = self.settings_manager.cams[cam_name]["pipelines"][self.settings_manager.cams_curr_pipeline[cam_name]] @@ -58,11 +60,11 @@ class CameraHandler: 'exposure': 15 }) - table = NetworkTables.getTable("/Chameleon-Vision/" + cam_name) - table.putString('Pipeline', self.settings_manager.cams_curr_pipeline[cam_name]) - table.addEntryListenerEx(pipeline_listener, key="Pipeline", + self.table = NetworkTables.getTable("/Chameleon-Vision/" + cam_name) + self.table.putString('Pipeline', self.settings_manager.cams_curr_pipeline[cam_name]) + self.table.addEntryListenerEx(pipeline_listener, key="Pipeline", flags=networktables.NetworkTablesInstance.NotifyFlags.UPDATE) - table.addEntryListenerEx(mode_listener, key="Driver_Mode", + self.table.addEntryListenerEx(mode_listener, key="Driver_Mode", flags=networktables.NetworkTablesInstance.NotifyFlags.UPDATE) # gettings video from curent camera cv_sink = self.cs.getVideo(camera=self.settings_manager.usb_cameras[cam_name]) @@ -87,27 +89,24 @@ class CameraHandler: change_camera_values(pipeline) def _image_thread(): - global image - global p_image - global time_stamp - image = numpy.zeros(shape=(width, height, 3), dtype=numpy.uint8) - p_image = image + self.image = numpy.zeros(shape=(width, height, 3), dtype=numpy.uint8) + self.p_image = self.image while True: - time_stamp, image = cv_sink.grabFrame(image) + self.time_stamp, self.image = cv_sink.grabFrame(self.image) def _publish_thread(): # asyncio.set_event_loop(asyncio.new_event_loop()) while True: try: - cv_publish.putFrame(p_image) - table.putBoolean('valid', nt_data['valid']) + cv_publish.putFrame(self.p_image) + self.table.putBoolean('valid', self.nt_data['valid']) # check if point is valid - if nt_data['valid']: + if self.nt_data['valid']: # send the point using network tables - table.putNumber('pitch', nt_data['pitch']) - table.putNumber('yaw', nt_data['yaw']) - table.putNumber('fps', nt_data['fps']) - table.putNumber('time_stamp', time_stamp) + self.table.putNumber('pitch', self.nt_data['pitch']) + self.table.putNumber('yaw', self.nt_data['yaw']) + self.table.putNumber('fps', self.nt_data['fps']) + self.table.putNumber('time_stamp', self.time_stamp) # if the selected camera in ui is this cam send the point to the ui except: pass @@ -122,17 +121,17 @@ class CameraHandler: pipeline=pipeline ), zmq.SNDMORE) - socket.send_pyobj(image) - p_image = socket.recv_pyobj() - nt_data = socket.recv_json() + socket.send_pyobj(self.image) + self.p_image = socket.recv_pyobj() + self.nt_data = socket.recv_json() if self.settings_manager.general_settings['curr_camera'] == cam_name: try: send_all_async({ - 'raw_point': nt_data['raw_point'], + 'raw_point': self.nt_data['raw_point'], 'point': { - 'pitch': nt_data['pitch'], - 'yaw': nt_data['yaw'], - 'fps': nt_data['fps'] + 'pitch': self.nt_data['pitch'], + 'yaw': self.nt_data['yaw'], + 'fps': self.nt_data['fps'] } }) except: From a468d8b1a0a3d6d7a2df2e45b522fddf135cf205 Mon Sep 17 00:00:00 2001 From: ori Date: Thu, 15 Aug 2019 10:53:42 -0700 Subject: [PATCH 11/15] removed websock --- backend/app/handlers/CameraHander.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/backend/app/handlers/CameraHander.py b/backend/app/handlers/CameraHander.py index 7873adb15..00b0834ff 100644 --- a/backend/app/handlers/CameraHander.py +++ b/backend/app/handlers/CameraHander.py @@ -124,18 +124,18 @@ class CameraHandler: socket.send_pyobj(self.image) self.p_image = socket.recv_pyobj() self.nt_data = socket.recv_json() - if self.settings_manager.general_settings['curr_camera'] == cam_name: - try: - send_all_async({ - 'raw_point': self.nt_data['raw_point'], - 'point': { - 'pitch': self.nt_data['pitch'], - 'yaw': self.nt_data['yaw'], - 'fps': self.nt_data['fps'] - } - }) - except: - pass + # if self.settings_manager.general_settings['curr_camera'] == cam_name: + # try: + # send_all_async({ + # 'raw_point': self.nt_data['raw_point'], + # 'point': { + # 'pitch': self.nt_data['pitch'], + # 'yaw': self.nt_data['yaw'], + # 'fps': self.nt_data['fps'] + # } + # }) + # except: + # pass def camera_process(self, cam_name, port, FOV): from fractions import Fraction From ff7a9b81555d5189a51cd7fbf7282f7f761237ab Mon Sep 17 00:00:00 2001 From: ori Date: Thu, 15 Aug 2019 11:17:26 -0700 Subject: [PATCH 12/15] test on rpi --- backend/Main.py | 10 +++++++--- backend/app/handlers/CameraHander.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/backend/Main.py b/backend/Main.py index 45f60d57c..7e8054a70 100644 --- a/backend/Main.py +++ b/backend/Main.py @@ -21,19 +21,23 @@ def run_server(): tornado.ioloop.IOLoop.current().start() tornado.ioloop.IOLoop.instance().add_timeout(timedelta(seconds=1), send_all_async) + + def run(): - # NetworkTables.startClientTeam(team=self.settings_manager.general_settings.get("team_number", 1577)) - NetworkTables.initialize("localhost") + NetworkTables.startClientTeam(team=settings_manager.general_settings.get("team_number", 1577)) + # NetworkTables.initialize("localhost") port = 5550 for cam_name in settings_manager.usb_cameras: CameraHandler(cam_name, port).run() port += 1 + if __name__ == "__main__": logging.basicConfig(level=logging.DEBUG) settings_manager = SettingsManager() + run() server_thread = threading.Thread(target=run_server) server_thread.start() - run() + while True: pass diff --git a/backend/app/handlers/CameraHander.py b/backend/app/handlers/CameraHander.py index 00b0834ff..d3541685e 100644 --- a/backend/app/handlers/CameraHander.py +++ b/backend/app/handlers/CameraHander.py @@ -124,7 +124,7 @@ class CameraHandler: socket.send_pyobj(self.image) self.p_image = socket.recv_pyobj() self.nt_data = socket.recv_json() - # if self.settings_manager.general_settings['curr_camera'] == cam_name: + # if self.settings_manager.general_settings['curr_camera'] == self.cam_name: # try: # send_all_async({ # 'raw_point': self.nt_data['raw_point'], From 81cee9cf2bb99ca0c6037224fc35ab98d6e4281d Mon Sep 17 00:00:00 2001 From: ori Date: Thu, 15 Aug 2019 15:21:51 -0700 Subject: [PATCH 13/15] fixed threading in camera class and code cleanup --- backend/Main.py | 1 - backend/app/ChameleonVisionApp.py | 1 - backend/app/classes/SettingsManager.py | 1 - backend/app/handlers/CameraHander.py | 105 +++++++++++++------------ backend/app/handlers/SocketHandler.py | 2 - backend/app/handlers/VisionHandler.py | 3 +- 6 files changed, 55 insertions(+), 58 deletions(-) diff --git a/backend/Main.py b/backend/Main.py index 7e8054a70..48631c8ef 100644 --- a/backend/Main.py +++ b/backend/Main.py @@ -5,7 +5,6 @@ import logging from app.ChameleonVisionApp import ChameleonApplication from app.classes.SettingsManager import SettingsManager from tornado.options import options -from app.handlers.VisionHandler import VisionHandler import threading import asyncio from app.handlers.SocketHandler import send_all_async diff --git a/backend/app/ChameleonVisionApp.py b/backend/app/ChameleonVisionApp.py index cd786c272..7336ae001 100644 --- a/backend/app/ChameleonVisionApp.py +++ b/backend/app/ChameleonVisionApp.py @@ -1,7 +1,6 @@ import tornado.web import tornado.websocket import os - from .handlers.MainHandler import MainHandler from .handlers.SocketHandler import ChameleonWebSocket from tornado.options import define diff --git a/backend/app/classes/SettingsManager.py b/backend/app/classes/SettingsManager.py index 9e74507cc..75ae2182b 100644 --- a/backend/app/classes/SettingsManager.py +++ b/backend/app/classes/SettingsManager.py @@ -1,4 +1,3 @@ -import socket import os import json import cv2 diff --git a/backend/app/handlers/CameraHander.py b/backend/app/handlers/CameraHander.py index d3541685e..04eb48ab7 100644 --- a/backend/app/handlers/CameraHander.py +++ b/backend/app/handlers/CameraHander.py @@ -16,6 +16,7 @@ from .VisionHandler import VisionHandler class CameraHandler: def __init__(self, cam_name, port): + #settings vars up for vision loop self.cs = CameraServer.getInstance() self.settings_manager = SettingsManager() self.vision_handler = VisionHandler() @@ -28,27 +29,24 @@ class CameraHandler: self.time_stamp = 0 def run(self): + #starting main thread threading.Thread(target=self.thread_proc).start() def thread_proc(self): - cam_name = self.cam_name - port = self.port - - asyncio.set_event_loop(asyncio.new_event_loop()) - self.settings_manager.cams_curr_pipeline[cam_name] = "pipeline0" - pipeline = self.settings_manager.cams[cam_name]["pipelines"][self.settings_manager.cams_curr_pipeline[cam_name]] - FOV = self.settings_manager.cams[cam_name]["FOV"] + self.settings_manager.cams_curr_pipeline[self.cam_name] = "pipeline0" + pipeline = self.settings_manager.cams[self.cam_name]["pipelines"][self.settings_manager.cams_curr_pipeline[self.cam_name]] + FOV = self.settings_manager.cams[self.cam_name]["FOV"] def change_camera_values(pipline): - self.settings_manager.usb_cameras[cam_name].setBrightness(pipeline['brightness']) - self.settings_manager.usb_cameras[cam_name].setExposureManual(pipeline['exposure']) - self.settings_manager.usb_cameras[cam_name].setWhiteBalanceAuto() + self.settings_manager.usb_cameras[self.cam_name].setBrightness(pipeline['brightness']) + self.settings_manager.usb_cameras[self.cam_name].setExposureManual(pipeline['exposure']) + self.settings_manager.usb_cameras[self.cam_name].setWhiteBalanceAuto() def pipeline_listener(table, key, value, is_new): asyncio.set_event_loop(asyncio.new_event_loop()) - self.settings_manager.cams_curr_pipeline[cam_name] = value + self.settings_manager.cams_curr_pipeline[self.cam_name] = value change_camera_values(pipeline) - if cam_name == self.settings_manager.general_settings['curr_camera']: + if self.cam_name == self.settings_manager.general_settings['curr_camera']: self.settings_manager.general_settings['curr_pipeline'] = value update_settings = self.settings_manager.get_curr_pipeline() update_settings['curr_pipeline'] = self.settings_manager.general_settings["curr_pipeline"] @@ -59,45 +57,43 @@ class CameraHandler: 'brightness': 25, 'exposure': 15 }) - - self.table = NetworkTables.getTable("/Chameleon-Vision/" + cam_name) - self.table.putString('Pipeline', self.settings_manager.cams_curr_pipeline[cam_name]) + #setting up network table + self.table = NetworkTables.getTable("/Chameleon-Vision/" + self.cam_name) + self.table.putString('Pipeline', self.settings_manager.cams_curr_pipeline[self.cam_name]) self.table.addEntryListenerEx(pipeline_listener, key="Pipeline", flags=networktables.NetworkTablesInstance.NotifyFlags.UPDATE) self.table.addEntryListenerEx(mode_listener, key="Driver_Mode", flags=networktables.NetworkTablesInstance.NotifyFlags.UPDATE) - # gettings video from curent camera - cv_sink = self.cs.getVideo(camera=self.settings_manager.usb_cameras[cam_name]) - width = self.settings_manager.cams[cam_name]["video_mode"]["width"] - height = self.settings_manager.cams[cam_name]["video_mode"]["height"] + # getting video from current camera + cv_sink = self.cs.getVideo(camera=self.settings_manager.usb_cameras[self.cam_name]) + + width = self.settings_manager.cams[self.cam_name]["video_mode"]["width"] + height = self.settings_manager.cams[self.cam_name]["video_mode"]["height"] # setting up a video server for camera - cv_publish = self.cs.putVideo(name=cam_name, width=width, height=height) + cv_publish = self.cs.putVideo(name=self.cam_name, width=width, height=height) # saving camera port in cam name dict for usage in client - self.settings_manager.cams_port[cam_name] = self.cs._sinks['serve_' + cam_name].getPort() + self.settings_manager.cams_port[self.cam_name] = self.cs._sinks['serve_' + self.cam_name].getPort() # setting up a zmq connection to the opencv subprocess context = zmq.Context() socket = context.socket(zmq.PAIR) - socket.bind('tcp://*:%s' % str(port)) + socket.bind('tcp://*:%s' % str(self.port)) - # starting the process with inital values - p = Process(target=self.camera_process, args=(cam_name, port, FOV)) + # starting the process with initial values + p = Process(target=self.camera_process, args=(self.cam_name, self.port, FOV)) p.start() change_camera_values(pipeline) - def _image_thread(): + def _publish_thread(): + #getting image values and publishing process image and data self.image = numpy.zeros(shape=(width, height, 3), dtype=numpy.uint8) self.p_image = self.image - while True: - self.time_stamp, self.image = cv_sink.grabFrame(self.image) - - def _publish_thread(): - # asyncio.set_event_loop(asyncio.new_event_loop()) while True: try: + self.time_stamp, self.image = cv_sink.grabFrame(self.image) cv_publish.putFrame(self.p_image) self.table.putBoolean('valid', self.nt_data['valid']) # check if point is valid @@ -111,12 +107,31 @@ class CameraHandler: except: pass - threading.Thread(target=_image_thread).start() + def _socket_thread(): + #publishing to websocket at slower interval + asyncio.set_event_loop(asyncio.new_event_loop()) + while True: + time.sleep(0.05) + if self.settings_manager.general_settings['curr_camera'] == self.cam_name: + try: + send_all_async({ + 'raw_point': self.nt_data['raw_point'], + 'point': { + 'pitch': self.nt_data['pitch'], + 'yaw': self.nt_data['yaw'], + 'fps': self.nt_data['fps'] + } + }) + except: + pass + threading.Thread(target=_publish_thread).start() + threading.Thread(target=_socket_thread).start() while True: - pipeline = self.settings_manager.cams[cam_name]["pipelines"][ - self.settings_manager.cams_curr_pipeline[cam_name]] + #sending and reciving data from opencv sub process + pipeline = self.settings_manager.cams[self.cam_name]["pipelines"][ + self.settings_manager.cams_curr_pipeline[self.cam_name]] socket.send_json(dict( pipeline=pipeline ), zmq.SNDMORE) @@ -124,23 +139,11 @@ class CameraHandler: socket.send_pyobj(self.image) self.p_image = socket.recv_pyobj() self.nt_data = socket.recv_json() - # if self.settings_manager.general_settings['curr_camera'] == self.cam_name: - # try: - # send_all_async({ - # 'raw_point': self.nt_data['raw_point'], - # 'point': { - # 'pitch': self.nt_data['pitch'], - # 'yaw': self.nt_data['yaw'], - # 'fps': self.nt_data['fps'] - # } - # }) - # except: - # pass def camera_process(self, cam_name, port, FOV): from fractions import Fraction - - diagonalView = math.radians(FOV) # needs to be implemented in client + #calc fov + diagonalView = math.radians(FOV) width = self.settings_manager.cams[cam_name]["video_mode"]["width"] height = self.settings_manager.cams[cam_name]["video_mode"]["height"] @@ -157,15 +160,18 @@ class CameraHandler: H_FOCAL_LENGTH = width / (2 * math.tan((horizontalView / 2))) V_FOCAL_LENGTH = height / (2 * math.tan((verticalView / 2))) - + #setting up zmq socket context = zmq.Context() socket = context.socket(zmq.PAIR) socket.connect('tcp://localhost:%s' % str(port)) + #setting up filter countours class filter_contours = self.vision_handler.Filter_Contours(center_x=centerX, center_y=centerY) + x = 1 counter = 0 start_time = time.time() fps = 0 + while True: obj = socket.recv_json() image = socket.recv_pyobj() @@ -217,6 +223,3 @@ class CameraHandler: fps = (counter / (time.time() - start_time)) counter = 0 start_time = time.time() - - - diff --git a/backend/app/handlers/SocketHandler.py b/backend/app/handlers/SocketHandler.py index 6a4eaad41..b6f54d136 100644 --- a/backend/app/handlers/SocketHandler.py +++ b/backend/app/handlers/SocketHandler.py @@ -1,5 +1,3 @@ -from datetime import timedelta - import tornado.websocket import json from ..classes.Exceptions import NoCameraConnectedException diff --git a/backend/app/handlers/VisionHandler.py b/backend/app/handlers/VisionHandler.py index 38a7ca9e4..b669c0177 100644 --- a/backend/app/handlers/VisionHandler.py +++ b/backend/app/handlers/VisionHandler.py @@ -1,11 +1,10 @@ import cv2 import numpy -from ..classes.Singleton import Singleton import math from enum import Enum, unique -class VisionHandler(metaclass=Singleton): +class VisionHandler(): def __init__(self): self.kernel = numpy.ones((5, 5), numpy.uint8) From 5bea7f8d5c6e24cdb2ce835062e26a80ef5f5f99 Mon Sep 17 00:00:00 2001 From: ori agranat Date: Fri, 16 Aug 2019 15:00:28 +0300 Subject: [PATCH 14/15] removed unnecessary loop from main file --- README.md | 6 +++++- backend/Main.py | 3 +-- backend/app/handlers/CameraHander.py | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1ccde2e98..69361b567 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,11 @@ of you can auto serve the ui by ``` npm run serve ``` - +## Hardware +this is important when choosing your sbc it is more important to have a good usb controller that a good cpu +on the odroid xu4 which is very fast i have got many bottlenecks from the usb controller and many times making the program crach +#### networking +it is very important to install Bonjour ## docs diff --git a/backend/Main.py b/backend/Main.py index 48631c8ef..02aa2c372 100644 --- a/backend/Main.py +++ b/backend/Main.py @@ -31,6 +31,7 @@ def run(): port += 1 + if __name__ == "__main__": logging.basicConfig(level=logging.DEBUG) settings_manager = SettingsManager() @@ -38,5 +39,3 @@ if __name__ == "__main__": server_thread = threading.Thread(target=run_server) server_thread.start() - while True: - pass diff --git a/backend/app/handlers/CameraHander.py b/backend/app/handlers/CameraHander.py index 04eb48ab7..8690693f4 100644 --- a/backend/app/handlers/CameraHander.py +++ b/backend/app/handlers/CameraHander.py @@ -111,7 +111,7 @@ class CameraHandler: #publishing to websocket at slower interval asyncio.set_event_loop(asyncio.new_event_loop()) while True: - time.sleep(0.05) + time.sleep(0.1) if self.settings_manager.general_settings['curr_camera'] == self.cam_name: try: send_all_async({ From 2cebee43c4d83d01fc7f4c08c7eb3886386c69fd Mon Sep 17 00:00:00 2001 From: ori agranat Date: Fri, 16 Aug 2019 18:10:49 +0300 Subject: [PATCH 15/15] updated readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 69361b567..1491e47d1 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ so in order to run this project we will need to install python in order to run t 2. apt-get dist-upgrade 3. sudo apt-get upgrade 4. sudo apt-get install python3-pip python3-dev cmake zip unzip build-essential git --fix-missing -5. sudo pip3 install numpy +5. sudo pip3 install numpy (if on raspberry pi do "sudo apt-get install python3-numpy") 6. OPENCV_VERSION=3.4.5 7. wget -O opencv.zip https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.zip 8. unzip opencv.zip