From 40d38e9d0155bfdd9e80188d07206f266f0a080c Mon Sep 17 00:00:00 2001 From: pabloinigoblasco Date: Wed, 11 Sep 2024 22:09:30 +0200 Subject: [PATCH 01/12] multiple input images implementation --- config/image_object_detection.yaml | 15 +- .../image_object_detection_node.py | 139 ++++++++++-------- 2 files changed, 90 insertions(+), 64 deletions(-) diff --git a/config/image_object_detection.yaml b/config/image_object_detection.yaml index 7fead2c..81187b4 100644 --- a/config/image_object_detection.yaml +++ b/config/image_object_detection.yaml @@ -6,7 +6,20 @@ image_object_detection_node: model.weights_file: yolov7-tiny.pt model.device: '0' - selected_detections: ['person'] + selected_detections: ['person'] # Classes to detect ['person', 'car'] show_image: False publish_debug_image: True + + # Lists of topics to subscribe + camera_topics: + - '/camera/image_raw' + # - '/camera1/image_raw' + # - '/camera2/image_raw' + # - '/camera3/image_raw' + + # QoS policy for the image subscriber + subscribers.qos_policy: 'best_effort' + + # QoS policy for the image debug publisher + image_debug_publisher.qos_policy: 'best_effort' diff --git a/src/image_object_detection/image_object_detection_node.py b/src/image_object_detection/image_object_detection_node.py index 855ddab..b566efa 100644 --- a/src/image_object_detection/image_object_detection_node.py +++ b/src/image_object_detection/image_object_detection_node.py @@ -19,6 +19,7 @@ import std_srvs.srv from sensor_msgs.msg import CompressedImage, Image +from vision_msgs.msg import Detection2D, ObjectHypothesisWithPose import torch import torch.backends.cudnn as cudnn @@ -27,7 +28,7 @@ from utils.general import check_img_size, non_max_suppression, scale_coords, xyxy2xywh, set_logging from utils.plots import plot_one_box from utils.torch_utils import select_device -from vision_msgs.msg import Detection2D, Detection2DArray, ObjectHypothesisWithPose +from vision_msgs.msg import Detection2DArray, Detection2D from ament_index_python.packages import get_package_share_directory PACKAGE_NAME = "image_object_detection" @@ -35,9 +36,9 @@ class ImageDetectObjectNode(Node): def __init__(self): - super().__init__("image_object_detection_node") + super().__init__("image_object_detection_node") - # parametros + # Model parameters self.declare_parameter("model.image_size", 640) self.model_image_size = ( self.get_parameter("model.image_size").get_parameter_value().integer_value @@ -129,60 +130,59 @@ def __init__(self): self.bridge = cv_bridge.CvBridge() - self.image_sub = self.create_subscription( - msg_type=Image, topic="image", callback=self.image_callback, qos_profile=self.qos + # Get the list of camera topics from the config file + self.declare_parameter("camera_topics", []) + self.camera_topics = ( + self.get_parameter("camera_topics").get_parameter_value().string_array_value ) - - self.image_compressed_sub = self.create_subscription( - msg_type=CompressedImage, - topic="image/compressed", - callback=self.image_compressed_callback, - qos_profile=self.qos, - ) - - self.detection_publisher = self.create_publisher( - msg_type=Detection2DArray, topic="detections", qos_profile=self.qos - ) - - if self.enable_publish_debug_image: - if self.qos_policy == "best_effort": - self.get_logger().info("Using best effort qos policy for debug image publisher") - self.qos = QoSProfile( - reliability=QoSReliabilityPolicy.BEST_EFFORT, - history=QoSHistoryPolicy.KEEP_LAST, - depth=1, - ) - else: - self.get_logger().info("Using reliable qos policy for debug image publisher") - self.qos = QoSProfile( - reliability=QoSReliabilityPolicy.RELIABLE, - history=QoSHistoryPolicy.KEEP_LAST, - depth=1, + self.get_logger().info(f"Subscribed to topics: {self.camera_topics}") + + # Initialize subscribers and publishers for each camera topic + self.subscribers = [] + self.detection_publishers = {} + self.debug_image_publishers = {} + + for topic in self.camera_topics: + # Create a subscriber for each camera topic + self.subscribers.append( + self.create_subscription( + Image, + topic, + callback=self.image_callback_factory(topic), + qos_profile=self.qos, ) + ) - self.debug_image_publisher = self.create_publisher( - msg_type=Image, topic="debug_image", qos_profile=self.qos + # Create a detection publisher for each camera + detection_topic = f"{topic}/detections" + self.detection_publishers[topic] = self.create_publisher( + Detection2DArray, detection_topic, self.qos ) + # Create a debug image publisher for each camera (if enabled) + if self.enable_publish_debug_image: + debug_image_topic = f"{topic}/debug_image" + self.debug_image_publishers[topic] = self.create_publisher( + Image, debug_image_topic, self.qos + ) + self.initialize_model() def initialize_model(self): with torch.no_grad(): - # Initialize set_logging() self.device = select_device(self.device) self.half = self.device.type != "cpu" - # Load model self.model = attempt_load( self.model_weights_file, map_location=self.device - ) # load FP32 model + ) self.stride = int(self.model.stride.max()) self.imgsz = check_img_size(self.model_image_size, s=self.stride) if self.half: - self.model.half() # to FP16 + self.model.half() cudnn.benchmark = True @@ -215,17 +215,13 @@ def accomodate_image_to_model(self, img0): def image_compressed_callback(self, msg): if not self.processing_enabled: return - - try: - self.cv_img = self.bridge.compressed_imgmsg_to_cv2(msg, self.debug_image_output_format) - img = self.accomodate_image_to_model(self.cv_img) - detections_msg, debugimg = self.predict(img, self.cv_img) + self.cv_img = self.bridge.compressed_imgmsg_to_cv2(msg, self.debug_image_output_format) + img = self.accomodate_image_to_model(self.cv_img) - self.detection_publisher.publish(detections_msg) - except CvBridgeError as e: - self.get_logger().error(f"Error converting image: {e}") - return + detections_msg, debugimg = self.predict(img, self.cv_img) + + self.detection_publisher.publish(detections_msg) if debugimg is not None: self.publish_debug_image(debugimg) @@ -234,25 +230,43 @@ def image_compressed_callback(self, msg): cv2.imshow("Compressed Image", debugimg) cv2.waitKey(1) - def image_callback(self, msg): - if not self.processing_enabled: - return + def image_callback_factory(self, topic): + def callback(msg): + try: + cv_img = self.bridge.imgmsg_to_cv2(msg, "bgr8") + self.image_queue[topic] = cv_img + except CvBridgeError as e: + self.get_logger().error(f"Error converting image from {topic}: {e}") - self.cv_img = self.bridge.imgmsg_to_cv2(msg, "bgr8") - img = self.accomodate_image_to_model(self.cv_img) + return callback + + def image_callback_factory(self, topic): + def callback(msg): + if not self.processing_enabled: + return - detections_msg, debugimg = self.predict(img, self.cv_img) + try: + cv_img = self.bridge.imgmsg_to_cv2(msg, "bgr8") + img = self.accomodate_image_to_model(cv_img) - self.detection_publisher.publish(detections_msg) + detections_msg, debugimg = self.predict(img, cv_img) - if debugimg is not None: - self.publish_debug_image(debugimg) + # Publish detections for the current camera + self.detection_publishers[topic].publish(detections_msg) - if self.show_image: - cv2.imshow("Detection", debugimg) - cv2.waitKey(1) + # Publish debug image for the current camera (if enabled) + if self.enable_publish_debug_image and topic in self.debug_image_publishers: + self.publish_debug_image(debugimg, topic) - def publish_debug_image(self, debugimg): + if self.show_image: + cv2.imshow(f"Detection from {topic}", debugimg) + cv2.waitKey(1) + except CvBridgeError as e: + self.get_logger().error(f"Error converting image from {topic}: {e}") + + return callback + + def publish_debug_image(self, debugimg, topic): if self.debug_image_output_format == "mono8": debugimg = cv2.cvtColor(debugimg, cv2.COLOR_RGB2GRAY) elif self.debug_image_output_format == "rgb8": @@ -261,11 +275,12 @@ def publish_debug_image(self, debugimg): debugimg = cv2.cvtColor(debugimg, cv2.COLOR_BGR2RGBA) else: self.get_logger().error( - "Unsupported debug image output format: {}".format(self.debug_image_output_format) + f"Unsupported debug image output format: {self.debug_image_output_format}" ) return - self.debug_image_publisher.publish( + # Publish the debug image for the current camera + self.debug_image_publishers[topic].publish( self.bridge.cv2_to_imgmsg(debugimg, self.debug_image_output_format) ) @@ -294,7 +309,6 @@ def predict(self, model_img, original_image): ).round() for *xyxy, conf, cls in reversed(det): - # clase clases deseadas if self.names[int(cls)] in self.selected_detections: detection2D_msg = Detection2D() xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() @@ -322,7 +336,6 @@ def predict(self, model_img, original_image): return detections_msg, original_image - def main(args=None): print(args) rclpy.init(args=sys.argv) From 5863fa0e170678a229d4da3dd4be13b8a49c8985 Mon Sep 17 00:00:00 2001 From: vlozano Date: Fri, 13 Sep 2024 07:29:55 +0200 Subject: [PATCH 02/12] Fix: Ahora recoge los valores del array camera_topics --- .../image_object_detection_node.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/image_object_detection/image_object_detection_node.py b/src/image_object_detection/image_object_detection_node.py index b566efa..febe503 100644 --- a/src/image_object_detection/image_object_detection_node.py +++ b/src/image_object_detection/image_object_detection_node.py @@ -36,7 +36,7 @@ class ImageDetectObjectNode(Node): def __init__(self): - super().__init__("image_object_detection_node") + super().__init__("image_object_detection_node") # Model parameters self.declare_parameter("model.image_size", 640) @@ -131,10 +131,9 @@ def __init__(self): self.bridge = cv_bridge.CvBridge() # Get the list of camera topics from the config file - self.declare_parameter("camera_topics", []) - self.camera_topics = ( - self.get_parameter("camera_topics").get_parameter_value().string_array_value - ) + self.declare_parameter("camera_topics", ["/cameras/frontleft_fisheye_image/image", "/cameras/frontright_fisheye_image/image", "/cameras/left_fisheye_image/image", "/cameras/right_fisheye_image/image"]) + self.camera_topics = self.get_parameter("camera_topics").get_parameter_value().string_array_value + self.get_logger().info(f"Subscribed to topics: {self.camera_topics}") # Initialize subscribers and publishers for each camera topic From d65a82579d3cf68ed38589c79b8707821b91eebb Mon Sep 17 00:00:00 2001 From: pabloinigoblasco Date: Thu, 28 Nov 2024 22:46:40 +0100 Subject: [PATCH 03/12] dynamic params, classes, model --- .../image_object_detection_node.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/image_object_detection/image_object_detection_node.py b/src/image_object_detection/image_object_detection_node.py index febe503..178bdaa 100644 --- a/src/image_object_detection/image_object_detection_node.py +++ b/src/image_object_detection/image_object_detection_node.py @@ -31,12 +31,18 @@ from vision_msgs.msg import Detection2DArray, Detection2D from ament_index_python.packages import get_package_share_directory +# Add this after the existing imports +from rcl_interfaces.msg import SetParametersResult + PACKAGE_NAME = "image_object_detection" class ImageDetectObjectNode(Node): def __init__(self): super().__init__("image_object_detection_node") + + # Add the parameter callback handler + self.add_on_set_parameters_callback(self.parameters_callback) # Model parameters self.declare_parameter("model.image_size", 640) @@ -166,6 +172,29 @@ def __init__(self): ) self.initialize_model() + def parameters_callback(self, params): + result = SetParametersResult(successful=True) + + for param in params: + if param.name == 'selected_detections': + self.selected_detections = param.value + self.get_logger().info(f"Updated selected_detections: {self.selected_detections}") + + elif param.name == 'model.iou_threshold': + self.iou_threshold = param.value + self.get_logger().info(f"Updated iou_threshold: {self.iou_threshold}") + + elif param.name == 'model.confidence': + self.confidence = param.value + self.get_logger().info(f"Updated confidence: {self.confidence}") + + elif param.name == 'model.weights_file': + self.model_weights_file = param.value + self.get_logger().info(f"Updated weights_file: {self.model_weights_file}") + # Reinitialize model with new weights + self.initialize_model() + + return result def initialize_model(self): with torch.no_grad(): From 60755596a567af192ea51fff05f423a9a8c1400f Mon Sep 17 00:00:00 2001 From: pabloinigoblasco Date: Thu, 28 Nov 2024 23:21:43 +0100 Subject: [PATCH 04/12] minor fix --- src/image_object_detection/image_object_detection_node.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/image_object_detection/image_object_detection_node.py b/src/image_object_detection/image_object_detection_node.py index 178bdaa..9b27f40 100644 --- a/src/image_object_detection/image_object_detection_node.py +++ b/src/image_object_detection/image_object_detection_node.py @@ -41,9 +41,6 @@ class ImageDetectObjectNode(Node): def __init__(self): super().__init__("image_object_detection_node") - # Add the parameter callback handler - self.add_on_set_parameters_callback(self.parameters_callback) - # Model parameters self.declare_parameter("model.image_size", 640) self.model_image_size = ( @@ -104,6 +101,10 @@ def __init__(self): .string_value ) + + # Add the parameter callback handler + self.add_on_set_parameters_callback(self.parameters_callback) + self.declare_parameter("subscribers.qos_policy", "best_effort") self.subscribers_qos = ( self.get_parameter("subscribers.qos_policy").get_parameter_value().string_value From e2d79b1fe3c83fca1de89c3e4cc21a8afc02d846 Mon Sep 17 00:00:00 2001 From: pabloinigoblasco Date: Thu, 28 Nov 2024 23:53:19 +0100 Subject: [PATCH 05/12] first progress --- .../image_object_detection_node.py | 33 +++++---- .../templates/index.html | 69 +++++++++++++++++++ .../web_interface_node.py | 57 +++++++++++++++ 3 files changed, 147 insertions(+), 12 deletions(-) create mode 100644 src/image_object_detection/templates/index.html create mode 100644 src/image_object_detection/web_interface_node.py diff --git a/src/image_object_detection/image_object_detection_node.py b/src/image_object_detection/image_object_detection_node.py index 9b27f40..2e31c86 100644 --- a/src/image_object_detection/image_object_detection_node.py +++ b/src/image_object_detection/image_object_detection_node.py @@ -366,15 +366,24 @@ def predict(self, model_img, original_image): return detections_msg, original_image def main(args=None): - print(args) - rclpy.init(args=sys.argv) - - minimal_publisher = ImageDetectObjectNode() - rclpy.spin(minimal_publisher) - - minimal_publisher.destroy_node() - rclpy.shutdown() - - -if __name__ == "__main__": - main(sys.argv) + rclpy.init(args=args) + + detection_node = ImageDetectObjectNode() + from image_object_detection.web_interface_node import WebInterfaceNode + web_interface = WebInterfaceNode(detection_node) + + # Use MultiThreadedExecutor to handle both nodes + executor = rclpy.executors.MultiThreadedExecutor() + executor.add_node(detection_node) + executor.add_node(web_interface) + + try: + executor.spin() + finally: + executor.shutdown() + detection_node.destroy_node() + web_interface.destroy_node() + rclpy.shutdown() + +if __name__ == '__main__': + main() diff --git a/src/image_object_detection/templates/index.html b/src/image_object_detection/templates/index.html new file mode 100644 index 0000000..b527d87 --- /dev/null +++ b/src/image_object_detection/templates/index.html @@ -0,0 +1,69 @@ + + + + Object Detection Parameters + + + +
+
+

Detection Classes

+
+ {% for class_name in available_classes %} +
+ + +
+ {% endfor %} +
+
+ +
+

Detection Parameters

+
+ + + {{ confidence }} +
+
+ + + {{ iou_threshold }} +
+
+ + +
+ + + + diff --git a/src/image_object_detection/web_interface_node.py b/src/image_object_detection/web_interface_node.py new file mode 100644 index 0000000..6e11a98 --- /dev/null +++ b/src/image_object_detection/web_interface_node.py @@ -0,0 +1,57 @@ +import rclpy +from rclpy.node import Node +from flask import Flask, render_template, request, jsonify +import threading +from rclpy.parameter import Parameter + +class WebInterfaceNode(Node): + def __init__(self, detection_node): + super().__init__('web_interface_node') + self.detection_node = detection_node + self.app = Flask(__name__) + self.setup_routes() + + # Start Flask in a separate thread + self.flask_thread = threading.Thread(target=self.run_flask) + self.flask_thread.daemon = True + self.flask_thread.start() + + def setup_routes(self): + @self.app.route('/') + def index(): + available_classes = self.detection_node.names + selected_classes = self.detection_node.selected_detections + current_confidence = self.detection_node.confidence + current_iou = self.detection_node.iou_threshold + + return render_template('index.html', + available_classes=available_classes, + selected_classes=selected_classes, + confidence=current_confidence, + iou_threshold=current_iou) + + @self.app.route('/update_params', methods=['POST']) + def update_params(): + if request.method == 'POST': + # Handle selected detections + selected_classes = request.form.getlist('classes[]') + self.detection_node.set_parameters([ + Parameter('selected_detections', value=selected_classes) + ]) + + # Handle confidence threshold + confidence = float(request.form.get('confidence', 0.25)) + self.detection_node.set_parameters([ + Parameter('model.confidence', value=confidence) + ]) + + # Handle IoU threshold + iou_threshold = float(request.form.get('iou_threshold', 0.45)) + self.detection_node.set_parameters([ + Parameter('model.iou_threshold', value=iou_threshold) + ]) + + return jsonify({'status': 'success'}) + + def run_flask(self): + self.app.run(host='0.0.0.0', port=5005) From c28c3410843239d1a4983160f8277585898aad31 Mon Sep 17 00:00:00 2001 From: pabloinigoblasco Date: Thu, 28 Nov 2024 23:53:50 +0100 Subject: [PATCH 06/12] progress --- .../image_object_detection_node.py | 93 ++++++++++++------- .../templates/index.html | 45 ++++++++- .../web_interface_node.py | 36 ++++--- 3 files changed, 128 insertions(+), 46 deletions(-) diff --git a/src/image_object_detection/image_object_detection_node.py b/src/image_object_detection/image_object_detection_node.py index 2e31c86..b30175d 100644 --- a/src/image_object_detection/image_object_detection_node.py +++ b/src/image_object_detection/image_object_detection_node.py @@ -137,19 +137,75 @@ def __init__(self): self.bridge = cv_bridge.CvBridge() - # Get the list of camera topics from the config file - self.declare_parameter("camera_topics", ["/cameras/frontleft_fisheye_image/image", "/cameras/frontright_fisheye_image/image", "/cameras/left_fisheye_image/image", "/cameras/right_fisheye_image/image"]) + # Initialize camera topics parameter + self.declare_parameter("camera_topics", [ + "/cameras/frontleft_fisheye_image/image", + "/cameras/frontright_fisheye_image/image", + "/cameras/left_fisheye_image/image", + "/cameras/right_fisheye_image/image" + ]) self.camera_topics = self.get_parameter("camera_topics").get_parameter_value().string_array_value - self.get_logger().info(f"Subscribed to topics: {self.camera_topics}") - # Initialize subscribers and publishers for each camera topic + # Initialize empty containers for subscribers and publishers + self.subscribers = [] + self.detection_publishers = {} + self.debug_image_publishers = {} + + # Set up camera topics using the extracted method + self.setup_camera_topics() + + self.initialize_model() + + def parameters_callback(self, params): + result = SetParametersResult(successful=True) + + init_model = False + for param in params: + if param.name == 'camera_topics': + self.camera_topics = param.value + self.get_logger().info(f"Updated camera_topics: {self.camera_topics}") + # Recreate subscribers and publishers for new topics + self.setup_camera_topics() + + elif param.name == 'selected_detections': + self.selected_detections = param.value + self.get_logger().info(f"Updated selected_detections: {self.selected_detections}") + + elif param.name == 'model.iou_threshold': + self.iou_threshold = param.value + self.get_logger().info(f"Updated iou_threshold: {self.iou_threshold}") + + elif param.name == 'model.confidence': + self.confidence = param.value + self.get_logger().info(f"Updated confidence: {self.confidence}") + + elif param.name == 'model.weights_file': + self.model_weights_file = param.value + self.get_logger().info(f"Updated weights_file: {self.model_weights_file}") + init_model = True + + elif param.name == 'model.publish_debug_image': + self.enable_publish_debug_image = param.value + self.get_logger().info(f"Updated publish_debug_image: {self.enable_publish_debug_image}") + self.setup_camera_topics() # Recreate publishers with new debug setting + + elif param.name == 'model.image_size': + self.model_image_size = param.value + self.get_logger().info(f"Updated image_size: {self.model_image_size}") + init_model = True + + if init_model: + self.initialize_model() + + return result + def setup_camera_topics(self): + # Clear existing subscribers and publishers self.subscribers = [] self.detection_publishers = {} self.debug_image_publishers = {} for topic in self.camera_topics: - # Create a subscriber for each camera topic self.subscribers.append( self.create_subscription( Image, @@ -159,44 +215,17 @@ def __init__(self): ) ) - # Create a detection publisher for each camera detection_topic = f"{topic}/detections" self.detection_publishers[topic] = self.create_publisher( Detection2DArray, detection_topic, self.qos ) - # Create a debug image publisher for each camera (if enabled) if self.enable_publish_debug_image: debug_image_topic = f"{topic}/debug_image" self.debug_image_publishers[topic] = self.create_publisher( Image, debug_image_topic, self.qos ) - self.initialize_model() - def parameters_callback(self, params): - result = SetParametersResult(successful=True) - - for param in params: - if param.name == 'selected_detections': - self.selected_detections = param.value - self.get_logger().info(f"Updated selected_detections: {self.selected_detections}") - - elif param.name == 'model.iou_threshold': - self.iou_threshold = param.value - self.get_logger().info(f"Updated iou_threshold: {self.iou_threshold}") - - elif param.name == 'model.confidence': - self.confidence = param.value - self.get_logger().info(f"Updated confidence: {self.confidence}") - - elif param.name == 'model.weights_file': - self.model_weights_file = param.value - self.get_logger().info(f"Updated weights_file: {self.model_weights_file}") - # Reinitialize model with new weights - self.initialize_model() - - return result - def initialize_model(self): with torch.no_grad(): set_logging() diff --git a/src/image_object_detection/templates/index.html b/src/image_object_detection/templates/index.html index b527d87..66c719f 100644 --- a/src/image_object_detection/templates/index.html +++ b/src/image_object_detection/templates/index.html @@ -14,6 +14,26 @@ border: 1px solid #ddd; border-radius: 5px; } + .video-grid { + display: grid; + grid-template-columns: repeat(2, 1fr); + gap: 20px; + margin: 20px; + } + .video-container { + border: 1px solid #ddd; + padding: 10px; + border-radius: 5px; + } + .video-container img { + width: 100%; + height: auto; + } + .video-title { + text-align: center; + margin-bottom: 10px; + font-weight: bold; + } @@ -47,8 +67,27 @@

Detection Parameters

- - + +
+

Camera Topics

+
+ + Enter one topic per line +
+
+ + + + +
+ {% for topic in camera_topics %} +
+
{{ topic }}
+ {{ topic }} stream +
+ {% endfor %} +
- + \ No newline at end of file diff --git a/src/image_object_detection/web_interface_node.py b/src/image_object_detection/web_interface_node.py index 6e11a98..81599a5 100644 --- a/src/image_object_detection/web_interface_node.py +++ b/src/image_object_detection/web_interface_node.py @@ -11,7 +11,6 @@ def __init__(self, detection_node): self.app = Flask(__name__) self.setup_routes() - # Start Flask in a separate thread self.flask_thread = threading.Thread(target=self.run_flask) self.flask_thread.daemon = True self.flask_thread.start() @@ -19,20 +18,25 @@ def __init__(self, detection_node): def setup_routes(self): @self.app.route('/') def index(): - available_classes = self.detection_node.names - selected_classes = self.detection_node.selected_detections - current_confidence = self.detection_node.confidence - current_iou = self.detection_node.iou_threshold - return render_template('index.html', - available_classes=available_classes, - selected_classes=selected_classes, - confidence=current_confidence, - iou_threshold=current_iou) + available_classes=self.detection_node.names, + selected_classes=self.detection_node.selected_detections, + confidence=self.detection_node.confidence, + iou_threshold=self.detection_node.iou_threshold, + camera_topics=self.detection_node.camera_topics, + publish_debug_image=self.detection_node.enable_publish_debug_image, + image_size=self.detection_node.model_image_size) @self.app.route('/update_params', methods=['POST']) def update_params(): if request.method == 'POST': + # Handle camera topics + camera_topics = request.form.get('camera_topics', '').split('\n') + camera_topics = [topic.strip() for topic in camera_topics if topic.strip()] + self.detection_node.set_parameters([ + Parameter('camera_topics', value=camera_topics) + ]) + # Handle selected detections selected_classes = request.form.getlist('classes[]') self.detection_node.set_parameters([ @@ -51,7 +55,17 @@ def update_params(): Parameter('model.iou_threshold', value=iou_threshold) ]) - return jsonify({'status': 'success'}) + # Add to existing parameter updates + publish_debug = request.form.get('publish_debug_image') == 'true' + self.detection_node.set_parameters([ + Parameter('model.publish_debug_image', value=publish_debug) + ]) + image_size = int(request.form.get('image_size', 640)) + self.detection_node.set_parameters([ + Parameter('model.image_size', value=image_size) + ]) + + return jsonify({'status': 'success'}) def run_flask(self): self.app.run(host='0.0.0.0', port=5005) From 7c3f57e073a23bf3e0f0561b69a61585ef4ffc41 Mon Sep 17 00:00:00 2001 From: pabloinigoblasco Date: Sun, 1 Dec 2024 19:29:26 +0100 Subject: [PATCH 07/12] minor --- .../templates/index.html | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/image_object_detection/templates/index.html b/src/image_object_detection/templates/index.html index 66c719f..c14c5d4 100644 --- a/src/image_object_detection/templates/index.html +++ b/src/image_object_detection/templates/index.html @@ -38,6 +38,25 @@
+
+

Model Parameters

+
+ +
+
+ + +
+

Detection Classes

@@ -105,4 +124,6 @@

Camera Topics

}; - \ No newline at end of file + + + From 117e3391c4b76febdef3726fd6f3ca79c9c7198c Mon Sep 17 00:00:00 2001 From: atobaruela-ibrobotics Date: Mon, 2 Dec 2024 01:57:25 +0100 Subject: [PATCH 08/12] minor --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 383335b..a40bf3d 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,7 @@ "console_scripts": [ "image_object_detection_node = image_object_detection.image_object_detection_node:main", "test_publisher_node = image_object_detection.test_publisher:main", + "web_interface_node = image_object_detection.web_interface_node:main", ], }, ) From 0ce4ebe929a26f84133d7cf7fc7ad9a598b2ea7e Mon Sep 17 00:00:00 2001 From: atobaruela-ibrobotics Date: Mon, 2 Dec 2024 01:58:30 +0100 Subject: [PATCH 09/12] Revert "minor" This reverts commit 117e3391c4b76febdef3726fd6f3ca79c9c7198c. --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index a40bf3d..383335b 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,6 @@ "console_scripts": [ "image_object_detection_node = image_object_detection.image_object_detection_node:main", "test_publisher_node = image_object_detection.test_publisher:main", - "web_interface_node = image_object_detection.web_interface_node:main", ], }, ) From 81f3ff8497974cde020a8cdfd462de28e07b96f1 Mon Sep 17 00:00:00 2001 From: atobaruela-ibrobotics Date: Mon, 2 Dec 2024 02:41:32 +0100 Subject: [PATCH 10/12] fixes --- setup.py | 1 + src/image_object_detection/web_interface_node.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 383335b..36178fb 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,7 @@ ("share/" + package_name, ["yolov7-tiny.pt"]), ("share/" + package_name + "/launch", ["launch/image_object_detection_launch.py"]), ("share/" + package_name + "/config", ["config/image_object_detection.yaml"]), + ("share/" + package_name + "/templates", ["src/image_object_detection/templates/index.html"]), ], install_requires=["setuptools"], zip_safe=True, diff --git a/src/image_object_detection/web_interface_node.py b/src/image_object_detection/web_interface_node.py index 81599a5..6339a8a 100644 --- a/src/image_object_detection/web_interface_node.py +++ b/src/image_object_detection/web_interface_node.py @@ -3,12 +3,14 @@ from flask import Flask, render_template, request, jsonify import threading from rclpy.parameter import Parameter +import os +from ament_index_python.packages import get_package_share_directory class WebInterfaceNode(Node): def __init__(self, detection_node): super().__init__('web_interface_node') self.detection_node = detection_node - self.app = Flask(__name__) + self.app = Flask(__name__, template_folder=os.path.join(get_package_share_directory("image_object_detection"), 'templates')) self.setup_routes() self.flask_thread = threading.Thread(target=self.run_flask) From 908814f37d895d4fcc06f0f5d18e98748750331e Mon Sep 17 00:00:00 2001 From: atobaruela-ibrobotics Date: Tue, 3 Dec 2024 16:50:55 +0100 Subject: [PATCH 11/12] fixing multiple publishers --- .../image_object_detection_node.py | 68 +++++++++++++------ .../templates/index.html | 4 +- .../web_interface_node.py | 10 ++- 3 files changed, 58 insertions(+), 24 deletions(-) diff --git a/src/image_object_detection/image_object_detection_node.py b/src/image_object_detection/image_object_detection_node.py index b30175d..8bd337b 100644 --- a/src/image_object_detection/image_object_detection_node.py +++ b/src/image_object_detection/image_object_detection_node.py @@ -102,8 +102,6 @@ def __init__(self): ) - # Add the parameter callback handler - self.add_on_set_parameters_callback(self.parameters_callback) self.declare_parameter("subscribers.qos_policy", "best_effort") self.subscribers_qos = ( @@ -157,6 +155,9 @@ def __init__(self): self.initialize_model() + # Add the parameter callback handler + self.add_on_set_parameters_callback(self.parameters_callback) + def parameters_callback(self, params): result = SetParametersResult(successful=True) @@ -200,27 +201,52 @@ def parameters_callback(self, params): return result def setup_camera_topics(self): - # Clear existing subscribers and publishers - self.subscribers = [] - self.detection_publishers = {} - self.debug_image_publishers = {} - - for topic in self.camera_topics: - self.subscribers.append( - self.create_subscription( - Image, - topic, - callback=self.image_callback_factory(topic), - qos_profile=self.qos, + # Create sets of existing topics + existing_sub_topics = {sub.topic_name for sub in self.subscribers} + existing_pub_topics = set(self.detection_publishers.keys()) + existing_debug_topics = set(self.debug_image_publishers.keys()) + + # Set of new topics + new_topics = set(self.camera_topics) + + # Remove subscribers for topics that no longer exist + for sub in list(self.subscribers): + if sub.topic_name not in new_topics: + sub.destroy() + self.subscribers.remove(sub) + + # Remove publishers for topics that no longer exist + for topic in list(self.detection_publishers.keys()): + if topic not in new_topics: + self.detection_publishers[topic].destroy() + self.detection_publishers[topic].destroy() + del self.detection_publishers[topic] + + # Remove debug image publishers for topics that no longer exist + for topic in list(self.debug_image_publishers.keys()): + if topic not in new_topics: + self.debug_image_publishers[topic].destroy() + del self.debug_image_publishers[topic] + + # Add new topics + for topic in new_topics: + if topic not in existing_sub_topics: + self.subscribers.append( + self.create_subscription( + Image, + topic, + callback=self.image_callback_factory(topic), + qos_profile=self.qos, + ) ) - ) - detection_topic = f"{topic}/detections" - self.detection_publishers[topic] = self.create_publisher( - Detection2DArray, detection_topic, self.qos - ) + if topic not in existing_pub_topics: + detection_topic = f"{topic}/detections" + self.detection_publishers[topic] = self.create_publisher( + Detection2DArray, detection_topic, self.qos + ) - if self.enable_publish_debug_image: + if self.enable_publish_debug_image and topic not in existing_debug_topics: debug_image_topic = f"{topic}/debug_image" self.debug_image_publishers[topic] = self.create_publisher( Image, debug_image_topic, self.qos @@ -229,7 +255,7 @@ def setup_camera_topics(self): def initialize_model(self): with torch.no_grad(): set_logging() - self.device = select_device(self.device) + self.device = select_device(str(self.device)) self.half = self.device.type != "cpu" self.model = attempt_load( diff --git a/src/image_object_detection/templates/index.html b/src/image_object_detection/templates/index.html index c14c5d4..52989d0 100644 --- a/src/image_object_detection/templates/index.html +++ b/src/image_object_detection/templates/index.html @@ -100,10 +100,10 @@

Camera Topics

- {% for topic in camera_topics %} + {% for topic in camera_debug_image_topics %}
{{ topic }}
- {{ topic }} stream + {{ topic }} stream
{% endfor %}
diff --git a/src/image_object_detection/web_interface_node.py b/src/image_object_detection/web_interface_node.py index 6339a8a..42460cb 100644 --- a/src/image_object_detection/web_interface_node.py +++ b/src/image_object_detection/web_interface_node.py @@ -20,6 +20,13 @@ def __init__(self, detection_node): def setup_routes(self): @self.app.route('/') def index(): + + camera_debug_image_topics = [] + for camera in self.detection_node.camera_topics: + camera_debug_image_topics.append(camera.replace("/image,", "/debug_image")) + + print(camera_debug_image_topics) + return render_template('index.html', available_classes=self.detection_node.names, selected_classes=self.detection_node.selected_detections, @@ -27,7 +34,8 @@ def index(): iou_threshold=self.detection_node.iou_threshold, camera_topics=self.detection_node.camera_topics, publish_debug_image=self.detection_node.enable_publish_debug_image, - image_size=self.detection_node.model_image_size) + image_size=self.detection_node.model_image_size, + camera_debug_image_topics= [topic.replace("/image", "/image/debug_image") for topic in self.detection_node.camera_topics]) @self.app.route('/update_params', methods=['POST']) def update_params(): From b95c3339f889bfe58165ff042b1e7a286003aaac Mon Sep 17 00:00:00 2001 From: atobaruela-ibrobotics Date: Wed, 4 Dec 2024 11:47:12 +0100 Subject: [PATCH 12/12] Forced video-container ip to .70.5 --- src/image_object_detection/templates/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/image_object_detection/templates/index.html b/src/image_object_detection/templates/index.html index 52989d0..969c8fa 100644 --- a/src/image_object_detection/templates/index.html +++ b/src/image_object_detection/templates/index.html @@ -103,7 +103,7 @@

Camera Topics

{% for topic in camera_debug_image_topics %}
{{ topic }}
- {{ topic }} stream + {{ topic }} stream
{% endfor %}