refactor: replace threading with multiprocessing

core/streaming/manager.py

@@ -1,14 +1,38 @@
 """
 Stream coordination and lifecycle management.
 Optimized for 1280x720@6fps RTSP and 2560x1440 HTTP snapshots.
+Supports both threading and multiprocessing modes for scalability.
 """
 import logging
 import threading
 import time
+import os
 from typing import Dict, Set, Optional, List, Any
 from dataclasses import dataclass
 from collections import defaultdict
 
+# Check if multiprocessing is enabled (default enabled with proper initialization)
+USE_MULTIPROCESSING = os.environ.get('USE_MULTIPROCESSING', 'true').lower() == 'true'
+
+logger = logging.getLogger(__name__)
+
+if USE_MULTIPROCESSING:
+    try:
+        from .process_manager import RTSPProcessManager, ProcessConfig
+        logger.info("Multiprocessing support enabled")
+        _mp_loaded = True
+    except ImportError as e:
+        logger.warning(f"Failed to load multiprocessing support: {e}")
+        USE_MULTIPROCESSING = False
+        _mp_loaded = False
+    except Exception as e:
+        logger.warning(f"Multiprocessing initialization failed: {e}")
+        USE_MULTIPROCESSING = False
+        _mp_loaded = False
+else:
+    logger.info("Multiprocessing support disabled (using threading mode)")
+    _mp_loaded = False
+
 from .readers import RTSPReader, HTTPSnapshotReader
 from .buffers import shared_cache_buffer, StreamType
 from ..tracking.integration import TrackingPipelineIntegration
@@ -50,6 +74,42 @@ class StreamManager:
         self._camera_subscribers: Dict[str, Set[str]] = defaultdict(set)  # camera_id -> set of subscription_ids
         self._lock = threading.RLock()
 
+        # Initialize multiprocessing manager if enabled (lazy initialization)
+        self.process_manager = None
+        self._frame_getter_thread = None
+        self._multiprocessing_enabled = USE_MULTIPROCESSING and _mp_loaded
+
+        if self._multiprocessing_enabled:
+            logger.info(f"Multiprocessing support enabled, will initialize on first use")
+        else:
+            logger.info(f"Multiprocessing support disabled, using threading mode")
+
+    def _initialize_multiprocessing(self) -> bool:
+        """Lazily initialize multiprocessing manager when first needed."""
+        if self.process_manager is not None:
+            return True
+
+        if not self._multiprocessing_enabled:
+            return False
+
+        try:
+            self.process_manager = RTSPProcessManager(max_processes=min(self.max_streams, 15))
+            # Start monitoring synchronously to ensure it's ready
+            self.process_manager.start_monitoring()
+            # Start frame getter thread
+            self._frame_getter_thread = threading.Thread(
+                target=self._multiprocess_frame_getter,
+                daemon=True
+            )
+            self._frame_getter_thread.start()
+            logger.info(f"Initialized multiprocessing manager with max {self.process_manager.max_processes} processes")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to initialize multiprocessing manager: {e}")
+            self.process_manager = None
+            self._multiprocessing_enabled = False  # Disable for future attempts
+            return False
+
     def add_subscription(self, subscription_id: str, stream_config: StreamConfig,
                         crop_coords: Optional[tuple] = None,
                         model_id: Optional[str] = None,
@@ -129,7 +189,24 @@ class StreamManager:
         """Start a stream for the given camera."""
         try:
             if stream_config.rtsp_url:
-                # RTSP stream
+                # Try multiprocessing for RTSP if enabled
+                if self._multiprocessing_enabled and self._initialize_multiprocessing():
+                    config = ProcessConfig(
+                        camera_id=camera_id,
+                        rtsp_url=stream_config.rtsp_url,
+                        expected_fps=6,
+                        buffer_size=3,
+                        max_retries=stream_config.max_retries
+                    )
+                    success = self.process_manager.add_camera(config)
+                    if success:
+                        self._streams[camera_id] = 'multiprocessing'  # Mark as multiprocessing stream
+                        logger.info(f"Started RTSP multiprocessing stream for camera {camera_id}")
+                        return True
+                    else:
+                        logger.warning(f"Failed to start multiprocessing stream for {camera_id}, falling back to threading")
+
+                # Fall back to threading mode for RTSP
                 reader = RTSPReader(
                     camera_id=camera_id,
                     rtsp_url=stream_config.rtsp_url,
@@ -138,10 +215,10 @@ class StreamManager:
                 reader.set_frame_callback(self._frame_callback)
                 reader.start()
                 self._streams[camera_id] = reader
-                logger.info(f"Started RTSP stream for camera {camera_id}")
+                logger.info(f"Started RTSP threading stream for camera {camera_id}")
 
             elif stream_config.snapshot_url:
-                # HTTP snapshot stream
+                # HTTP snapshot stream (always use threading)
                 reader = HTTPSnapshotReader(
                     camera_id=camera_id,
                     snapshot_url=stream_config.snapshot_url,
@@ -167,10 +244,18 @@ class StreamManager:
         """Stop a stream for the given camera."""
         if camera_id in self._streams:
             try:
-                self._streams[camera_id].stop()
+                stream_obj = self._streams[camera_id]
+                if stream_obj == 'multiprocessing' and self.process_manager:
+                    # Remove from multiprocessing manager
+                    self.process_manager.remove_camera(camera_id)
+                    logger.info(f"Stopped multiprocessing stream for camera {camera_id}")
+                else:
+                    # Stop threading stream
+                    stream_obj.stop()
+                    logger.info(f"Stopped threading stream for camera {camera_id}")
+
                 del self._streams[camera_id]
                 shared_cache_buffer.clear_camera(camera_id)
-                logger.info(f"Stopped stream for camera {camera_id}")
             except Exception as e:
                 logger.error(f"Error stopping stream for camera {camera_id}: {e}")
 
@@ -190,6 +275,38 @@ class StreamManager:
         except Exception as e:
             logger.error(f"Error in frame callback for camera {camera_id}: {e}")
 
+    def _multiprocess_frame_getter(self):
+        """Background thread to get frames from multiprocessing manager."""
+        if not self.process_manager:
+            return
+
+        logger.info("Started multiprocessing frame getter thread")
+
+        while self.process_manager:
+            try:
+                # Get frames from all multiprocessing cameras
+                with self._lock:
+                    mp_cameras = [cid for cid, s in self._streams.items() if s == 'multiprocessing']
+
+                for camera_id in mp_cameras:
+                    try:
+                        result = self.process_manager.get_frame(camera_id)
+                        if result:
+                            frame, timestamp = result
+                            # Detect stream type and store in cache
+                            stream_type = self._detect_stream_type(frame)
+                            shared_cache_buffer.put_frame(camera_id, frame, stream_type)
+                            # Process tracking
+                            self._process_tracking_for_camera(camera_id, frame)
+                    except Exception as e:
+                        logger.debug(f"Error getting frame for {camera_id}: {e}")
+
+                time.sleep(0.05)  # 20 FPS polling rate
+
+            except Exception as e:
+                logger.error(f"Error in multiprocess frame getter: {e}")
+                time.sleep(1.0)
+
     def _process_tracking_for_camera(self, camera_id: str, frame):
         """Process tracking for all subscriptions of a camera."""
         try:
@@ -362,6 +479,12 @@ class StreamManager:
             for camera_id in list(self._streams.keys()):
                 self._stop_stream(camera_id)
 
+            # Stop multiprocessing manager if exists
+            if self.process_manager:
+                self.process_manager.stop_all()
+                self.process_manager = None
+                logger.info("Stopped multiprocessing manager")
+
             # Clear all tracking
             self._subscriptions.clear()
             self._camera_subscribers.clear()
@@ -434,9 +557,12 @@ class StreamManager:
             # Add stream type information
             stream_types = {}
             for camera_id in self._streams.keys():
-                if isinstance(self._streams[camera_id], RTSPReader):
-                    stream_types[camera_id] = 'rtsp'
-                elif isinstance(self._streams[camera_id], HTTPSnapshotReader):
+                stream_obj = self._streams[camera_id]
+                if stream_obj == 'multiprocessing':
+                    stream_types[camera_id] = 'rtsp_multiprocessing'
+                elif isinstance(stream_obj, RTSPReader):
+                    stream_types[camera_id] = 'rtsp_threading'
+                elif isinstance(stream_obj, HTTPSnapshotReader):
                     stream_types[camera_id] = 'http'
                 else:
                     stream_types[camera_id] = 'unknown'

core/streaming/process_manager.py

@@ -0,0 +1,453 @@
+"""
+Multiprocessing-based RTSP stream management for scalability.
+Handles multiple camera streams using separate processes to bypass GIL limitations.
+"""
+
+import multiprocessing as mp
+import time
+import logging
+import cv2
+import numpy as np
+import queue
+import threading
+import os
+import psutil
+from typing import Dict, Optional, Tuple, Any, Callable
+from dataclasses import dataclass
+from multiprocessing import Process, Queue, Lock, Value, Array, Manager
+from multiprocessing.shared_memory import SharedMemory
+import signal
+import sys
+
+# Ensure proper multiprocessing context for uvicorn compatibility
+try:
+    mp.set_start_method('spawn', force=True)
+except RuntimeError:
+    pass  # Already set
+
+logger = logging.getLogger("detector_worker.process_manager")
+
+# Frame dimensions (1280x720 RGB)
+FRAME_WIDTH = 1280
+FRAME_HEIGHT = 720
+FRAME_CHANNELS = 3
+FRAME_SIZE = FRAME_WIDTH * FRAME_HEIGHT * FRAME_CHANNELS
+
+@dataclass
+class ProcessConfig:
+    """Configuration for camera process."""
+    camera_id: str
+    rtsp_url: str
+    expected_fps: int = 6
+    buffer_size: int = 3
+    max_retries: int = 30
+    reconnect_delay: float = 5.0
+
+
+class SharedFrameBuffer:
+    """Thread-safe shared memory frame buffer with double buffering."""
+
+    def __init__(self, camera_id: str):
+        self.camera_id = camera_id
+        self.lock = mp.Lock()
+
+        # Double buffering for lock-free reads
+        self.buffer_a = mp.Array('B', FRAME_SIZE, lock=False)
+        self.buffer_b = mp.Array('B', FRAME_SIZE, lock=False)
+
+        # Atomic index for current read buffer (0 or 1)
+        self.read_buffer_idx = mp.Value('i', 0)
+
+        # Frame metadata (atomic access)
+        self.timestamp = mp.Value('d', 0.0)
+        self.frame_number = mp.Value('L', 0)
+        self.is_valid = mp.Value('b', False)
+
+        # Statistics
+        self.frames_written = mp.Value('L', 0)
+        self.frames_dropped = mp.Value('L', 0)
+
+    def write_frame(self, frame: np.ndarray, timestamp: float) -> bool:
+        """Write frame to buffer with atomic swap."""
+        if frame is None or frame.size == 0:
+            return False
+
+        # Resize if needed
+        if frame.shape != (FRAME_HEIGHT, FRAME_WIDTH, FRAME_CHANNELS):
+            frame = cv2.resize(frame, (FRAME_WIDTH, FRAME_HEIGHT))
+
+        # Get write buffer (opposite of read buffer)
+        write_idx = 1 - self.read_buffer_idx.value
+        write_buffer = self.buffer_a if write_idx == 0 else self.buffer_b
+
+        try:
+            # Write to buffer without lock (safe because of double buffering)
+            frame_flat = frame.flatten()
+            write_buffer[:] = frame_flat.astype(np.uint8)
+
+            # Update metadata
+            self.timestamp.value = timestamp
+            self.frame_number.value += 1
+
+            # Atomic swap of buffers
+            with self.lock:
+                self.read_buffer_idx.value = write_idx
+                self.is_valid.value = True
+                self.frames_written.value += 1
+
+            return True
+
+        except Exception as e:
+            logger.error(f"Error writing frame for {self.camera_id}: {e}")
+            self.frames_dropped.value += 1
+            return False
+
+    def read_frame(self) -> Optional[Tuple[np.ndarray, float]]:
+        """Read frame from buffer without blocking writers."""
+        if not self.is_valid.value:
+            return None
+
+        # Get current read buffer index (atomic read)
+        read_idx = self.read_buffer_idx.value
+        read_buffer = self.buffer_a if read_idx == 0 else self.buffer_b
+
+        # Read timestamp (atomic)
+        timestamp = self.timestamp.value
+
+        # Copy frame data (no lock needed for read)
+        try:
+            frame_data = np.array(read_buffer, dtype=np.uint8)
+            frame = frame_data.reshape((FRAME_HEIGHT, FRAME_WIDTH, FRAME_CHANNELS))
+            return frame.copy(), timestamp
+        except Exception as e:
+            logger.error(f"Error reading frame for {self.camera_id}: {e}")
+            return None
+
+    def get_stats(self) -> Dict[str, int]:
+        """Get buffer statistics."""
+        return {
+            'frames_written': self.frames_written.value,
+            'frames_dropped': self.frames_dropped.value,
+            'frame_number': self.frame_number.value,
+            'is_valid': self.is_valid.value
+        }
+
+
+def camera_worker_process(
+    config: ProcessConfig,
+    frame_buffer: SharedFrameBuffer,
+    command_queue: Queue,
+    status_queue: Queue,
+    stop_event: mp.Event
+):
+    """
+    Worker process for individual camera stream.
+    Runs in separate process to bypass GIL.
+    """
+    # Set process name for debugging
+    mp.current_process().name = f"Camera-{config.camera_id}"
+
+    # Configure logging for subprocess
+    logging.basicConfig(
+        level=logging.INFO,
+        format=f'%(asctime)s [%(levelname)s] Camera-{config.camera_id}: %(message)s'
+    )
+
+    logger.info(f"Starting camera worker for {config.camera_id}")
+
+    cap = None
+    consecutive_errors = 0
+    frame_interval = 1.0 / config.expected_fps
+    last_frame_time = 0
+
+    def initialize_capture():
+        """Initialize OpenCV capture with optimized settings."""
+        nonlocal cap
+
+        try:
+            # Set RTSP transport to TCP for reliability
+            os.environ['OPENCV_FFMPEG_CAPTURE_OPTIONS'] = 'rtsp_transport;tcp'
+
+            # Create capture
+            cap = cv2.VideoCapture(config.rtsp_url, cv2.CAP_FFMPEG)
+
+            if not cap.isOpened():
+                logger.error(f"Failed to open RTSP stream")
+                return False
+
+            # Set capture properties
+            cap.set(cv2.CAP_PROP_FRAME_WIDTH, FRAME_WIDTH)
+            cap.set(cv2.CAP_PROP_FRAME_HEIGHT, FRAME_HEIGHT)
+            cap.set(cv2.CAP_PROP_FPS, config.expected_fps)
+            cap.set(cv2.CAP_PROP_BUFFERSIZE, config.buffer_size)
+
+            # Read initial frames to stabilize
+            for _ in range(3):
+                ret, _ = cap.read()
+                if not ret:
+                    logger.warning("Failed to read initial frames")
+                time.sleep(0.1)
+
+            logger.info(f"Successfully initialized capture")
+            return True
+
+        except Exception as e:
+            logger.error(f"Error initializing capture: {e}")
+            return False
+
+    # Main processing loop
+    while not stop_event.is_set():
+        try:
+            # Check for commands (non-blocking)
+            try:
+                command = command_queue.get_nowait()
+                if command == "reinit":
+                    logger.info("Received reinit command")
+                    if cap:
+                        cap.release()
+                        cap = None
+                    consecutive_errors = 0
+            except queue.Empty:
+                pass
+
+            # Initialize capture if needed
+            if cap is None or not cap.isOpened():
+                if not initialize_capture():
+                    time.sleep(config.reconnect_delay)
+                    consecutive_errors += 1
+                    if consecutive_errors > config.max_retries and config.max_retries > 0:
+                        logger.error("Max retries reached, exiting")
+                        break
+                    continue
+                else:
+                    consecutive_errors = 0
+
+            # Read frame with timing control
+            current_time = time.time()
+            if current_time - last_frame_time < frame_interval:
+                time.sleep(0.01)  # Small sleep to prevent busy waiting
+                continue
+
+            ret, frame = cap.read()
+
+            if not ret or frame is None:
+                consecutive_errors += 1
+
+                if consecutive_errors >= config.max_retries:
+                    logger.error(f"Too many consecutive errors ({consecutive_errors}), reinitializing")
+                    if cap:
+                        cap.release()
+                        cap = None
+                    consecutive_errors = 0
+                    time.sleep(config.reconnect_delay)
+                else:
+                    if consecutive_errors <= 5:
+                        logger.debug(f"Frame read failed (error {consecutive_errors})")
+                    elif consecutive_errors % 10 == 0:
+                        logger.warning(f"Continuing frame failures (error {consecutive_errors})")
+
+                    # Exponential backoff
+                    sleep_time = min(0.1 * (1.5 ** min(consecutive_errors, 10)), 1.0)
+                    time.sleep(sleep_time)
+                continue
+
+            # Frame read successful
+            consecutive_errors = 0
+            last_frame_time = current_time
+
+            # Write to shared buffer
+            if frame_buffer.write_frame(frame, current_time):
+                # Send status update periodically
+                if frame_buffer.frame_number.value % 30 == 0:  # Every 30 frames
+                    status_queue.put({
+                        'camera_id': config.camera_id,
+                        'status': 'running',
+                        'frames': frame_buffer.frame_number.value,
+                        'timestamp': current_time
+                    })
+
+        except KeyboardInterrupt:
+            logger.info("Received interrupt signal")
+            break
+        except Exception as e:
+            logger.error(f"Error in camera worker: {e}")
+            consecutive_errors += 1
+            time.sleep(1.0)
+
+    # Cleanup
+    if cap:
+        cap.release()
+
+    logger.info(f"Camera worker stopped")
+    status_queue.put({
+        'camera_id': config.camera_id,
+        'status': 'stopped',
+        'frames': frame_buffer.frame_number.value
+    })
+
+
+class RTSPProcessManager:
+    """
+    Manages multiple camera processes with health monitoring and auto-restart.
+    """
+
+    def __init__(self, max_processes: int = None):
+        self.max_processes = max_processes or (mp.cpu_count() - 2)
+        self.processes: Dict[str, Process] = {}
+        self.frame_buffers: Dict[str, SharedFrameBuffer] = {}
+        self.command_queues: Dict[str, Queue] = {}
+        self.status_queue = mp.Queue()
+        self.stop_events: Dict[str, mp.Event] = {}
+        self.configs: Dict[str, ProcessConfig] = {}
+
+        # Manager for shared objects
+        self.manager = Manager()
+        self.process_stats = self.manager.dict()
+
+        # Health monitoring thread
+        self.monitor_thread = None
+        self.monitor_stop = threading.Event()
+
+        logger.info(f"RTSPProcessManager initialized with max_processes={self.max_processes}")
+
+    def add_camera(self, config: ProcessConfig) -> bool:
+        """Add a new camera stream."""
+        if config.camera_id in self.processes:
+            logger.warning(f"Camera {config.camera_id} already exists")
+            return False
+
+        if len(self.processes) >= self.max_processes:
+            logger.error(f"Max processes ({self.max_processes}) reached")
+            return False
+
+        try:
+            # Create shared resources
+            frame_buffer = SharedFrameBuffer(config.camera_id)
+            command_queue = mp.Queue()
+            stop_event = mp.Event()
+
+            # Store resources
+            self.frame_buffers[config.camera_id] = frame_buffer
+            self.command_queues[config.camera_id] = command_queue
+            self.stop_events[config.camera_id] = stop_event
+            self.configs[config.camera_id] = config
+
+            # Start process
+            process = mp.Process(
+                target=camera_worker_process,
+                args=(config, frame_buffer, command_queue, self.status_queue, stop_event),
+                name=f"Camera-{config.camera_id}"
+            )
+            process.start()
+            self.processes[config.camera_id] = process
+
+            logger.info(f"Started process for camera {config.camera_id} (PID: {process.pid})")
+            return True
+
+        except Exception as e:
+            logger.error(f"Error adding camera {config.camera_id}: {e}")
+            self._cleanup_camera(config.camera_id)
+            return False
+
+    def remove_camera(self, camera_id: str) -> bool:
+        """Remove a camera stream."""
+        if camera_id not in self.processes:
+            return False
+
+        logger.info(f"Removing camera {camera_id}")
+
+        # Signal stop
+        if camera_id in self.stop_events:
+            self.stop_events[camera_id].set()
+
+        # Wait for process to stop
+        process = self.processes.get(camera_id)
+        if process and process.is_alive():
+            process.join(timeout=5.0)
+            if process.is_alive():
+                logger.warning(f"Force terminating process for {camera_id}")
+                process.terminate()
+                process.join(timeout=2.0)
+
+        # Cleanup
+        self._cleanup_camera(camera_id)
+        return True
+
+    def _cleanup_camera(self, camera_id: str):
+        """Clean up camera resources."""
+        for collection in [self.processes, self.frame_buffers,
+                          self.command_queues, self.stop_events, self.configs]:
+            collection.pop(camera_id, None)
+
+    def get_frame(self, camera_id: str) -> Optional[Tuple[np.ndarray, float]]:
+        """Get latest frame from camera."""
+        buffer = self.frame_buffers.get(camera_id)
+        if buffer:
+            return buffer.read_frame()
+        return None
+
+    def get_stats(self) -> Dict[str, Any]:
+        """Get statistics for all cameras."""
+        stats = {}
+        for camera_id, buffer in self.frame_buffers.items():
+            process = self.processes.get(camera_id)
+            stats[camera_id] = {
+                'buffer_stats': buffer.get_stats(),
+                'process_alive': process.is_alive() if process else False,
+                'process_pid': process.pid if process else None
+            }
+        return stats
+
+    def start_monitoring(self):
+        """Start health monitoring thread."""
+        if self.monitor_thread and self.monitor_thread.is_alive():
+            return
+
+        self.monitor_stop.clear()
+        self.monitor_thread = threading.Thread(target=self._monitor_processes)
+        self.monitor_thread.start()
+        logger.info("Started process monitoring")
+
+    def _monitor_processes(self):
+        """Monitor process health and restart if needed."""
+        while not self.monitor_stop.is_set():
+            try:
+                # Check status queue
+                try:
+                    while True:
+                        status = self.status_queue.get_nowait()
+                        self.process_stats[status['camera_id']] = status
+                except queue.Empty:
+                    pass
+
+                # Check process health
+                for camera_id in list(self.processes.keys()):
+                    process = self.processes.get(camera_id)
+                    if process and not process.is_alive():
+                        logger.warning(f"Process for {camera_id} died, restarting")
+                        config = self.configs.get(camera_id)
+                        if config:
+                            self.remove_camera(camera_id)
+                            time.sleep(1.0)
+                            self.add_camera(config)
+
+                time.sleep(5.0)  # Check every 5 seconds
+
+            except Exception as e:
+                logger.error(f"Error in monitor thread: {e}")
+                time.sleep(5.0)
+
+    def stop_all(self):
+        """Stop all camera processes."""
+        logger.info("Stopping all camera processes")
+
+        # Stop monitoring
+        if self.monitor_thread:
+            self.monitor_stop.set()
+            self.monitor_thread.join(timeout=5.0)
+
+        # Stop all cameras
+        for camera_id in list(self.processes.keys()):
+            self.remove_camera(camera_id)
+
+        logger.info("All processes stopped")

core/streaming/readers.py

@@ -1,6 +1,10 @@
 """
 Frame readers for RTSP streams and HTTP snapshots.
 Optimized for 1280x720@6fps RTSP and 2560x1440 HTTP snapshots.
+
+NOTE: This module provides threading-based readers for fallback compatibility.
+For RTSP streams, the new multiprocessing implementation in process_manager.py
+is preferred and used by default for better scalability and performance.
 """
 import cv2
 import logging