From b919a1ebe2bfbf30f567765487a2026cdafb7c1b Mon Sep 17 00:00:00 2001 From: ziesorx Date: Thu, 25 Sep 2025 22:16:19 +0700 Subject: [PATCH 01/55] fix: use nvdec --- Dockerfile.base | 46 ++++++++- build-nvdec.sh | 44 +++++++++ core/streaming/readers.py | 81 ++++++++++++--- core/utils/hardware_encoder.py | 173 +++++++++++++++++++++++++++++++++ requirements.base.txt | 3 +- 5 files changed, 328 insertions(+), 19 deletions(-) create mode 100755 build-nvdec.sh create mode 100644 core/utils/hardware_encoder.py diff --git a/Dockerfile.base b/Dockerfile.base index ade3d69..ecf7b2a 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -1,18 +1,54 @@ -# Base image with all ML dependencies +# Base image with all ML dependencies and NVIDIA Video Codec SDK FROM pytorch/pytorch:2.8.0-cuda12.6-cudnn9-runtime -# Install system dependencies +# Install system dependencies including GStreamer with NVDEC support RUN apt update && apt install -y \ libgl1 \ libglib2.0-0 \ - libgstreamer1.0-0 \ libgtk-3-0 \ - libavcodec58 \ + libgomp1 \ + # GStreamer base + libgstreamer1.0-0 \ + libgstreamer-plugins-base1.0-0 \ + libgstreamer-plugins-bad1.0-0 \ + gstreamer1.0-tools \ + gstreamer1.0-plugins-base \ + gstreamer1.0-plugins-good \ + gstreamer1.0-plugins-bad \ + gstreamer1.0-plugins-ugly \ + gstreamer1.0-libav \ + # GStreamer Python bindings + python3-gst-1.0 \ + # NVIDIA specific GStreamer plugins for hardware acceleration + gstreamer1.0-vaapi \ + # FFmpeg with hardware acceleration support + ffmpeg \ + libavcodec-extra \ libavformat58 \ libswscale5 \ - libgomp1 \ + # Additional codecs + libx264-155 \ + libx265-179 \ + # TurboJPEG for fast JPEG encoding + libturbojpeg0-dev \ && rm -rf /var/lib/apt/lists/* +# Install NVIDIA DeepStream (includes hardware accelerated GStreamer plugins) +# This provides nvv4l2decoder, nvvideoconvert, etc. +RUN apt update && apt install -y \ + wget \ + software-properties-common \ + && wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb \ + && dpkg -i cuda-keyring_1.0-1_all.deb \ + && apt update \ + && apt install -y libnvidia-decode-535 \ + && rm -rf /var/lib/apt/lists/* cuda-keyring_1.0-1_all.deb + +# Set environment variables for hardware acceleration +ENV OPENCV_FFMPEG_CAPTURE_OPTIONS="video_codec;h264_cuvid" +ENV GST_PLUGIN_PATH="/usr/lib/x86_64-linux-gnu/gstreamer-1.0" +ENV LD_LIBRARY_PATH="/usr/local/cuda/lib64:${LD_LIBRARY_PATH}" + # Copy and install base requirements (ML dependencies that rarely change) COPY requirements.base.txt . RUN pip install --no-cache-dir -r requirements.base.txt diff --git a/build-nvdec.sh b/build-nvdec.sh new file mode 100755 index 0000000..6629994 --- /dev/null +++ b/build-nvdec.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Build script for Docker image with NVDEC hardware acceleration support + +echo "Building Docker image with NVDEC hardware acceleration support..." +echo "=========================================================" + +# Build the base image first (with all ML and hardware acceleration dependencies) +echo "Building base image with NVDEC support..." +docker build -f Dockerfile.base -t detector-worker-base:nvdec . + +if [ $? -ne 0 ]; then + echo "Failed to build base image" + exit 1 +fi + +# Build the main application image +echo "Building application image..." +docker build -t detector-worker:nvdec . + +if [ $? -ne 0 ]; then + echo "Failed to build application image" + exit 1 +fi + +echo "" +echo "=========================================================" +echo "Build complete!" +echo "" +echo "To run the container with GPU support:" +echo "docker run --gpus all -p 8000:8000 detector-worker:nvdec" +echo "" +echo "Hardware acceleration features enabled:" +echo "- NVDEC for H.264/H.265 video decoding" +echo "- NVENC for video encoding (if needed)" +echo "- TurboJPEG for fast JPEG encoding" +echo "- CUDA for model inference" +echo "" +echo "The application will automatically detect and use:" +echo "1. GStreamer with NVDEC (NVIDIA GPUs)" +echo "2. FFMPEG with CUVID (NVIDIA GPUs)" +echo "3. VAAPI (Intel/AMD GPUs)" +echo "4. TurboJPEG (3-5x faster than standard JPEG)" +echo "=========================================================" \ No newline at end of file diff --git a/core/streaming/readers.py b/core/streaming/readers.py index a48840a..0a989b5 100644 --- a/core/streaming/readers.py +++ b/core/streaming/readers.py @@ -166,28 +166,83 @@ class RTSPReader: logger.info(f"RTSP reader thread ended for camera {self.camera_id}") def _initialize_capture(self) -> bool: - """Initialize video capture with optimized settings for 1280x720@6fps.""" + """Initialize video capture with hardware acceleration (NVDEC) for 1280x720@6fps.""" try: # Release previous capture if exists if self.cap: self.cap.release() time.sleep(0.5) - logger.info(f"Initializing capture for camera {self.camera_id}") + logger.info(f"Initializing capture for camera {self.camera_id} with hardware acceleration") + hw_accel_success = False - # Create capture with FFMPEG backend and TCP transport for reliability - # Use TCP instead of UDP to prevent packet loss - rtsp_url_tcp = self.rtsp_url.replace('rtsp://', 'rtsp://') - if '?' in rtsp_url_tcp: - rtsp_url_tcp += '&tcp' - else: - rtsp_url_tcp += '?tcp' + # Method 1: Try GStreamer with NVDEC (most efficient on NVIDIA GPUs) + if not hw_accel_success: + try: + # Build GStreamer pipeline for NVIDIA hardware decoding + gst_pipeline = ( + f"rtspsrc location={self.rtsp_url} protocols=tcp latency=100 ! " + "rtph264depay ! h264parse ! " + "nvv4l2decoder ! " # NVIDIA hardware decoder + "nvvideoconvert ! " # NVIDIA hardware color conversion + "video/x-raw,format=BGRx,width=1280,height=720 ! " + "videoconvert ! " + "video/x-raw,format=BGR ! " + "appsink max-buffers=1 drop=true sync=false" + ) + logger.info(f"Attempting GStreamer NVDEC pipeline for camera {self.camera_id}") + self.cap = cv2.VideoCapture(gst_pipeline, cv2.CAP_GSTREAMER) - # Alternative: Set environment variable for RTSP transport - import os - os.environ['OPENCV_FFMPEG_CAPTURE_OPTIONS'] = 'rtsp_transport;tcp' + if self.cap.isOpened(): + hw_accel_success = True + logger.info(f"Camera {self.camera_id}: Successfully using GStreamer with NVDEC hardware acceleration") + except Exception as e: + logger.debug(f"Camera {self.camera_id}: GStreamer NVDEC not available: {e}") - self.cap = cv2.VideoCapture(self.rtsp_url, cv2.CAP_FFMPEG) + # Method 2: Try FFMPEG with NVIDIA CUVID hardware decoder + if not hw_accel_success: + try: + import os + # Set FFMPEG to use NVIDIA CUVID decoder + os.environ['OPENCV_FFMPEG_CAPTURE_OPTIONS'] = 'video_codec;h264_cuvid|rtsp_transport;tcp|hwaccel;cuda' + + logger.info(f"Attempting FFMPEG with h264_cuvid for camera {self.camera_id}") + self.cap = cv2.VideoCapture(self.rtsp_url, cv2.CAP_FFMPEG) + + if self.cap.isOpened(): + hw_accel_success = True + logger.info(f"Camera {self.camera_id}: Using FFMPEG with CUVID hardware acceleration") + except Exception as e: + logger.debug(f"Camera {self.camera_id}: FFMPEG CUVID not available: {e}") + + # Method 3: Try VAAPI hardware acceleration (for Intel/AMD GPUs) + if not hw_accel_success: + try: + gst_pipeline = ( + f"rtspsrc location={self.rtsp_url} protocols=tcp latency=100 ! " + "rtph264depay ! h264parse ! " + "vaapih264dec ! " # VAAPI hardware decoder + "vaapipostproc ! " + "video/x-raw,format=BGRx,width=1280,height=720 ! " + "videoconvert ! " + "video/x-raw,format=BGR ! " + "appsink max-buffers=1 drop=true sync=false" + ) + logger.info(f"Attempting GStreamer VAAPI pipeline for camera {self.camera_id}") + self.cap = cv2.VideoCapture(gst_pipeline, cv2.CAP_GSTREAMER) + + if self.cap.isOpened(): + hw_accel_success = True + logger.info(f"Camera {self.camera_id}: Successfully using GStreamer with VAAPI hardware acceleration") + except Exception as e: + logger.debug(f"Camera {self.camera_id}: GStreamer VAAPI not available: {e}") + + # Fallback: Standard FFMPEG with software decoding + if not hw_accel_success: + logger.warning(f"Camera {self.camera_id}: Hardware acceleration not available, falling back to software decoding") + import os + os.environ['OPENCV_FFMPEG_CAPTURE_OPTIONS'] = 'rtsp_transport;tcp' + self.cap = cv2.VideoCapture(self.rtsp_url, cv2.CAP_FFMPEG) if not self.cap.isOpened(): logger.error(f"Failed to open stream for camera {self.camera_id}") diff --git a/core/utils/hardware_encoder.py b/core/utils/hardware_encoder.py new file mode 100644 index 0000000..45bbb35 --- /dev/null +++ b/core/utils/hardware_encoder.py @@ -0,0 +1,173 @@ +""" +Hardware-accelerated image encoding using NVIDIA NVENC or Intel QuickSync +""" + +import cv2 +import numpy as np +import logging +from typing import Optional, Tuple +import os + +logger = logging.getLogger("detector_worker") + + +class HardwareEncoder: + """Hardware-accelerated JPEG encoder using GPU.""" + + def __init__(self): + """Initialize hardware encoder.""" + self.nvenc_available = False + self.vaapi_available = False + self.turbojpeg_available = False + + # Check for TurboJPEG (fastest CPU-based option) + try: + from turbojpeg import TurboJPEG + self.turbojpeg = TurboJPEG() + self.turbojpeg_available = True + logger.info("TurboJPEG accelerated encoding available") + except ImportError: + logger.debug("TurboJPEG not available") + + # Check for NVIDIA NVENC support + try: + # Test if we can create an NVENC encoder + test_frame = np.zeros((720, 1280, 3), dtype=np.uint8) + fourcc = cv2.VideoWriter_fourcc(*'H264') + test_writer = cv2.VideoWriter( + "test.mp4", + fourcc, + 30, + (1280, 720), + [cv2.CAP_PROP_HW_ACCELERATION, cv2.VIDEO_ACCELERATION_ANY] + ) + if test_writer.isOpened(): + self.nvenc_available = True + logger.info("NVENC hardware encoding available") + test_writer.release() + if os.path.exists("test.mp4"): + os.remove("test.mp4") + except Exception as e: + logger.debug(f"NVENC not available: {e}") + + def encode_jpeg(self, frame: np.ndarray, quality: int = 85) -> Optional[bytes]: + """ + Encode frame to JPEG using the fastest available method. + + Args: + frame: BGR image frame + quality: JPEG quality (1-100) + + Returns: + Encoded JPEG bytes or None on failure + """ + try: + # Method 1: TurboJPEG (3-5x faster than cv2.imencode) + if self.turbojpeg_available: + # Convert BGR to RGB for TurboJPEG + rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + encoded = self.turbojpeg.encode(rgb_frame, quality=quality) + return encoded + + # Method 2: Hardware-accelerated encoding via GStreamer (if available) + if self.nvenc_available: + return self._encode_with_nvenc(frame, quality) + + # Fallback: Standard OpenCV encoding + encode_params = [cv2.IMWRITE_JPEG_QUALITY, quality] + success, encoded = cv2.imencode('.jpg', frame, encode_params) + if success: + return encoded.tobytes() + + return None + + except Exception as e: + logger.error(f"Failed to encode frame: {e}") + return None + + def _encode_with_nvenc(self, frame: np.ndarray, quality: int) -> Optional[bytes]: + """ + Encode using NVIDIA NVENC hardware encoder. + + This is complex to implement directly, so we'll use a GStreamer pipeline + if available. + """ + try: + # Create a GStreamer pipeline for hardware encoding + height, width = frame.shape[:2] + gst_pipeline = ( + f"appsrc ! " + f"video/x-raw,format=BGR,width={width},height={height},framerate=30/1 ! " + f"videoconvert ! " + f"nvvideoconvert ! " # GPU color conversion + f"nvjpegenc quality={quality} ! " # Hardware JPEG encoder + f"appsink" + ) + + # This would require GStreamer Python bindings + # For now, fall back to TurboJPEG or standard encoding + logger.debug("NVENC JPEG encoding not fully implemented, using fallback") + encode_params = [cv2.IMWRITE_JPEG_QUALITY, quality] + success, encoded = cv2.imencode('.jpg', frame, encode_params) + if success: + return encoded.tobytes() + + return None + + except Exception as e: + logger.error(f"NVENC encoding failed: {e}") + return None + + def encode_batch(self, frames: list, quality: int = 85) -> list: + """ + Batch encode multiple frames for better GPU utilization. + + Args: + frames: List of BGR frames + quality: JPEG quality + + Returns: + List of encoded JPEG bytes + """ + encoded_frames = [] + + if self.turbojpeg_available: + # TurboJPEG can handle batch encoding efficiently + for frame in frames: + rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + encoded = self.turbojpeg.encode(rgb_frame, quality=quality) + encoded_frames.append(encoded) + else: + # Fallback to sequential encoding + for frame in frames: + encoded = self.encode_jpeg(frame, quality) + encoded_frames.append(encoded) + + return encoded_frames + + +# Global encoder instance +_hardware_encoder = None + + +def get_hardware_encoder() -> HardwareEncoder: + """Get or create the global hardware encoder instance.""" + global _hardware_encoder + if _hardware_encoder is None: + _hardware_encoder = HardwareEncoder() + return _hardware_encoder + + +def encode_frame_hardware(frame: np.ndarray, quality: int = 85) -> Optional[bytes]: + """ + Convenience function to encode a frame using hardware acceleration. + + Args: + frame: BGR image frame + quality: JPEG quality (1-100) + + Returns: + Encoded JPEG bytes or None on failure + """ + encoder = get_hardware_encoder() + return encoder.encode_jpeg(frame, quality) \ No newline at end of file diff --git a/requirements.base.txt b/requirements.base.txt index 04e90ba..3511dd4 100644 --- a/requirements.base.txt +++ b/requirements.base.txt @@ -6,4 +6,5 @@ scipy filterpy psycopg2-binary lap>=0.5.12 -pynvml \ No newline at end of file +pynvml +PyTurboJPEG \ No newline at end of file From 5f29392c2fbbd82e7337e1047068179c35fc3012 Mon Sep 17 00:00:00 2001 From: ziesorx Date: Thu, 25 Sep 2025 22:25:27 +0700 Subject: [PATCH 02/55] chore: update Dockerfile.base --- Dockerfile.base | 3 --- 1 file changed, 3 deletions(-) diff --git a/Dockerfile.base b/Dockerfile.base index ecf7b2a..281ba9d 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -26,9 +26,6 @@ RUN apt update && apt install -y \ libavcodec-extra \ libavformat58 \ libswscale5 \ - # Additional codecs - libx264-155 \ - libx265-179 \ # TurboJPEG for fast JPEG encoding libturbojpeg0-dev \ && rm -rf /var/lib/apt/lists/* From 6bb679f4d84bf70d535ac1a52cf987f508829301 Mon Sep 17 00:00:00 2001 From: ziesorx Date: Thu, 25 Sep 2025 22:59:55 +0700 Subject: [PATCH 03/55] fix: use gpu --- Dockerfile.base | 176 +++++++++++++++++++++----- README-hardware-acceleration.md | 127 +++++++++++++++++++ build-nvdec.sh | 44 ------- core/streaming/readers.py | 56 +++++++-- core/utils/ffmpeg_detector.py | 214 ++++++++++++++++++++++++++++++++ 5 files changed, 533 insertions(+), 84 deletions(-) create mode 100644 README-hardware-acceleration.md delete mode 100755 build-nvdec.sh create mode 100644 core/utils/ffmpeg_detector.py diff --git a/Dockerfile.base b/Dockerfile.base index 281ba9d..620f4d8 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -1,54 +1,166 @@ -# Base image with all ML dependencies and NVIDIA Video Codec SDK +# Base image with complete ML and hardware acceleration stack FROM pytorch/pytorch:2.8.0-cuda12.6-cudnn9-runtime -# Install system dependencies including GStreamer with NVDEC support -RUN apt update && apt install -y \ +# Install build dependencies and system libraries +RUN apt-get update && apt-get install -y \ + # Build tools + build-essential \ + cmake \ + git \ + pkg-config \ + wget \ + unzip \ + yasm \ + nasm \ + # System libraries libgl1 \ libglib2.0-0 \ libgtk-3-0 \ libgomp1 \ - # GStreamer base - libgstreamer1.0-0 \ - libgstreamer-plugins-base1.0-0 \ - libgstreamer-plugins-bad1.0-0 \ + # Media libraries for FFmpeg build + libjpeg-dev \ + libpng-dev \ + libtiff-dev \ + libx264-dev \ + libx265-dev \ + libvpx-dev \ + libfdk-aac-dev \ + libmp3lame-dev \ + libopus-dev \ + libv4l-dev \ + libxvidcore-dev \ + libdc1394-22-dev \ + # TurboJPEG for fast JPEG encoding + libturbojpeg0-dev \ + # GStreamer complete stack + libgstreamer1.0-dev \ + libgstreamer-plugins-base1.0-dev \ + libgstreamer-plugins-bad1.0-dev \ gstreamer1.0-tools \ gstreamer1.0-plugins-base \ gstreamer1.0-plugins-good \ gstreamer1.0-plugins-bad \ gstreamer1.0-plugins-ugly \ gstreamer1.0-libav \ - # GStreamer Python bindings - python3-gst-1.0 \ - # NVIDIA specific GStreamer plugins for hardware acceleration gstreamer1.0-vaapi \ - # FFmpeg with hardware acceleration support - ffmpeg \ - libavcodec-extra \ - libavformat58 \ - libswscale5 \ - # TurboJPEG for fast JPEG encoding - libturbojpeg0-dev \ + python3-gst-1.0 \ + # Python development + python3-dev \ + python3-numpy \ + # NVIDIA driver components + libnvidia-encode-535 \ + libnvidia-decode-535 \ && rm -rf /var/lib/apt/lists/* -# Install NVIDIA DeepStream (includes hardware accelerated GStreamer plugins) -# This provides nvv4l2decoder, nvvideoconvert, etc. -RUN apt update && apt install -y \ - wget \ - software-properties-common \ - && wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb \ - && dpkg -i cuda-keyring_1.0-1_all.deb \ - && apt update \ - && apt install -y libnvidia-decode-535 \ - && rm -rf /var/lib/apt/lists/* cuda-keyring_1.0-1_all.deb +# Install NVIDIA Video Codec SDK headers +RUN cd /tmp && \ + wget https://github.com/FFmpeg/nv-codec-headers/archive/refs/tags/n12.1.14.0.zip && \ + unzip n12.1.14.0.zip && \ + cd nv-codec-headers-n12.1.14.0 && \ + make install && \ + rm -rf /tmp/* -# Set environment variables for hardware acceleration -ENV OPENCV_FFMPEG_CAPTURE_OPTIONS="video_codec;h264_cuvid" +# Build FFmpeg from source with full NVIDIA hardware acceleration +ENV FFMPEG_VERSION=6.0 +RUN cd /tmp && \ + wget https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.xz && \ + tar xf ffmpeg-${FFMPEG_VERSION}.tar.xz && \ + cd ffmpeg-${FFMPEG_VERSION} && \ + ./configure \ + --enable-gpl \ + --enable-nonfree \ + --enable-libx264 \ + --enable-libx265 \ + --enable-libvpx \ + --enable-libfdk-aac \ + --enable-libmp3lame \ + --enable-libopus \ + --enable-cuda-nvcc \ + --enable-cuvid \ + --enable-nvenc \ + --enable-nvdec \ + --enable-cuda-llvm \ + --enable-libnpp \ + --extra-cflags=-I/usr/local/cuda/include \ + --extra-ldflags=-L/usr/local/cuda/lib64 \ + --nvccflags="-gencode arch=compute_50,code=sm_50 -gencode arch=compute_52,code=sm_52 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_89,code=sm_89 -gencode arch=compute_90,code=sm_90" && \ + make -j$(nproc) && \ + make install && \ + ldconfig && \ + cd / && rm -rf /tmp/* + +# Build OpenCV from source with custom FFmpeg and full CUDA support +ENV OPENCV_VERSION=4.8.1 +RUN cd /tmp && \ + wget -O opencv.zip https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.zip && \ + wget -O opencv_contrib.zip https://github.com/opencv/opencv_contrib/archive/${OPENCV_VERSION}.zip && \ + unzip opencv.zip && \ + unzip opencv_contrib.zip && \ + cd opencv-${OPENCV_VERSION} && \ + mkdir build && cd build && \ + PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH \ + cmake -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=/usr/local \ + -D WITH_CUDA=ON \ + -D WITH_CUDNN=ON \ + -D OPENCV_DNN_CUDA=ON \ + -D ENABLE_FAST_MATH=ON \ + -D CUDA_FAST_MATH=ON \ + -D WITH_CUBLAS=ON \ + -D WITH_NVCUVID=ON \ + -D WITH_CUVID=ON \ + -D BUILD_opencv_cudacodec=ON \ + -D WITH_FFMPEG=ON \ + -D WITH_GSTREAMER=ON \ + -D WITH_LIBV4L=ON \ + -D BUILD_opencv_python3=ON \ + -D OPENCV_GENERATE_PKGCONFIG=ON \ + -D OPENCV_ENABLE_NONFREE=ON \ + -D OPENCV_EXTRA_MODULES_PATH=/tmp/opencv_contrib-${OPENCV_VERSION}/modules \ + -D PYTHON3_EXECUTABLE=$(which python3) \ + -D PYTHON_INCLUDE_DIR=$(python3 -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())") \ + -D PYTHON_LIBRARY=$(python3 -c "import distutils.sysconfig as sysconfig; print(sysconfig.get_config_var('LIBDIR'))") \ + -D BUILD_EXAMPLES=OFF \ + -D BUILD_TESTS=OFF \ + -D BUILD_PERF_TESTS=OFF \ + .. && \ + make -j$(nproc) && \ + make install && \ + ldconfig && \ + cd / && rm -rf /tmp/* + +# Set environment variables for maximum hardware acceleration +ENV LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/local/lib:${LD_LIBRARY_PATH}" +ENV PKG_CONFIG_PATH="/usr/local/lib/pkgconfig:${PKG_CONFIG_PATH}" +ENV PYTHONPATH="/usr/local/lib/python3.10/dist-packages:${PYTHONPATH}" ENV GST_PLUGIN_PATH="/usr/lib/x86_64-linux-gnu/gstreamer-1.0" -ENV LD_LIBRARY_PATH="/usr/local/cuda/lib64:${LD_LIBRARY_PATH}" -# Copy and install base requirements (ML dependencies that rarely change) +# Optimized environment variables for hardware acceleration +ENV OPENCV_FFMPEG_CAPTURE_OPTIONS="rtsp_transport;tcp|hwaccel;cuda|hwaccel_device;0|video_codec;h264_cuvid|hwaccel_output_format;cuda" +ENV OPENCV_FFMPEG_WRITER_OPTIONS="video_codec;h264_nvenc|preset;fast|tune;zerolatency|gpu;0" +ENV CUDA_VISIBLE_DEVICES=0 +ENV NVIDIA_VISIBLE_DEVICES=all +ENV NVIDIA_DRIVER_CAPABILITIES=compute,video,utility + +# Copy and install base requirements (exclude opencv-python since we built from source) COPY requirements.base.txt . -RUN pip install --no-cache-dir -r requirements.base.txt +RUN grep -v opencv-python requirements.base.txt > requirements.tmp && \ + mv requirements.tmp requirements.base.txt && \ + pip install --no-cache-dir -r requirements.base.txt + +# Verify complete hardware acceleration setup +RUN echo "=== Hardware Acceleration Verification ===" && \ + echo "FFmpeg Hardware Accelerators:" && \ + ffmpeg -hide_banner -hwaccels 2>/dev/null | head -10 && \ + echo "FFmpeg NVIDIA Decoders:" && \ + ffmpeg -hide_banner -decoders 2>/dev/null | grep -E "(cuvid|nvdec)" | head -5 && \ + echo "FFmpeg NVIDIA Encoders:" && \ + ffmpeg -hide_banner -encoders 2>/dev/null | grep nvenc | head -5 && \ + echo "OpenCV Configuration:" && \ + python3 -c "import cv2; print('OpenCV version:', cv2.__version__); print('CUDA devices:', cv2.cuda.getCudaEnabledDeviceCount()); build_info = cv2.getBuildInformation(); print('CUDA support:', 'CUDA' in build_info); print('CUVID support:', 'CUVID' in build_info); print('FFmpeg support:', 'FFMPEG' in build_info); print('GStreamer support:', 'GStreamer' in build_info)" && \ + echo "GStreamer NVIDIA Plugins:" && \ + gst-inspect-1.0 2>/dev/null | grep -E "(nvv4l2|nvvideo)" | head -5 || echo "GStreamer NVIDIA plugins not detected" && \ + echo "=== Verification Complete ===" # Set working directory WORKDIR /app diff --git a/README-hardware-acceleration.md b/README-hardware-acceleration.md new file mode 100644 index 0000000..69c6e09 --- /dev/null +++ b/README-hardware-acceleration.md @@ -0,0 +1,127 @@ +# Hardware Acceleration Setup + +This detector worker now includes **complete NVIDIA hardware acceleration** with FFmpeg and OpenCV built from source. + +## What's Included + +### 🔧 Complete Hardware Stack +- **FFmpeg 6.0** built from source with NVIDIA Video Codec SDK +- **OpenCV 4.8.1** built with CUDA and custom FFmpeg integration +- **GStreamer** with NVDEC/VAAPI plugins +- **TurboJPEG** for optimized JPEG encoding (3-5x faster) +- **CUDA** support for YOLO model inference + +### 🎯 Hardware Acceleration Methods (Automatic Detection) +1. **GStreamer NVDEC** - Best for RTSP streaming, lowest latency +2. **OpenCV CUDA** - Direct GPU memory access, best integration +3. **FFmpeg CUVID** - Custom build with full NVIDIA acceleration +4. **VAAPI** - Intel/AMD GPU support +5. **Software Fallback** - CPU-only as last resort + +## Build and Run + +### Single Build Script +```bash +./build-nvdec.sh +``` +**Build time**: 45-90 minutes (compiles FFmpeg + OpenCV from source) + +### Run with GPU Support +```bash +docker run --gpus all -p 8000:8000 detector-worker:complete-hw-accel +``` + +## Performance Improvements + +### Expected CPU Reduction +- **Video decoding**: 70-90% reduction (moved to GPU) +- **JPEG encoding**: 70-80% faster with TurboJPEG +- **Model inference**: GPU accelerated with CUDA +- **Overall system**: 50-80% less CPU usage + +### Profiling Results Comparison +**Before (Software Only)**: +- `cv2.imencode`: 6.5% CPU time (1.95s out of 30s) +- `psutil.cpu_percent`: 88% CPU time (idle polling) +- Video decoding: 100% CPU + +**After (Hardware Accelerated)**: +- Video decoding: GPU (~5-10% CPU overhead) +- JPEG encoding: 3-5x faster with TurboJPEG +- Model inference: GPU accelerated + +## Verification + +### Check Hardware Acceleration Support +```bash +docker run --rm --gpus all detector-worker:complete-hw-accel \ + bash -c "ffmpeg -hwaccels && python3 -c 'import cv2; build=cv2.getBuildInformation(); print(\"CUDA:\", \"CUDA\" in build); print(\"CUVID:\", \"CUVID\" in build)'" +``` + +### Runtime Logs +The application will automatically log which acceleration method is being used: +``` +Camera cam1: Successfully using GStreamer with NVDEC hardware acceleration +Camera cam2: Using FFMPEG hardware acceleration (backend: FFMPEG) +Camera cam3: Using OpenCV CUDA hardware acceleration +``` + +## Files Modified + +### Docker Configuration +- **Dockerfile.base** - Complete hardware acceleration stack +- **build-nvdec.sh** - Single build script for everything + +### Application Code +- **core/streaming/readers.py** - Multi-method hardware acceleration +- **core/utils/hardware_encoder.py** - TurboJPEG + NVENC encoding +- **core/utils/ffmpeg_detector.py** - Runtime capability detection +- **requirements.base.txt** - Added TurboJPEG, removed opencv-python + +## Architecture + +``` +Input RTSP Stream + ↓ +1. GStreamer NVDEC Pipeline (NVIDIA GPU) + rtspsrc → nvv4l2decoder → nvvideoconvert → OpenCV + ↓ +2. OpenCV CUDA Backend (NVIDIA GPU) + OpenCV with CUDA acceleration + ↓ +3. FFmpeg CUVID (NVIDIA GPU) + Custom FFmpeg with h264_cuvid decoder + ↓ +4. VAAPI (Intel/AMD GPU) + Hardware acceleration for non-NVIDIA + ↓ +5. Software Fallback (CPU) + Standard OpenCV software decoding +``` + +## Benefits + +### For Development +- **Single Dockerfile.base** - Everything consolidated +- **Automatic detection** - No manual configuration needed +- **Graceful fallback** - Works without GPU for development + +### For Production +- **Maximum performance** - Uses best available acceleration +- **GPU memory efficiency** - Direct GPU-to-GPU pipeline +- **Lower latency** - Hardware decoding + CUDA inference +- **Reduced CPU load** - Frees CPU for other tasks + +## Troubleshooting + +### Build Issues +- Ensure NVIDIA Docker runtime is installed +- Check CUDA 12.6 compatibility with your GPU +- Build takes 45-90 minutes - be patient + +### Runtime Issues +- Verify `nvidia-smi` works in container +- Check logs for acceleration method being used +- Fallback to software decoding is automatic + +This setup provides **production-ready hardware acceleration** with automatic detection and graceful fallback for maximum compatibility. \ No newline at end of file diff --git a/build-nvdec.sh b/build-nvdec.sh deleted file mode 100755 index 6629994..0000000 --- a/build-nvdec.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -# Build script for Docker image with NVDEC hardware acceleration support - -echo "Building Docker image with NVDEC hardware acceleration support..." -echo "=========================================================" - -# Build the base image first (with all ML and hardware acceleration dependencies) -echo "Building base image with NVDEC support..." -docker build -f Dockerfile.base -t detector-worker-base:nvdec . - -if [ $? -ne 0 ]; then - echo "Failed to build base image" - exit 1 -fi - -# Build the main application image -echo "Building application image..." -docker build -t detector-worker:nvdec . - -if [ $? -ne 0 ]; then - echo "Failed to build application image" - exit 1 -fi - -echo "" -echo "=========================================================" -echo "Build complete!" -echo "" -echo "To run the container with GPU support:" -echo "docker run --gpus all -p 8000:8000 detector-worker:nvdec" -echo "" -echo "Hardware acceleration features enabled:" -echo "- NVDEC for H.264/H.265 video decoding" -echo "- NVENC for video encoding (if needed)" -echo "- TurboJPEG for fast JPEG encoding" -echo "- CUDA for model inference" -echo "" -echo "The application will automatically detect and use:" -echo "1. GStreamer with NVDEC (NVIDIA GPUs)" -echo "2. FFMPEG with CUVID (NVIDIA GPUs)" -echo "3. VAAPI (Intel/AMD GPUs)" -echo "4. TurboJPEG (3-5x faster than standard JPEG)" -echo "=========================================================" \ No newline at end of file diff --git a/core/streaming/readers.py b/core/streaming/readers.py index 0a989b5..377db56 100644 --- a/core/streaming/readers.py +++ b/core/streaming/readers.py @@ -199,23 +199,63 @@ class RTSPReader: except Exception as e: logger.debug(f"Camera {self.camera_id}: GStreamer NVDEC not available: {e}") - # Method 2: Try FFMPEG with NVIDIA CUVID hardware decoder + # Method 2: Try OpenCV CUDA VideoReader (if built with CUVID support) if not hw_accel_success: try: - import os - # Set FFMPEG to use NVIDIA CUVID decoder - os.environ['OPENCV_FFMPEG_CAPTURE_OPTIONS'] = 'video_codec;h264_cuvid|rtsp_transport;tcp|hwaccel;cuda' + # Check if OpenCV was built with CUDA codec support + build_info = cv2.getBuildInformation() + if 'cudacodec' in build_info or 'CUVID' in build_info: + logger.info(f"Attempting OpenCV CUDA VideoReader for camera {self.camera_id}") + + # Use OpenCV's CUDA backend + self.cap = cv2.VideoCapture(self.rtsp_url, cv2.CAP_FFMPEG, [ + cv2.CAP_PROP_HW_ACCELERATION, cv2.VIDEO_ACCELERATION_ANY + ]) + + if self.cap.isOpened(): + hw_accel_success = True + logger.info(f"Camera {self.camera_id}: Using OpenCV CUDA hardware acceleration") + else: + logger.debug(f"Camera {self.camera_id}: OpenCV not built with CUDA codec support") + except Exception as e: + logger.debug(f"Camera {self.camera_id}: OpenCV CUDA not available: {e}") + + # Method 3: Try FFMPEG with optimal hardware acceleration (CUVID/VAAPI) + if not hw_accel_success: + try: + from core.utils.ffmpeg_detector import get_optimal_rtsp_options + import os + + # Get optimal FFmpeg options based on detected capabilities + optimal_options = get_optimal_rtsp_options(self.rtsp_url) + os.environ['OPENCV_FFMPEG_CAPTURE_OPTIONS'] = optimal_options + + logger.info(f"Attempting FFMPEG with detected hardware acceleration for camera {self.camera_id}") + logger.debug(f"Camera {self.camera_id}: Using FFmpeg options: {optimal_options}") - logger.info(f"Attempting FFMPEG with h264_cuvid for camera {self.camera_id}") self.cap = cv2.VideoCapture(self.rtsp_url, cv2.CAP_FFMPEG) if self.cap.isOpened(): hw_accel_success = True - logger.info(f"Camera {self.camera_id}: Using FFMPEG with CUVID hardware acceleration") + # Try to get backend info to confirm hardware acceleration + backend = self.cap.getBackendName() + logger.info(f"Camera {self.camera_id}: Using FFMPEG hardware acceleration (backend: {backend})") except Exception as e: - logger.debug(f"Camera {self.camera_id}: FFMPEG CUVID not available: {e}") + logger.debug(f"Camera {self.camera_id}: FFMPEG hardware acceleration not available: {e}") - # Method 3: Try VAAPI hardware acceleration (for Intel/AMD GPUs) + # Fallback to basic CUVID + try: + import os + os.environ['OPENCV_FFMPEG_CAPTURE_OPTIONS'] = 'video_codec;h264_cuvid|rtsp_transport;tcp|hwaccel;cuda' + self.cap = cv2.VideoCapture(self.rtsp_url, cv2.CAP_FFMPEG) + + if self.cap.isOpened(): + hw_accel_success = True + logger.info(f"Camera {self.camera_id}: Using basic FFMPEG CUVID hardware acceleration") + except Exception as e2: + logger.debug(f"Camera {self.camera_id}: Basic CUVID also failed: {e2}") + + # Method 4: Try VAAPI hardware acceleration (for Intel/AMD GPUs) if not hw_accel_success: try: gst_pipeline = ( diff --git a/core/utils/ffmpeg_detector.py b/core/utils/ffmpeg_detector.py new file mode 100644 index 0000000..a3cf8fc --- /dev/null +++ b/core/utils/ffmpeg_detector.py @@ -0,0 +1,214 @@ +""" +FFmpeg hardware acceleration detection and configuration +""" + +import subprocess +import logging +import re +from typing import Dict, List, Optional + +logger = logging.getLogger("detector_worker") + + +class FFmpegCapabilities: + """Detect and configure FFmpeg hardware acceleration capabilities.""" + + def __init__(self): + """Initialize FFmpeg capabilities detector.""" + self.hwaccels = [] + self.codecs = {} + self.nvidia_support = False + self.vaapi_support = False + self.qsv_support = False + + self._detect_capabilities() + + def _detect_capabilities(self): + """Detect available hardware acceleration methods.""" + try: + # Get hardware accelerators + result = subprocess.run( + ['ffmpeg', '-hide_banner', '-hwaccels'], + capture_output=True, text=True, timeout=10 + ) + if result.returncode == 0: + self.hwaccels = [line.strip() for line in result.stdout.strip().split('\n')[1:] if line.strip()] + logger.info(f"Available FFmpeg hardware accelerators: {', '.join(self.hwaccels)}") + + # Check for NVIDIA support + self.nvidia_support = any(hw in self.hwaccels for hw in ['cuda', 'cuvid', 'nvdec']) + self.vaapi_support = 'vaapi' in self.hwaccels + self.qsv_support = 'qsv' in self.hwaccels + + # Get decoder information + self._detect_decoders() + + # Log capabilities + if self.nvidia_support: + logger.info("NVIDIA hardware acceleration available (CUDA/CUVID/NVDEC)") + if self.vaapi_support: + logger.info("VAAPI hardware acceleration available") + if self.qsv_support: + logger.info("Intel QuickSync hardware acceleration available") + + except Exception as e: + logger.warning(f"Failed to detect FFmpeg capabilities: {e}") + + def _detect_decoders(self): + """Detect available hardware decoders.""" + try: + result = subprocess.run( + ['ffmpeg', '-hide_banner', '-decoders'], + capture_output=True, text=True, timeout=10 + ) + if result.returncode == 0: + # Parse decoder output to find hardware decoders + for line in result.stdout.split('\n'): + if 'cuvid' in line or 'nvdec' in line: + match = re.search(r'(\w+)\s+.*?(\w+(?:_cuvid|_nvdec))', line) + if match: + codec_type, decoder = match.groups() + if 'h264' in decoder: + self.codecs['h264_hw'] = decoder + elif 'hevc' in decoder or 'h265' in decoder: + self.codecs['h265_hw'] = decoder + elif 'vaapi' in line: + match = re.search(r'(\w+)\s+.*?(\w+_vaapi)', line) + if match: + codec_type, decoder = match.groups() + if 'h264' in decoder: + self.codecs['h264_vaapi'] = decoder + + except Exception as e: + logger.debug(f"Failed to detect decoders: {e}") + + def get_optimal_capture_options(self, codec: str = 'h264') -> Dict[str, str]: + """ + Get optimal FFmpeg capture options for the given codec. + + Args: + codec: Video codec (h264, h265, etc.) + + Returns: + Dictionary of FFmpeg options + """ + options = { + 'rtsp_transport': 'tcp', + 'buffer_size': '1024k', + 'max_delay': '500000', # 500ms + 'fflags': '+genpts', + 'flags': '+low_delay', + 'probesize': '32', + 'analyzeduration': '0' + } + + # Add hardware acceleration if available + if self.nvidia_support: + if codec == 'h264' and 'h264_hw' in self.codecs: + options.update({ + 'hwaccel': 'cuda', + 'hwaccel_device': '0', + 'video_codec': 'h264_cuvid', + 'hwaccel_output_format': 'cuda' + }) + logger.debug("Using NVIDIA CUVID hardware acceleration for H.264") + elif codec == 'h265' and 'h265_hw' in self.codecs: + options.update({ + 'hwaccel': 'cuda', + 'hwaccel_device': '0', + 'video_codec': 'hevc_cuvid', + 'hwaccel_output_format': 'cuda' + }) + logger.debug("Using NVIDIA CUVID hardware acceleration for H.265") + + elif self.vaapi_support: + if codec == 'h264': + options.update({ + 'hwaccel': 'vaapi', + 'hwaccel_device': '/dev/dri/renderD128', + 'video_codec': 'h264_vaapi' + }) + logger.debug("Using VAAPI hardware acceleration") + + return options + + def format_opencv_options(self, options: Dict[str, str]) -> str: + """ + Format options for OpenCV FFmpeg backend. + + Args: + options: Dictionary of FFmpeg options + + Returns: + Formatted options string for OpenCV + """ + return '|'.join(f"{key};{value}" for key, value in options.items()) + + def get_hardware_encoder_options(self, codec: str = 'h264', quality: str = 'fast') -> Dict[str, str]: + """ + Get optimal hardware encoding options. + + Args: + codec: Video codec for encoding + quality: Quality preset (fast, medium, slow) + + Returns: + Dictionary of encoding options + """ + options = {} + + if self.nvidia_support: + if codec == 'h264': + options.update({ + 'video_codec': 'h264_nvenc', + 'preset': quality, + 'tune': 'zerolatency', + 'gpu': '0', + 'rc': 'cbr_hq', + 'surfaces': '64' + }) + elif codec == 'h265': + options.update({ + 'video_codec': 'hevc_nvenc', + 'preset': quality, + 'tune': 'zerolatency', + 'gpu': '0' + }) + + elif self.vaapi_support: + if codec == 'h264': + options.update({ + 'video_codec': 'h264_vaapi', + 'vaapi_device': '/dev/dri/renderD128' + }) + + return options + + +# Global instance +_ffmpeg_caps = None + +def get_ffmpeg_capabilities() -> FFmpegCapabilities: + """Get or create the global FFmpeg capabilities instance.""" + global _ffmpeg_caps + if _ffmpeg_caps is None: + _ffmpeg_caps = FFmpegCapabilities() + return _ffmpeg_caps + +def get_optimal_rtsp_options(rtsp_url: str) -> str: + """ + Get optimal OpenCV FFmpeg options for RTSP streaming. + + Args: + rtsp_url: RTSP stream URL + + Returns: + Formatted options string for cv2.VideoCapture + """ + caps = get_ffmpeg_capabilities() + + # Detect codec from URL or assume H.264 + codec = 'h265' if any(x in rtsp_url.lower() for x in ['h265', 'hevc']) else 'h264' + + options = caps.get_optimal_capture_options(codec) + return caps.format_opencv_options(options) \ No newline at end of file From a45f76884fd18d50918f573490fd2d441d08b865 Mon Sep 17 00:00:00 2001 From: ziesorx Date: Thu, 25 Sep 2025 23:23:56 +0700 Subject: [PATCH 04/55] fix: make ffmpeg support --- Dockerfile.base | 117 +++++++++++++++++------------ README-hardware-acceleration.md | 127 -------------------------------- core/streaming/readers.py | 89 ++++++++-------------- 3 files changed, 102 insertions(+), 231 deletions(-) delete mode 100644 README-hardware-acceleration.md diff --git a/Dockerfile.base b/Dockerfile.base index 620f4d8..9fd9020 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -13,44 +13,39 @@ RUN apt-get update && apt-get install -y \ yasm \ nasm \ # System libraries - libgl1 \ + libgl1-mesa-glx \ libglib2.0-0 \ - libgtk-3-0 \ libgomp1 \ - # Media libraries for FFmpeg build + # Core media libraries (essential ones only) libjpeg-dev \ libpng-dev \ - libtiff-dev \ libx264-dev \ libx265-dev \ libvpx-dev \ - libfdk-aac-dev \ libmp3lame-dev \ - libopus-dev \ libv4l-dev \ - libxvidcore-dev \ - libdc1394-22-dev \ # TurboJPEG for fast JPEG encoding libturbojpeg0-dev \ - # GStreamer complete stack - libgstreamer1.0-dev \ - libgstreamer-plugins-base1.0-dev \ - libgstreamer-plugins-bad1.0-dev \ - gstreamer1.0-tools \ - gstreamer1.0-plugins-base \ - gstreamer1.0-plugins-good \ - gstreamer1.0-plugins-bad \ - gstreamer1.0-plugins-ugly \ - gstreamer1.0-libav \ - gstreamer1.0-vaapi \ - python3-gst-1.0 \ # Python development python3-dev \ python3-numpy \ - # NVIDIA driver components + && rm -rf /var/lib/apt/lists/* + +# Install CUDA development tools (required for FFmpeg CUDA compilation) +RUN apt-get update && apt-get install -y \ + cuda-nvcc-12-6 \ + libcuda1 \ + cuda-cudart-dev-12-6 \ + cuda-driver-dev-12-6 \ + || echo "CUDA development packages not available, continuing without them" && \ + rm -rf /var/lib/apt/lists/* + +# Try to install NVIDIA packages (may not be available in all environments) +RUN apt-get update && apt-get install -y \ libnvidia-encode-535 \ libnvidia-decode-535 \ - && rm -rf /var/lib/apt/lists/* + || echo "NVIDIA packages not available, continuing without them" && \ + rm -rf /var/lib/apt/lists/* # Install NVIDIA Video Codec SDK headers RUN cd /tmp && \ @@ -60,33 +55,60 @@ RUN cd /tmp && \ make install && \ rm -rf /tmp/* -# Build FFmpeg from source with full NVIDIA hardware acceleration +# Build FFmpeg from source with NVIDIA CUVID support ENV FFMPEG_VERSION=6.0 +# Ensure CUDA paths are available for FFmpeg compilation +ENV PATH="/usr/local/cuda/bin:${PATH}" +ENV LD_LIBRARY_PATH="/usr/local/cuda/lib64:${LD_LIBRARY_PATH}" RUN cd /tmp && \ wget https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.xz && \ tar xf ffmpeg-${FFMPEG_VERSION}.tar.xz && \ cd ffmpeg-${FFMPEG_VERSION} && \ - ./configure \ + # Configure with explicit CUVID support (with fallback) + (./configure \ --enable-gpl \ --enable-nonfree \ + --enable-shared \ --enable-libx264 \ --enable-libx265 \ --enable-libvpx \ - --enable-libfdk-aac \ --enable-libmp3lame \ - --enable-libopus \ --enable-cuda-nvcc \ - --enable-cuvid \ - --enable-nvenc \ - --enable-nvdec \ --enable-cuda-llvm \ + --enable-cuvid \ + --enable-nvdec \ + --enable-nvenc \ --enable-libnpp \ - --extra-cflags=-I/usr/local/cuda/include \ - --extra-ldflags=-L/usr/local/cuda/lib64 \ - --nvccflags="-gencode arch=compute_50,code=sm_50 -gencode arch=compute_52,code=sm_52 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_89,code=sm_89 -gencode arch=compute_90,code=sm_90" && \ - make -j$(nproc) && \ + --enable-decoder=h264_cuvid \ + --enable-decoder=hevc_cuvid \ + --enable-decoder=mjpeg_cuvid \ + --enable-decoder=mpeg1_cuvid \ + --enable-decoder=mpeg2_cuvid \ + --enable-decoder=mpeg4_cuvid \ + --enable-decoder=vc1_cuvid \ + --enable-encoder=h264_nvenc \ + --enable-encoder=hevc_nvenc \ + --extra-cflags="-I/usr/local/cuda/include" \ + --extra-ldflags="-L/usr/local/cuda/lib64" \ + --extra-libs="-lcuda -lcudart -lnvcuvid -lnvidia-encode" \ + --nvccflags="-gencode arch=compute_60,code=sm_60 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86" \ + || echo "CUDA configuration failed, trying basic configuration..." && \ + ./configure \ + --enable-gpl \ + --enable-nonfree \ + --enable-shared \ + --enable-libx264 \ + --enable-libx265 \ + --enable-libvpx \ + --enable-libmp3lame) \ + && make -j$(nproc) && \ make install && \ ldconfig && \ + # Verify CUVID decoders are available + echo "=== Verifying FFmpeg CUVID Support ===" && \ + ffmpeg -hide_banner -decoders 2>/dev/null | grep cuvid && \ + echo "=== Verifying FFmpeg NVENC Support ===" && \ + ffmpeg -hide_banner -encoders 2>/dev/null | grep nvenc && \ cd / && rm -rf /tmp/* # Build OpenCV from source with custom FFmpeg and full CUDA support @@ -111,15 +133,14 @@ RUN cd /tmp && \ -D WITH_CUVID=ON \ -D BUILD_opencv_cudacodec=ON \ -D WITH_FFMPEG=ON \ - -D WITH_GSTREAMER=ON \ -D WITH_LIBV4L=ON \ -D BUILD_opencv_python3=ON \ -D OPENCV_GENERATE_PKGCONFIG=ON \ -D OPENCV_ENABLE_NONFREE=ON \ -D OPENCV_EXTRA_MODULES_PATH=/tmp/opencv_contrib-${OPENCV_VERSION}/modules \ -D PYTHON3_EXECUTABLE=$(which python3) \ - -D PYTHON_INCLUDE_DIR=$(python3 -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())") \ - -D PYTHON_LIBRARY=$(python3 -c "import distutils.sysconfig as sysconfig; print(sysconfig.get_config_var('LIBDIR'))") \ + -D PYTHON_INCLUDE_DIR=$(python3 -c "import sysconfig; print(sysconfig.get_path('include'))") \ + -D PYTHON_LIBRARY=$(python3 -c "import sysconfig; print(sysconfig.get_config_var('LIBDIR'))") \ -D BUILD_EXAMPLES=OFF \ -D BUILD_TESTS=OFF \ -D BUILD_PERF_TESTS=OFF \ @@ -133,7 +154,6 @@ RUN cd /tmp && \ ENV LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/local/lib:${LD_LIBRARY_PATH}" ENV PKG_CONFIG_PATH="/usr/local/lib/pkgconfig:${PKG_CONFIG_PATH}" ENV PYTHONPATH="/usr/local/lib/python3.10/dist-packages:${PYTHONPATH}" -ENV GST_PLUGIN_PATH="/usr/lib/x86_64-linux-gnu/gstreamer-1.0" # Optimized environment variables for hardware acceleration ENV OPENCV_FFMPEG_CAPTURE_OPTIONS="rtsp_transport;tcp|hwaccel;cuda|hwaccel_device;0|video_codec;h264_cuvid|hwaccel_output_format;cuda" @@ -151,16 +171,21 @@ RUN grep -v opencv-python requirements.base.txt > requirements.tmp && \ # Verify complete hardware acceleration setup RUN echo "=== Hardware Acceleration Verification ===" && \ echo "FFmpeg Hardware Accelerators:" && \ - ffmpeg -hide_banner -hwaccels 2>/dev/null | head -10 && \ - echo "FFmpeg NVIDIA Decoders:" && \ - ffmpeg -hide_banner -decoders 2>/dev/null | grep -E "(cuvid|nvdec)" | head -5 && \ - echo "FFmpeg NVIDIA Encoders:" && \ - ffmpeg -hide_banner -encoders 2>/dev/null | grep nvenc | head -5 && \ + (ffmpeg -hide_banner -hwaccels 2>/dev/null || echo "FFmpeg hwaccels command failed") && \ + echo "" && \ + echo "FFmpeg CUVID Decoders (NVIDIA):" && \ + (ffmpeg -hide_banner -decoders 2>/dev/null | grep -E "cuvid" || echo "No CUVID decoders found") && \ + echo "" && \ + echo "FFmpeg NVENC Encoders (NVIDIA):" && \ + (ffmpeg -hide_banner -encoders 2>/dev/null | grep -E "nvenc" || echo "No NVENC encoders found") && \ + echo "" && \ + echo "Testing CUVID decoder compilation (no GPU required):" && \ + (ffmpeg -hide_banner -f lavfi -i testsrc=duration=0.1:size=64x64:rate=1 -c:v libx264 -f null - 2>/dev/null && echo "✅ FFmpeg basic functionality working" || echo "❌ FFmpeg basic test failed") && \ + echo "" && \ echo "OpenCV Configuration:" && \ - python3 -c "import cv2; print('OpenCV version:', cv2.__version__); print('CUDA devices:', cv2.cuda.getCudaEnabledDeviceCount()); build_info = cv2.getBuildInformation(); print('CUDA support:', 'CUDA' in build_info); print('CUVID support:', 'CUVID' in build_info); print('FFmpeg support:', 'FFMPEG' in build_info); print('GStreamer support:', 'GStreamer' in build_info)" && \ - echo "GStreamer NVIDIA Plugins:" && \ - gst-inspect-1.0 2>/dev/null | grep -E "(nvv4l2|nvvideo)" | head -5 || echo "GStreamer NVIDIA plugins not detected" && \ - echo "=== Verification Complete ===" + (python3 -c "import cv2; print('OpenCV version:', cv2.__version__); build_info = cv2.getBuildInformation(); print('CUDA support:', 'CUDA' in build_info); print('CUVID support:', 'CUVID' in build_info); print('FFmpeg support:', 'FFMPEG' in build_info)" || echo "OpenCV verification failed") && \ + echo "" && \ + echo "=== Verification Complete (build-time only) ===" # Set working directory WORKDIR /app diff --git a/README-hardware-acceleration.md b/README-hardware-acceleration.md deleted file mode 100644 index 69c6e09..0000000 --- a/README-hardware-acceleration.md +++ /dev/null @@ -1,127 +0,0 @@ -# Hardware Acceleration Setup - -This detector worker now includes **complete NVIDIA hardware acceleration** with FFmpeg and OpenCV built from source. - -## What's Included - -### 🔧 Complete Hardware Stack -- **FFmpeg 6.0** built from source with NVIDIA Video Codec SDK -- **OpenCV 4.8.1** built with CUDA and custom FFmpeg integration -- **GStreamer** with NVDEC/VAAPI plugins -- **TurboJPEG** for optimized JPEG encoding (3-5x faster) -- **CUDA** support for YOLO model inference - -### 🎯 Hardware Acceleration Methods (Automatic Detection) -1. **GStreamer NVDEC** - Best for RTSP streaming, lowest latency -2. **OpenCV CUDA** - Direct GPU memory access, best integration -3. **FFmpeg CUVID** - Custom build with full NVIDIA acceleration -4. **VAAPI** - Intel/AMD GPU support -5. **Software Fallback** - CPU-only as last resort - -## Build and Run - -### Single Build Script -```bash -./build-nvdec.sh -``` -**Build time**: 45-90 minutes (compiles FFmpeg + OpenCV from source) - -### Run with GPU Support -```bash -docker run --gpus all -p 8000:8000 detector-worker:complete-hw-accel -``` - -## Performance Improvements - -### Expected CPU Reduction -- **Video decoding**: 70-90% reduction (moved to GPU) -- **JPEG encoding**: 70-80% faster with TurboJPEG -- **Model inference**: GPU accelerated with CUDA -- **Overall system**: 50-80% less CPU usage - -### Profiling Results Comparison -**Before (Software Only)**: -- `cv2.imencode`: 6.5% CPU time (1.95s out of 30s) -- `psutil.cpu_percent`: 88% CPU time (idle polling) -- Video decoding: 100% CPU - -**After (Hardware Accelerated)**: -- Video decoding: GPU (~5-10% CPU overhead) -- JPEG encoding: 3-5x faster with TurboJPEG -- Model inference: GPU accelerated - -## Verification - -### Check Hardware Acceleration Support -```bash -docker run --rm --gpus all detector-worker:complete-hw-accel \ - bash -c "ffmpeg -hwaccels && python3 -c 'import cv2; build=cv2.getBuildInformation(); print(\"CUDA:\", \"CUDA\" in build); print(\"CUVID:\", \"CUVID\" in build)'" -``` - -### Runtime Logs -The application will automatically log which acceleration method is being used: -``` -Camera cam1: Successfully using GStreamer with NVDEC hardware acceleration -Camera cam2: Using FFMPEG hardware acceleration (backend: FFMPEG) -Camera cam3: Using OpenCV CUDA hardware acceleration -``` - -## Files Modified - -### Docker Configuration -- **Dockerfile.base** - Complete hardware acceleration stack -- **build-nvdec.sh** - Single build script for everything - -### Application Code -- **core/streaming/readers.py** - Multi-method hardware acceleration -- **core/utils/hardware_encoder.py** - TurboJPEG + NVENC encoding -- **core/utils/ffmpeg_detector.py** - Runtime capability detection -- **requirements.base.txt** - Added TurboJPEG, removed opencv-python - -## Architecture - -``` -Input RTSP Stream - ↓ -1. GStreamer NVDEC Pipeline (NVIDIA GPU) - rtspsrc → nvv4l2decoder → nvvideoconvert → OpenCV - ↓ -2. OpenCV CUDA Backend (NVIDIA GPU) - OpenCV with CUDA acceleration - ↓ -3. FFmpeg CUVID (NVIDIA GPU) - Custom FFmpeg with h264_cuvid decoder - ↓ -4. VAAPI (Intel/AMD GPU) - Hardware acceleration for non-NVIDIA - ↓ -5. Software Fallback (CPU) - Standard OpenCV software decoding -``` - -## Benefits - -### For Development -- **Single Dockerfile.base** - Everything consolidated -- **Automatic detection** - No manual configuration needed -- **Graceful fallback** - Works without GPU for development - -### For Production -- **Maximum performance** - Uses best available acceleration -- **GPU memory efficiency** - Direct GPU-to-GPU pipeline -- **Lower latency** - Hardware decoding + CUDA inference -- **Reduced CPU load** - Frees CPU for other tasks - -## Troubleshooting - -### Build Issues -- Ensure NVIDIA Docker runtime is installed -- Check CUDA 12.6 compatibility with your GPU -- Build takes 45-90 minutes - be patient - -### Runtime Issues -- Verify `nvidia-smi` works in container -- Check logs for acceleration method being used -- Fallback to software decoding is automatic - -This setup provides **production-ready hardware acceleration** with automatic detection and graceful fallback for maximum compatibility. \ No newline at end of file diff --git a/core/streaming/readers.py b/core/streaming/readers.py index 377db56..9a3db6d 100644 --- a/core/streaming/readers.py +++ b/core/streaming/readers.py @@ -166,40 +166,17 @@ class RTSPReader: logger.info(f"RTSP reader thread ended for camera {self.camera_id}") def _initialize_capture(self) -> bool: - """Initialize video capture with hardware acceleration (NVDEC) for 1280x720@6fps.""" + """Initialize video capture with FFmpeg hardware acceleration (CUVID/NVDEC) for 1280x720@6fps.""" try: # Release previous capture if exists if self.cap: self.cap.release() time.sleep(0.5) - logger.info(f"Initializing capture for camera {self.camera_id} with hardware acceleration") + logger.info(f"Initializing capture for camera {self.camera_id} with FFmpeg hardware acceleration") hw_accel_success = False - # Method 1: Try GStreamer with NVDEC (most efficient on NVIDIA GPUs) - if not hw_accel_success: - try: - # Build GStreamer pipeline for NVIDIA hardware decoding - gst_pipeline = ( - f"rtspsrc location={self.rtsp_url} protocols=tcp latency=100 ! " - "rtph264depay ! h264parse ! " - "nvv4l2decoder ! " # NVIDIA hardware decoder - "nvvideoconvert ! " # NVIDIA hardware color conversion - "video/x-raw,format=BGRx,width=1280,height=720 ! " - "videoconvert ! " - "video/x-raw,format=BGR ! " - "appsink max-buffers=1 drop=true sync=false" - ) - logger.info(f"Attempting GStreamer NVDEC pipeline for camera {self.camera_id}") - self.cap = cv2.VideoCapture(gst_pipeline, cv2.CAP_GSTREAMER) - - if self.cap.isOpened(): - hw_accel_success = True - logger.info(f"Camera {self.camera_id}: Successfully using GStreamer with NVDEC hardware acceleration") - except Exception as e: - logger.debug(f"Camera {self.camera_id}: GStreamer NVDEC not available: {e}") - - # Method 2: Try OpenCV CUDA VideoReader (if built with CUVID support) + # Method 1: Try OpenCV CUDA VideoReader (if built with CUVID support) if not hw_accel_success: try: # Check if OpenCV was built with CUDA codec support @@ -220,7 +197,7 @@ class RTSPReader: except Exception as e: logger.debug(f"Camera {self.camera_id}: OpenCV CUDA not available: {e}") - # Method 3: Try FFMPEG with optimal hardware acceleration (CUVID/VAAPI) + # Method 2: Try FFmpeg with optimal hardware acceleration (CUVID/NVDEC) if not hw_accel_success: try: from core.utils.ffmpeg_detector import get_optimal_rtsp_options @@ -230,7 +207,7 @@ class RTSPReader: optimal_options = get_optimal_rtsp_options(self.rtsp_url) os.environ['OPENCV_FFMPEG_CAPTURE_OPTIONS'] = optimal_options - logger.info(f"Attempting FFMPEG with detected hardware acceleration for camera {self.camera_id}") + logger.info(f"Attempting FFmpeg with detected hardware acceleration for camera {self.camera_id}") logger.debug(f"Camera {self.camera_id}: Using FFmpeg options: {optimal_options}") self.cap = cv2.VideoCapture(self.rtsp_url, cv2.CAP_FFMPEG) @@ -239,45 +216,41 @@ class RTSPReader: hw_accel_success = True # Try to get backend info to confirm hardware acceleration backend = self.cap.getBackendName() - logger.info(f"Camera {self.camera_id}: Using FFMPEG hardware acceleration (backend: {backend})") + logger.info(f"Camera {self.camera_id}: Using FFmpeg hardware acceleration (backend: {backend})") except Exception as e: - logger.debug(f"Camera {self.camera_id}: FFMPEG hardware acceleration not available: {e}") + logger.debug(f"Camera {self.camera_id}: FFmpeg optimal hardware acceleration not available: {e}") - # Fallback to basic CUVID - try: - import os - os.environ['OPENCV_FFMPEG_CAPTURE_OPTIONS'] = 'video_codec;h264_cuvid|rtsp_transport;tcp|hwaccel;cuda' - self.cap = cv2.VideoCapture(self.rtsp_url, cv2.CAP_FFMPEG) - - if self.cap.isOpened(): - hw_accel_success = True - logger.info(f"Camera {self.camera_id}: Using basic FFMPEG CUVID hardware acceleration") - except Exception as e2: - logger.debug(f"Camera {self.camera_id}: Basic CUVID also failed: {e2}") - - # Method 4: Try VAAPI hardware acceleration (for Intel/AMD GPUs) + # Method 3: Try FFmpeg with basic NVIDIA CUVID if not hw_accel_success: try: - gst_pipeline = ( - f"rtspsrc location={self.rtsp_url} protocols=tcp latency=100 ! " - "rtph264depay ! h264parse ! " - "vaapih264dec ! " # VAAPI hardware decoder - "vaapipostproc ! " - "video/x-raw,format=BGRx,width=1280,height=720 ! " - "videoconvert ! " - "video/x-raw,format=BGR ! " - "appsink max-buffers=1 drop=true sync=false" - ) - logger.info(f"Attempting GStreamer VAAPI pipeline for camera {self.camera_id}") - self.cap = cv2.VideoCapture(gst_pipeline, cv2.CAP_GSTREAMER) + import os + os.environ['OPENCV_FFMPEG_CAPTURE_OPTIONS'] = 'video_codec;h264_cuvid|rtsp_transport;tcp|hwaccel;cuda|hwaccel_device;0' + + logger.info(f"Attempting FFmpeg with basic CUVID for camera {self.camera_id}") + self.cap = cv2.VideoCapture(self.rtsp_url, cv2.CAP_FFMPEG) if self.cap.isOpened(): hw_accel_success = True - logger.info(f"Camera {self.camera_id}: Successfully using GStreamer with VAAPI hardware acceleration") + logger.info(f"Camera {self.camera_id}: Using FFmpeg CUVID hardware acceleration") except Exception as e: - logger.debug(f"Camera {self.camera_id}: GStreamer VAAPI not available: {e}") + logger.debug(f"Camera {self.camera_id}: FFmpeg CUVID not available: {e}") - # Fallback: Standard FFMPEG with software decoding + # Method 4: Try FFmpeg with VAAPI (Intel/AMD GPUs) + if not hw_accel_success: + try: + import os + os.environ['OPENCV_FFMPEG_CAPTURE_OPTIONS'] = 'hwaccel;vaapi|hwaccel_device;/dev/dri/renderD128|video_codec;h264|rtsp_transport;tcp' + + logger.info(f"Attempting FFmpeg with VAAPI for camera {self.camera_id}") + self.cap = cv2.VideoCapture(self.rtsp_url, cv2.CAP_FFMPEG) + + if self.cap.isOpened(): + hw_accel_success = True + logger.info(f"Camera {self.camera_id}: Using FFmpeg VAAPI hardware acceleration") + except Exception as e: + logger.debug(f"Camera {self.camera_id}: FFmpeg VAAPI not available: {e}") + + # Fallback: Standard FFmpeg with software decoding if not hw_accel_success: logger.warning(f"Camera {self.camera_id}: Hardware acceleration not available, falling back to software decoding") import os From ff56c1b666072a1f6fd1f8f0eb52a62f8e0918a4 Mon Sep 17 00:00:00 2001 From: ziesorx Date: Thu, 25 Sep 2025 23:36:07 +0700 Subject: [PATCH 05/55] fix: dockerfile base --- Dockerfile.base | 75 +++++++++++++++++-------------------------------- 1 file changed, 25 insertions(+), 50 deletions(-) diff --git a/Dockerfile.base b/Dockerfile.base index 9fd9020..557a88e 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -47,7 +47,13 @@ RUN apt-get update && apt-get install -y \ || echo "NVIDIA packages not available, continuing without them" && \ rm -rf /var/lib/apt/lists/* -# Install NVIDIA Video Codec SDK headers +# Use pre-built FFmpeg with CUDA support using the build script +ENV FFMPEG_BUILD_SCRIPT_VERSION=1.43 +# Ensure CUDA paths are available +ENV PATH="/usr/local/cuda/bin:${PATH}" +ENV LD_LIBRARY_PATH="/usr/local/cuda/lib64:${LD_LIBRARY_PATH}" + +# Install NVIDIA Video Codec SDK headers first RUN cd /tmp && \ wget https://github.com/FFmpeg/nv-codec-headers/archive/refs/tags/n12.1.14.0.zip && \ unzip n12.1.14.0.zip && \ @@ -55,60 +61,29 @@ RUN cd /tmp && \ make install && \ rm -rf /tmp/* -# Build FFmpeg from source with NVIDIA CUVID support -ENV FFMPEG_VERSION=6.0 -# Ensure CUDA paths are available for FFmpeg compilation -ENV PATH="/usr/local/cuda/bin:${PATH}" -ENV LD_LIBRARY_PATH="/usr/local/cuda/lib64:${LD_LIBRARY_PATH}" +# Build FFmpeg using the well-maintained build script with CUDA support RUN cd /tmp && \ - wget https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.xz && \ - tar xf ffmpeg-${FFMPEG_VERSION}.tar.xz && \ - cd ffmpeg-${FFMPEG_VERSION} && \ - # Configure with explicit CUVID support (with fallback) - (./configure \ - --enable-gpl \ - --enable-nonfree \ - --enable-shared \ - --enable-libx264 \ - --enable-libx265 \ - --enable-libvpx \ - --enable-libmp3lame \ - --enable-cuda-nvcc \ - --enable-cuda-llvm \ - --enable-cuvid \ - --enable-nvdec \ - --enable-nvenc \ - --enable-libnpp \ - --enable-decoder=h264_cuvid \ - --enable-decoder=hevc_cuvid \ - --enable-decoder=mjpeg_cuvid \ - --enable-decoder=mpeg1_cuvid \ - --enable-decoder=mpeg2_cuvid \ - --enable-decoder=mpeg4_cuvid \ - --enable-decoder=vc1_cuvid \ - --enable-encoder=h264_nvenc \ - --enable-encoder=hevc_nvenc \ - --extra-cflags="-I/usr/local/cuda/include" \ - --extra-ldflags="-L/usr/local/cuda/lib64" \ - --extra-libs="-lcuda -lcudart -lnvcuvid -lnvidia-encode" \ - --nvccflags="-gencode arch=compute_60,code=sm_60 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86" \ - || echo "CUDA configuration failed, trying basic configuration..." && \ - ./configure \ - --enable-gpl \ - --enable-nonfree \ - --enable-shared \ - --enable-libx264 \ - --enable-libx265 \ - --enable-libvpx \ - --enable-libmp3lame) \ - && make -j$(nproc) && \ - make install && \ + echo "Building FFmpeg with CUDA support using build script..." && \ + curl -sL "https://raw.githubusercontent.com/markus-perl/ffmpeg-build-script/master/build-ffmpeg" -o build-ffmpeg && \ + chmod +x build-ffmpeg && \ + # Configure the build script for CUDA support + SKIPINSTALL=yes \ + AUTOINSTALL=yes \ + ./build-ffmpeg \ + --build \ + --enable-gpl-and-non-free \ + --latest \ + --cuda \ + && \ + # Copy built binaries to system paths + cp workspace/bin/* /usr/local/bin/ && \ + cp workspace/lib/* /usr/local/lib/ && \ ldconfig && \ # Verify CUVID decoders are available echo "=== Verifying FFmpeg CUVID Support ===" && \ - ffmpeg -hide_banner -decoders 2>/dev/null | grep cuvid && \ + (ffmpeg -hide_banner -decoders 2>/dev/null | grep cuvid || echo "No CUVID decoders found") && \ echo "=== Verifying FFmpeg NVENC Support ===" && \ - ffmpeg -hide_banner -encoders 2>/dev/null | grep nvenc && \ + (ffmpeg -hide_banner -encoders 2>/dev/null | grep nvenc || echo "No NVENC encoders found") && \ cd / && rm -rf /tmp/* # Build OpenCV from source with custom FFmpeg and full CUDA support From 47d4fa6b8f10099eb04e06d454ec84428e2220c2 Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Thu, 25 Sep 2025 23:48:35 +0700 Subject: [PATCH 06/55] refactor: streamline FFmpeg installation process and remove unnecessary CUDA development tools --- Dockerfile.base | 102 +++++------------------------------------------- 1 file changed, 10 insertions(+), 92 deletions(-) diff --git a/Dockerfile.base b/Dockerfile.base index 557a88e..e2baf08 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -31,24 +31,7 @@ RUN apt-get update && apt-get install -y \ python3-numpy \ && rm -rf /var/lib/apt/lists/* -# Install CUDA development tools (required for FFmpeg CUDA compilation) -RUN apt-get update && apt-get install -y \ - cuda-nvcc-12-6 \ - libcuda1 \ - cuda-cudart-dev-12-6 \ - cuda-driver-dev-12-6 \ - || echo "CUDA development packages not available, continuing without them" && \ - rm -rf /var/lib/apt/lists/* - -# Try to install NVIDIA packages (may not be available in all environments) -RUN apt-get update && apt-get install -y \ - libnvidia-encode-535 \ - libnvidia-decode-535 \ - || echo "NVIDIA packages not available, continuing without them" && \ - rm -rf /var/lib/apt/lists/* - -# Use pre-built FFmpeg with CUDA support using the build script -ENV FFMPEG_BUILD_SCRIPT_VERSION=1.43 +# Install prebuilt FFmpeg with CUDA support # Ensure CUDA paths are available ENV PATH="/usr/local/cuda/bin:${PATH}" ENV LD_LIBRARY_PATH="/usr/local/cuda/lib64:${LD_LIBRARY_PATH}" @@ -61,23 +44,16 @@ RUN cd /tmp && \ make install && \ rm -rf /tmp/* -# Build FFmpeg using the well-maintained build script with CUDA support +# Download and install prebuilt FFmpeg with CUDA support RUN cd /tmp && \ - echo "Building FFmpeg with CUDA support using build script..." && \ - curl -sL "https://raw.githubusercontent.com/markus-perl/ffmpeg-build-script/master/build-ffmpeg" -o build-ffmpeg && \ - chmod +x build-ffmpeg && \ - # Configure the build script for CUDA support - SKIPINSTALL=yes \ - AUTOINSTALL=yes \ - ./build-ffmpeg \ - --build \ - --enable-gpl-and-non-free \ - --latest \ - --cuda \ - && \ - # Copy built binaries to system paths - cp workspace/bin/* /usr/local/bin/ && \ - cp workspace/lib/* /usr/local/lib/ && \ + echo "Installing prebuilt FFmpeg with CUDA support..." && \ + wget https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz && \ + tar -xf ffmpeg-master-latest-linux64-gpl.tar.xz && \ + cd ffmpeg-master-latest-linux64-gpl && \ + # Copy binaries to system paths + cp bin/* /usr/local/bin/ && \ + cp -r lib/* /usr/local/lib/ && \ + cp -r include/* /usr/local/include/ && \ ldconfig && \ # Verify CUVID decoders are available echo "=== Verifying FFmpeg CUVID Support ===" && \ @@ -86,45 +62,6 @@ RUN cd /tmp && \ (ffmpeg -hide_banner -encoders 2>/dev/null | grep nvenc || echo "No NVENC encoders found") && \ cd / && rm -rf /tmp/* -# Build OpenCV from source with custom FFmpeg and full CUDA support -ENV OPENCV_VERSION=4.8.1 -RUN cd /tmp && \ - wget -O opencv.zip https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.zip && \ - wget -O opencv_contrib.zip https://github.com/opencv/opencv_contrib/archive/${OPENCV_VERSION}.zip && \ - unzip opencv.zip && \ - unzip opencv_contrib.zip && \ - cd opencv-${OPENCV_VERSION} && \ - mkdir build && cd build && \ - PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH \ - cmake -D CMAKE_BUILD_TYPE=RELEASE \ - -D CMAKE_INSTALL_PREFIX=/usr/local \ - -D WITH_CUDA=ON \ - -D WITH_CUDNN=ON \ - -D OPENCV_DNN_CUDA=ON \ - -D ENABLE_FAST_MATH=ON \ - -D CUDA_FAST_MATH=ON \ - -D WITH_CUBLAS=ON \ - -D WITH_NVCUVID=ON \ - -D WITH_CUVID=ON \ - -D BUILD_opencv_cudacodec=ON \ - -D WITH_FFMPEG=ON \ - -D WITH_LIBV4L=ON \ - -D BUILD_opencv_python3=ON \ - -D OPENCV_GENERATE_PKGCONFIG=ON \ - -D OPENCV_ENABLE_NONFREE=ON \ - -D OPENCV_EXTRA_MODULES_PATH=/tmp/opencv_contrib-${OPENCV_VERSION}/modules \ - -D PYTHON3_EXECUTABLE=$(which python3) \ - -D PYTHON_INCLUDE_DIR=$(python3 -c "import sysconfig; print(sysconfig.get_path('include'))") \ - -D PYTHON_LIBRARY=$(python3 -c "import sysconfig; print(sysconfig.get_config_var('LIBDIR'))") \ - -D BUILD_EXAMPLES=OFF \ - -D BUILD_TESTS=OFF \ - -D BUILD_PERF_TESTS=OFF \ - .. && \ - make -j$(nproc) && \ - make install && \ - ldconfig && \ - cd / && rm -rf /tmp/* - # Set environment variables for maximum hardware acceleration ENV LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/local/lib:${LD_LIBRARY_PATH}" ENV PKG_CONFIG_PATH="/usr/local/lib/pkgconfig:${PKG_CONFIG_PATH}" @@ -143,25 +80,6 @@ RUN grep -v opencv-python requirements.base.txt > requirements.tmp && \ mv requirements.tmp requirements.base.txt && \ pip install --no-cache-dir -r requirements.base.txt -# Verify complete hardware acceleration setup -RUN echo "=== Hardware Acceleration Verification ===" && \ - echo "FFmpeg Hardware Accelerators:" && \ - (ffmpeg -hide_banner -hwaccels 2>/dev/null || echo "FFmpeg hwaccels command failed") && \ - echo "" && \ - echo "FFmpeg CUVID Decoders (NVIDIA):" && \ - (ffmpeg -hide_banner -decoders 2>/dev/null | grep -E "cuvid" || echo "No CUVID decoders found") && \ - echo "" && \ - echo "FFmpeg NVENC Encoders (NVIDIA):" && \ - (ffmpeg -hide_banner -encoders 2>/dev/null | grep -E "nvenc" || echo "No NVENC encoders found") && \ - echo "" && \ - echo "Testing CUVID decoder compilation (no GPU required):" && \ - (ffmpeg -hide_banner -f lavfi -i testsrc=duration=0.1:size=64x64:rate=1 -c:v libx264 -f null - 2>/dev/null && echo "✅ FFmpeg basic functionality working" || echo "❌ FFmpeg basic test failed") && \ - echo "" && \ - echo "OpenCV Configuration:" && \ - (python3 -c "import cv2; print('OpenCV version:', cv2.__version__); build_info = cv2.getBuildInformation(); print('CUDA support:', 'CUDA' in build_info); print('CUVID support:', 'CUVID' in build_info); print('FFmpeg support:', 'FFMPEG' in build_info)" || echo "OpenCV verification failed") && \ - echo "" && \ - echo "=== Verification Complete (build-time only) ===" - # Set working directory WORKDIR /app From dc1db635d0a0b88e47cda200a069ebf05af4c3d8 Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Thu, 25 Sep 2025 23:56:29 +0700 Subject: [PATCH 07/55] fix: remove unnecessary copying of FFmpeg library and include files --- Dockerfile.base | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile.base b/Dockerfile.base index e2baf08..8c104d2 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -52,8 +52,6 @@ RUN cd /tmp && \ cd ffmpeg-master-latest-linux64-gpl && \ # Copy binaries to system paths cp bin/* /usr/local/bin/ && \ - cp -r lib/* /usr/local/lib/ && \ - cp -r include/* /usr/local/include/ && \ ldconfig && \ # Verify CUVID decoders are available echo "=== Verifying FFmpeg CUVID Support ===" && \ From 719d16ae4d32c25c35a09bdd4e8fe1a7c9b83488 Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Fri, 26 Sep 2025 00:07:48 +0700 Subject: [PATCH 08/55] refactor: simplify frame handling by removing stream type management and enhancing validation --- .claude/settings.local.json | 9 +++ core/streaming/buffers.py | 134 +++++++----------------------------- core/streaming/manager.py | 41 +---------- core/streaming/readers.py | 49 ++++--------- 4 files changed, 51 insertions(+), 182 deletions(-) create mode 100644 .claude/settings.local.json diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 0000000..b06024d --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,9 @@ +{ + "permissions": { + "allow": [ + "Bash(dir:*)" + ], + "deny": [], + "ask": [] + } +} \ No newline at end of file diff --git a/core/streaming/buffers.py b/core/streaming/buffers.py index 602e028..fd29fbb 100644 --- a/core/streaming/buffers.py +++ b/core/streaming/buffers.py @@ -9,53 +9,25 @@ import logging import numpy as np from typing import Optional, Dict, Any, Tuple from collections import defaultdict -from enum import Enum logger = logging.getLogger(__name__) -class StreamType(Enum): - """Stream type enumeration.""" - RTSP = "rtsp" # 1280x720 @ 6fps - HTTP = "http" # 2560x1440 high quality - - class FrameBuffer: - """Thread-safe frame buffer optimized for different stream types.""" + """Thread-safe frame buffer for all camera streams.""" def __init__(self, max_age_seconds: int = 5): self.max_age_seconds = max_age_seconds self._frames: Dict[str, Dict[str, Any]] = {} - self._stream_types: Dict[str, StreamType] = {} self._lock = threading.RLock() - # Stream-specific settings - self.rtsp_config = { - 'width': 1280, - 'height': 720, - 'fps': 6, - 'max_size_mb': 3 # 1280x720x3 bytes = ~2.6MB - } - self.http_config = { - 'width': 2560, - 'height': 1440, - 'max_size_mb': 10 - } - - def put_frame(self, camera_id: str, frame: np.ndarray, stream_type: Optional[StreamType] = None): - """Store a frame for the given camera ID with type-specific validation.""" + def put_frame(self, camera_id: str, frame: np.ndarray): + """Store a frame for the given camera ID.""" with self._lock: - # Detect stream type if not provided - if stream_type is None: - stream_type = self._detect_stream_type(frame) - - # Store stream type - self._stream_types[camera_id] = stream_type - - # Validate frame based on stream type - if not self._validate_frame(frame, stream_type): - logger.warning(f"Frame validation failed for camera {camera_id} ({stream_type.value})") + # Validate frame + if not self._validate_frame(frame): + logger.warning(f"Frame validation failed for camera {camera_id}") return self._frames[camera_id] = { @@ -63,14 +35,9 @@ class FrameBuffer: 'timestamp': time.time(), 'shape': frame.shape, 'dtype': str(frame.dtype), - 'stream_type': stream_type.value, 'size_mb': frame.nbytes / (1024 * 1024) } - # Commented out verbose frame storage logging - # logger.debug(f"Stored {stream_type.value} frame for camera {camera_id}: " - # f"{frame.shape[1]}x{frame.shape[0]}, {frame.nbytes / (1024 * 1024):.2f}MB") - def get_frame(self, camera_id: str) -> Optional[np.ndarray]: """Get the latest frame for the given camera ID.""" with self._lock: @@ -84,8 +51,6 @@ class FrameBuffer: if age > self.max_age_seconds: logger.debug(f"Frame for camera {camera_id} is {age:.1f}s old, discarding") del self._frames[camera_id] - if camera_id in self._stream_types: - del self._stream_types[camera_id] return None return frame_data['frame'].copy() @@ -101,8 +66,6 @@ class FrameBuffer: if age > self.max_age_seconds: del self._frames[camera_id] - if camera_id in self._stream_types: - del self._stream_types[camera_id] return None return { @@ -110,7 +73,6 @@ class FrameBuffer: 'age': age, 'shape': frame_data['shape'], 'dtype': frame_data['dtype'], - 'stream_type': frame_data.get('stream_type', 'unknown'), 'size_mb': frame_data.get('size_mb', 0) } @@ -123,8 +85,6 @@ class FrameBuffer: with self._lock: if camera_id in self._frames: del self._frames[camera_id] - if camera_id in self._stream_types: - del self._stream_types[camera_id] logger.debug(f"Cleared frames for camera {camera_id}") def clear_all(self): @@ -132,7 +92,6 @@ class FrameBuffer: with self._lock: count = len(self._frames) self._frames.clear() - self._stream_types.clear() logger.debug(f"Cleared all frames ({count} cameras)") def get_camera_list(self) -> list: @@ -152,8 +111,6 @@ class FrameBuffer: # Clean up expired frames for camera_id in expired_cameras: del self._frames[camera_id] - if camera_id in self._stream_types: - del self._stream_types[camera_id] return valid_cameras @@ -165,15 +122,12 @@ class FrameBuffer: 'total_cameras': len(self._frames), 'valid_cameras': 0, 'expired_cameras': 0, - 'rtsp_cameras': 0, - 'http_cameras': 0, 'total_memory_mb': 0, 'cameras': {} } for camera_id, frame_data in self._frames.items(): age = current_time - frame_data['timestamp'] - stream_type = frame_data.get('stream_type', 'unknown') size_mb = frame_data.get('size_mb', 0) if age <= self.max_age_seconds: @@ -181,11 +135,6 @@ class FrameBuffer: else: stats['expired_cameras'] += 1 - if stream_type == StreamType.RTSP.value: - stats['rtsp_cameras'] += 1 - elif stream_type == StreamType.HTTP.value: - stats['http_cameras'] += 1 - stats['total_memory_mb'] += size_mb stats['cameras'][camera_id] = { @@ -193,74 +142,45 @@ class FrameBuffer: 'valid': age <= self.max_age_seconds, 'shape': frame_data['shape'], 'dtype': frame_data['dtype'], - 'stream_type': stream_type, 'size_mb': size_mb } return stats - def _detect_stream_type(self, frame: np.ndarray) -> StreamType: - """Detect stream type based on frame dimensions.""" - h, w = frame.shape[:2] - - # Check if it matches RTSP dimensions (1280x720) - if w == self.rtsp_config['width'] and h == self.rtsp_config['height']: - return StreamType.RTSP - - # Check if it matches HTTP dimensions (2560x1440) or close to it - if w >= 2000 and h >= 1000: - return StreamType.HTTP - - # Default based on size - if w <= 1920 and h <= 1080: - return StreamType.RTSP - else: - return StreamType.HTTP - - def _validate_frame(self, frame: np.ndarray, stream_type: StreamType) -> bool: - """Validate frame based on stream type.""" + def _validate_frame(self, frame: np.ndarray) -> bool: + """Validate frame - basic validation for any stream type.""" if frame is None or frame.size == 0: return False h, w = frame.shape[:2] size_mb = frame.nbytes / (1024 * 1024) - if stream_type == StreamType.RTSP: - config = self.rtsp_config - # Allow some tolerance for RTSP streams - if abs(w - config['width']) > 100 or abs(h - config['height']) > 100: - logger.warning(f"RTSP frame size mismatch: {w}x{h} (expected {config['width']}x{config['height']})") - if size_mb > config['max_size_mb']: - logger.warning(f"RTSP frame too large: {size_mb:.2f}MB (max {config['max_size_mb']}MB)") - return False + # Basic size validation - reject extremely large frames regardless of type + max_size_mb = 50 # Generous limit for any frame type + if size_mb > max_size_mb: + logger.warning(f"Frame too large: {size_mb:.2f}MB (max {max_size_mb}MB) for {w}x{h}") + return False - elif stream_type == StreamType.HTTP: - config = self.http_config - # More flexible for HTTP snapshots - if size_mb > config['max_size_mb']: - logger.warning(f"HTTP snapshot too large: {size_mb:.2f}MB (max {config['max_size_mb']}MB)") - return False + # Basic dimension validation + if w < 100 or h < 100: + logger.warning(f"Frame too small: {w}x{h}") + return False return True class CacheBuffer: - """Enhanced frame cache with support for cropping and optimized for different formats.""" + """Enhanced frame cache with support for cropping.""" def __init__(self, max_age_seconds: int = 10): self.frame_buffer = FrameBuffer(max_age_seconds) self._crop_cache: Dict[str, Dict[str, Any]] = {} self._cache_lock = threading.RLock() + self.jpeg_quality = 95 # High quality for all frames - # Quality settings for different stream types - self.jpeg_quality = { - StreamType.RTSP: 90, # Good quality for 720p - StreamType.HTTP: 95 # High quality for 2K - } - - def put_frame(self, camera_id: str, frame: np.ndarray, stream_type: Optional[StreamType] = None): + def put_frame(self, camera_id: str, frame: np.ndarray): """Store a frame and clear any associated crop cache.""" - self.frame_buffer.put_frame(camera_id, frame, stream_type) + self.frame_buffer.put_frame(camera_id, frame) # Clear crop cache for this camera since we have a new frame with self._cache_lock: @@ -325,21 +245,15 @@ class CacheBuffer: def get_frame_as_jpeg(self, camera_id: str, crop_coords: Optional[Tuple[int, int, int, int]] = None, quality: Optional[int] = None) -> Optional[bytes]: - """Get frame as JPEG bytes with format-specific quality settings.""" + """Get frame as JPEG bytes.""" frame = self.get_frame(camera_id, crop_coords) if frame is None: return None try: - # Determine quality based on stream type if not specified + # Use specified quality or default if quality is None: - frame_info = self.frame_buffer.get_frame_info(camera_id) - if frame_info: - stream_type_str = frame_info.get('stream_type', StreamType.RTSP.value) - stream_type = StreamType.RTSP if stream_type_str == StreamType.RTSP.value else StreamType.HTTP - quality = self.jpeg_quality[stream_type] - else: - quality = 90 # Default + quality = self.jpeg_quality # Encode as JPEG with specified quality encode_params = [cv2.IMWRITE_JPEG_QUALITY, quality] diff --git a/core/streaming/manager.py b/core/streaming/manager.py index 7bd44c1..1e3719f 100644 --- a/core/streaming/manager.py +++ b/core/streaming/manager.py @@ -10,7 +10,7 @@ from dataclasses import dataclass from collections import defaultdict from .readers import RTSPReader, HTTPSnapshotReader -from .buffers import shared_cache_buffer, StreamType +from .buffers import shared_cache_buffer from ..tracking.integration import TrackingPipelineIntegration @@ -177,12 +177,8 @@ class StreamManager: def _frame_callback(self, camera_id: str, frame): """Callback for when a new frame is available.""" try: - # Detect stream type based on frame dimensions - stream_type = self._detect_stream_type(frame) - - # Store frame in shared buffer with stream type - shared_cache_buffer.put_frame(camera_id, frame, stream_type) - + # Store frame in shared buffer + shared_cache_buffer.put_frame(camera_id, frame) # Process tracking for subscriptions with tracking integration self._process_tracking_for_camera(camera_id, frame) @@ -404,26 +400,6 @@ class StreamManager: stats[subscription_id] = subscription_info.tracking_integration.get_statistics() return stats - def _detect_stream_type(self, frame) -> StreamType: - """Detect stream type based on frame dimensions.""" - if frame is None: - return StreamType.RTSP # Default - - h, w = frame.shape[:2] - - # RTSP: 1280x720 - if w == 1280 and h == 720: - return StreamType.RTSP - - # HTTP: 2560x1440 or larger - if w >= 2000 and h >= 1000: - return StreamType.HTTP - - # Default based on size - if w <= 1920 and h <= 1080: - return StreamType.RTSP - else: - return StreamType.HTTP def get_stats(self) -> Dict[str, Any]: """Get comprehensive streaming statistics.""" @@ -431,22 +407,11 @@ class StreamManager: buffer_stats = shared_cache_buffer.get_stats() tracking_stats = self.get_tracking_stats() - # Add stream type information - stream_types = {} - for camera_id in self._streams.keys(): - if isinstance(self._streams[camera_id], RTSPReader): - stream_types[camera_id] = 'rtsp' - elif isinstance(self._streams[camera_id], HTTPSnapshotReader): - stream_types[camera_id] = 'http' - else: - stream_types[camera_id] = 'unknown' - return { 'active_subscriptions': len(self._subscriptions), 'active_streams': len(self._streams), 'cameras_with_subscribers': len(self._camera_subscribers), 'max_streams': self.max_streams, - 'stream_types': stream_types, 'subscriptions_by_camera': { camera_id: len(subscribers) for camera_id, subscribers in self._camera_subscribers.items() diff --git a/core/streaming/readers.py b/core/streaming/readers.py index 9a3db6d..53c9643 100644 --- a/core/streaming/readers.py +++ b/core/streaming/readers.py @@ -37,7 +37,6 @@ class RTSPReader: self.expected_fps = 6 # Frame processing parameters - self.frame_interval = 1.0 / self.expected_fps # ~167ms for 6fps self.error_recovery_delay = 5.0 # Increased from 2.0 for stability self.max_consecutive_errors = 30 # Increased from 10 to handle network jitter self.stream_timeout = 30.0 @@ -72,7 +71,6 @@ class RTSPReader: frame_count = 0 last_log_time = time.time() last_successful_frame_time = time.time() - last_frame_time = 0 while not self.stop_event.is_set(): try: @@ -90,12 +88,7 @@ class RTSPReader: last_successful_frame_time = time.time() continue - # Rate limiting for 6fps - current_time = time.time() - if current_time - last_frame_time < self.frame_interval: - time.sleep(0.01) # Small sleep to avoid busy waiting - continue - + # Read frame immediately without rate limiting for minimum latency ret, frame = self.cap.read() if not ret or frame is None: @@ -118,15 +111,10 @@ class RTSPReader: time.sleep(sleep_time) continue - # Validate frame dimensions - if frame.shape[1] != self.expected_width or frame.shape[0] != self.expected_height: - logger.warning(f"Camera {self.camera_id}: Unexpected frame dimensions {frame.shape[1]}x{frame.shape[0]}") - # Try to resize if dimensions are wrong - if frame.shape[1] > 0 and frame.shape[0] > 0: - frame = cv2.resize(frame, (self.expected_width, self.expected_height)) - else: - consecutive_errors += 1 - continue + # Accept any valid frame dimensions - don't force specific resolution + if frame.shape[1] <= 0 or frame.shape[0] <= 0: + consecutive_errors += 1 + continue # Check for corrupted frames (all black, all white, excessive noise) if self._is_frame_corrupted(frame): @@ -138,7 +126,6 @@ class RTSPReader: consecutive_errors = 0 frame_count += 1 last_successful_frame_time = time.time() - last_frame_time = current_time # Call frame callback if self.frame_callback: @@ -148,6 +135,7 @@ class RTSPReader: logger.error(f"Camera {self.camera_id}: Frame callback error: {e}") # Log progress every 30 seconds + current_time = time.time() if current_time - last_log_time >= 30: logger.info(f"Camera {self.camera_id}: {frame_count} frames processed") last_log_time = current_time @@ -261,14 +249,12 @@ class RTSPReader: logger.error(f"Failed to open stream for camera {self.camera_id}") return False - # Set capture properties for 1280x720@6fps - self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, self.expected_width) - self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, self.expected_height) - self.cap.set(cv2.CAP_PROP_FPS, self.expected_fps) + # Don't force resolution/fps - let the stream determine its natural specs + # The camera will provide whatever resolution/fps it supports - # Set moderate buffer to handle network jitter while avoiding excessive latency - # Buffer of 3 frames provides resilience without major delay - self.cap.set(cv2.CAP_PROP_BUFFERSIZE, 3) + # Set minimal buffer for lowest latency - single frame buffer + # This ensures we always get the most recent frame + self.cap.set(cv2.CAP_PROP_BUFFERSIZE, 1) # Set FFMPEG options for better H.264 handling self.cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*'H264')) @@ -405,15 +391,10 @@ class HTTPSnapshotReader: time.sleep(min(2.0, interval_seconds)) continue - # Validate image dimensions - if frame.shape[1] != self.expected_width or frame.shape[0] != self.expected_height: - logger.info(f"Camera {self.camera_id}: Snapshot dimensions {frame.shape[1]}x{frame.shape[0]} " - f"(expected {self.expected_width}x{self.expected_height})") - # Resize if needed (maintaining aspect ratio for high quality) - if frame.shape[1] > 0 and frame.shape[0] > 0: - # Only resize if significantly different - if abs(frame.shape[1] - self.expected_width) > 100: - frame = self._resize_maintain_aspect(frame, self.expected_width, self.expected_height) + # Accept any valid image dimensions - don't force specific resolution + if frame.shape[1] <= 0 or frame.shape[0] <= 0: + logger.warning(f"Camera {self.camera_id}: Invalid frame dimensions {frame.shape[1]}x{frame.shape[0]}") + continue # Reset retry counter on successful fetch retries = 0 From 360a4ab89031e289ed387b96b79d7e1b833ee351 Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Fri, 26 Sep 2025 00:16:49 +0700 Subject: [PATCH 09/55] feat: enhance logging for detected hardware codecs and improve CUDA acceleration handling --- core/utils/ffmpeg_detector.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/core/utils/ffmpeg_detector.py b/core/utils/ffmpeg_detector.py index a3cf8fc..92aecfc 100644 --- a/core/utils/ffmpeg_detector.py +++ b/core/utils/ffmpeg_detector.py @@ -46,6 +46,7 @@ class FFmpegCapabilities: # Log capabilities if self.nvidia_support: logger.info("NVIDIA hardware acceleration available (CUDA/CUVID/NVDEC)") + logger.info(f"Detected hardware codecs: {self.codecs}") if self.vaapi_support: logger.info("VAAPI hardware acceleration available") if self.qsv_support: @@ -104,22 +105,23 @@ class FFmpegCapabilities: # Add hardware acceleration if available if self.nvidia_support: - if codec == 'h264' and 'h264_hw' in self.codecs: + # Force enable CUDA hardware acceleration for H.264 if CUDA is available + if codec == 'h264': options.update({ 'hwaccel': 'cuda', 'hwaccel_device': '0', 'video_codec': 'h264_cuvid', 'hwaccel_output_format': 'cuda' }) - logger.debug("Using NVIDIA CUVID hardware acceleration for H.264") - elif codec == 'h265' and 'h265_hw' in self.codecs: + logger.info("Using NVIDIA CUVID hardware acceleration for H.264") + elif codec == 'h265': options.update({ 'hwaccel': 'cuda', 'hwaccel_device': '0', 'video_codec': 'hevc_cuvid', 'hwaccel_output_format': 'cuda' }) - logger.debug("Using NVIDIA CUVID hardware acceleration for H.265") + logger.info("Using NVIDIA CUVID hardware acceleration for H.265") elif self.vaapi_support: if codec == 'h264': From 59e8448f0d5c62b6a26df2a4d7a14bc55ef95da0 Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Fri, 26 Sep 2025 00:27:08 +0700 Subject: [PATCH 10/55] fix: add missing FFmpeg development libraries for OpenCV integration --- Dockerfile.base | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Dockerfile.base b/Dockerfile.base index 8c104d2..6c2f97b 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -24,6 +24,14 @@ RUN apt-get update && apt-get install -y \ libvpx-dev \ libmp3lame-dev \ libv4l-dev \ + # FFmpeg development libraries for OpenCV integration + libavcodec-dev \ + libavformat-dev \ + libavutil-dev \ + libavdevice-dev \ + libavfilter-dev \ + libswscale-dev \ + libswresample-dev \ # TurboJPEG for fast JPEG encoding libturbojpeg0-dev \ # Python development From e2e535604762d1b4aad21f96dff0c17a4fffc023 Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Fri, 26 Sep 2025 00:41:49 +0700 Subject: [PATCH 11/55] refactor: build FFmpeg from source with NVIDIA CUDA support and remove unnecessary development libraries --- Dockerfile.base | 43 ++++++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/Dockerfile.base b/Dockerfile.base index 6c2f97b..56b4159 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -24,14 +24,6 @@ RUN apt-get update && apt-get install -y \ libvpx-dev \ libmp3lame-dev \ libv4l-dev \ - # FFmpeg development libraries for OpenCV integration - libavcodec-dev \ - libavformat-dev \ - libavutil-dev \ - libavdevice-dev \ - libavfilter-dev \ - libswscale-dev \ - libswresample-dev \ # TurboJPEG for fast JPEG encoding libturbojpeg0-dev \ # Python development @@ -52,14 +44,35 @@ RUN cd /tmp && \ make install && \ rm -rf /tmp/* -# Download and install prebuilt FFmpeg with CUDA support +# Build FFmpeg from source with NVIDIA CUDA support RUN cd /tmp && \ - echo "Installing prebuilt FFmpeg with CUDA support..." && \ - wget https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz && \ - tar -xf ffmpeg-master-latest-linux64-gpl.tar.xz && \ - cd ffmpeg-master-latest-linux64-gpl && \ - # Copy binaries to system paths - cp bin/* /usr/local/bin/ && \ + echo "Building FFmpeg with NVIDIA CUDA support..." && \ + # Download FFmpeg source + wget https://ffmpeg.org/releases/ffmpeg-7.1.tar.xz && \ + tar -xf ffmpeg-7.1.tar.xz && \ + cd ffmpeg-7.1 && \ + # Configure with NVIDIA support + ./configure \ + --prefix=/usr/local \ + --enable-shared \ + --enable-pic \ + --enable-gpl \ + --enable-version3 \ + --enable-nonfree \ + --enable-cuda-nvcc \ + --enable-cuvid \ + --enable-nvdec \ + --enable-nvenc \ + --enable-libnpp \ + --extra-cflags=-I/usr/local/cuda/include \ + --extra-ldflags=-L/usr/local/cuda/lib64 \ + --enable-libx264 \ + --enable-libx265 \ + --enable-libvpx \ + --enable-libmp3lame && \ + # Build and install + make -j$(nproc) && \ + make install && \ ldconfig && \ # Verify CUVID decoders are available echo "=== Verifying FFmpeg CUVID Support ===" && \ From 6fe4b6ebf0d5f3c666ea724515d89cab38a05a54 Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Fri, 26 Sep 2025 00:48:06 +0700 Subject: [PATCH 12/55] refactor: update Dockerfile to use development image and enhance FFmpeg build process with NVIDIA support --- Dockerfile.base | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/Dockerfile.base b/Dockerfile.base index 56b4159..8d19778 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -1,5 +1,5 @@ # Base image with complete ML and hardware acceleration stack -FROM pytorch/pytorch:2.8.0-cuda12.6-cudnn9-runtime +FROM pytorch/pytorch:2.8.0-cuda12.6-cudnn9-devel # Install build dependencies and system libraries RUN apt-get update && apt-get install -y \ @@ -12,6 +12,12 @@ RUN apt-get update && apt-get install -y \ unzip \ yasm \ nasm \ + # Additional dependencies for FFmpeg/NVIDIA build + libtool \ + libc6 \ + libc6-dev \ + libnuma1 \ + libnuma-dev \ # System libraries libgl1-mesa-glx \ libglib2.0-0 \ @@ -31,41 +37,45 @@ RUN apt-get update && apt-get install -y \ python3-numpy \ && rm -rf /var/lib/apt/lists/* -# Install prebuilt FFmpeg with CUDA support +# CUDA development tools already available in devel image + # Ensure CUDA paths are available ENV PATH="/usr/local/cuda/bin:${PATH}" ENV LD_LIBRARY_PATH="/usr/local/cuda/lib64:${LD_LIBRARY_PATH}" -# Install NVIDIA Video Codec SDK headers first +# Install NVIDIA Video Codec SDK headers (official method) RUN cd /tmp && \ - wget https://github.com/FFmpeg/nv-codec-headers/archive/refs/tags/n12.1.14.0.zip && \ - unzip n12.1.14.0.zip && \ - cd nv-codec-headers-n12.1.14.0 && \ + git clone https://git.videolan.org/git/ffmpeg/nv-codec-headers.git && \ + cd nv-codec-headers && \ make install && \ - rm -rf /tmp/* + cd / && rm -rf /tmp/* # Build FFmpeg from source with NVIDIA CUDA support RUN cd /tmp && \ echo "Building FFmpeg with NVIDIA CUDA support..." && \ - # Download FFmpeg source - wget https://ffmpeg.org/releases/ffmpeg-7.1.tar.xz && \ - tar -xf ffmpeg-7.1.tar.xz && \ - cd ffmpeg-7.1 && \ - # Configure with NVIDIA support + # Download FFmpeg source (official method) + git clone https://git.ffmpeg.org/ffmpeg.git ffmpeg/ && \ + cd ffmpeg && \ + # Configure with NVIDIA support (following official NVIDIA documentation) ./configure \ --prefix=/usr/local \ --enable-shared \ - --enable-pic \ - --enable-gpl \ - --enable-version3 \ + --disable-static \ --enable-nonfree \ + --enable-gpl \ --enable-cuda-nvcc \ + --enable-cuda-llvm \ --enable-cuvid \ --enable-nvdec \ --enable-nvenc \ --enable-libnpp \ + --nvcc=/usr/local/cuda/bin/nvcc \ --extra-cflags=-I/usr/local/cuda/include \ --extra-ldflags=-L/usr/local/cuda/lib64 \ + --extra-libs=-lcuda \ + --extra-libs=-lcudart \ + --extra-libs=-lnvcuvid \ + --extra-libs=-lnvidia-encode \ --enable-libx264 \ --enable-libx265 \ --enable-libvpx \ From fa3ab5c6d2a49e064258ca18f5963a0d7ecd011a Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Fri, 26 Sep 2025 00:48:39 +0700 Subject: [PATCH 13/55] refactor: update base image to runtime version and install minimal CUDA development tools for FFmpeg --- Dockerfile.base | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Dockerfile.base b/Dockerfile.base index 8d19778..2569ebd 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -1,5 +1,5 @@ # Base image with complete ML and hardware acceleration stack -FROM pytorch/pytorch:2.8.0-cuda12.6-cudnn9-devel +FROM pytorch/pytorch:2.8.0-cuda12.6-cudnn9-runtime # Install build dependencies and system libraries RUN apt-get update && apt-get install -y \ @@ -37,7 +37,13 @@ RUN apt-get update && apt-get install -y \ python3-numpy \ && rm -rf /var/lib/apt/lists/* -# CUDA development tools already available in devel image +# Install minimal CUDA development tools (just what we need for FFmpeg) +RUN apt-get update && apt-get install -y \ + cuda-nvcc-12-6 \ + cuda-cudart-dev-12-6 \ + libnvidia-encode-12-6 \ + libnvidia-decode-12-6 \ + && rm -rf /var/lib/apt/lists/* # Ensure CUDA paths are available ENV PATH="/usr/local/cuda/bin:${PATH}" From bdbf6889465a250e01e9b59e4cb50623102ba77c Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Fri, 26 Sep 2025 01:11:32 +0700 Subject: [PATCH 14/55] refactor: streamline CUDA development tools installation and simplify FFmpeg configuration for NVIDIA support --- Dockerfile.base | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/Dockerfile.base b/Dockerfile.base index 2569ebd..9684325 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -18,6 +18,11 @@ RUN apt-get update && apt-get install -y \ libc6-dev \ libnuma1 \ libnuma-dev \ + # Essential compilation libraries + gcc \ + g++ \ + libc6-dev \ + linux-libc-dev \ # System libraries libgl1-mesa-glx \ libglib2.0-0 \ @@ -37,13 +42,18 @@ RUN apt-get update && apt-get install -y \ python3-numpy \ && rm -rf /var/lib/apt/lists/* -# Install minimal CUDA development tools (just what we need for FFmpeg) -RUN apt-get update && apt-get install -y \ +# Add NVIDIA CUDA repository and install minimal development tools +RUN apt-get update && apt-get install -y wget gnupg && \ + wget -O - https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub | apt-key add - && \ + echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 /" > /etc/apt/sources.list.d/cuda.list && \ + apt-get update && \ + apt-get install -y \ cuda-nvcc-12-6 \ cuda-cudart-dev-12-6 \ - libnvidia-encode-12-6 \ - libnvidia-decode-12-6 \ - && rm -rf /var/lib/apt/lists/* + libnpp-dev-12-6 \ + && apt-get remove -y wget gnupg && \ + apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* # Ensure CUDA paths are available ENV PATH="/usr/local/cuda/bin:${PATH}" @@ -62,7 +72,7 @@ RUN cd /tmp && \ # Download FFmpeg source (official method) git clone https://git.ffmpeg.org/ffmpeg.git ffmpeg/ && \ cd ffmpeg && \ - # Configure with NVIDIA support (following official NVIDIA documentation) + # Configure with NVIDIA support (simplified to avoid configure issues) ./configure \ --prefix=/usr/local \ --enable-shared \ @@ -70,18 +80,12 @@ RUN cd /tmp && \ --enable-nonfree \ --enable-gpl \ --enable-cuda-nvcc \ - --enable-cuda-llvm \ --enable-cuvid \ --enable-nvdec \ --enable-nvenc \ --enable-libnpp \ - --nvcc=/usr/local/cuda/bin/nvcc \ --extra-cflags=-I/usr/local/cuda/include \ --extra-ldflags=-L/usr/local/cuda/lib64 \ - --extra-libs=-lcuda \ - --extra-libs=-lcudart \ - --extra-libs=-lnvcuvid \ - --extra-libs=-lnvidia-encode \ --enable-libx264 \ --enable-libx265 \ --enable-libvpx \ From cb9ff7bc861cef272397da5aaa9f3ed1fbe467f2 Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Fri, 26 Sep 2025 01:33:41 +0700 Subject: [PATCH 15/55] refactor: update FFmpeg hardware acceleration to use NVDEC instead of CUVID for improved performance --- core/streaming/readers.py | 10 +++++----- core/utils/ffmpeg_detector.py | 6 ++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/core/streaming/readers.py b/core/streaming/readers.py index 53c9643..32a424a 100644 --- a/core/streaming/readers.py +++ b/core/streaming/readers.py @@ -208,20 +208,20 @@ class RTSPReader: except Exception as e: logger.debug(f"Camera {self.camera_id}: FFmpeg optimal hardware acceleration not available: {e}") - # Method 3: Try FFmpeg with basic NVIDIA CUVID + # Method 3: Try FFmpeg with NVIDIA NVDEC (better for RTX 3060) if not hw_accel_success: try: import os - os.environ['OPENCV_FFMPEG_CAPTURE_OPTIONS'] = 'video_codec;h264_cuvid|rtsp_transport;tcp|hwaccel;cuda|hwaccel_device;0' + os.environ['OPENCV_FFMPEG_CAPTURE_OPTIONS'] = 'hwaccel;cuda|hwaccel_device;0|rtsp_transport;tcp' - logger.info(f"Attempting FFmpeg with basic CUVID for camera {self.camera_id}") + logger.info(f"Attempting FFmpeg with NVDEC hardware acceleration for camera {self.camera_id}") self.cap = cv2.VideoCapture(self.rtsp_url, cv2.CAP_FFMPEG) if self.cap.isOpened(): hw_accel_success = True - logger.info(f"Camera {self.camera_id}: Using FFmpeg CUVID hardware acceleration") + logger.info(f"Camera {self.camera_id}: Using FFmpeg NVDEC hardware acceleration") except Exception as e: - logger.debug(f"Camera {self.camera_id}: FFmpeg CUVID not available: {e}") + logger.debug(f"Camera {self.camera_id}: FFmpeg NVDEC not available: {e}") # Method 4: Try FFmpeg with VAAPI (Intel/AMD GPUs) if not hw_accel_success: diff --git a/core/utils/ffmpeg_detector.py b/core/utils/ffmpeg_detector.py index 92aecfc..565713c 100644 --- a/core/utils/ffmpeg_detector.py +++ b/core/utils/ffmpeg_detector.py @@ -109,11 +109,9 @@ class FFmpegCapabilities: if codec == 'h264': options.update({ 'hwaccel': 'cuda', - 'hwaccel_device': '0', - 'video_codec': 'h264_cuvid', - 'hwaccel_output_format': 'cuda' + 'hwaccel_device': '0' }) - logger.info("Using NVIDIA CUVID hardware acceleration for H.264") + logger.info("Using NVIDIA NVDEC hardware acceleration for H.264") elif codec == 'h265': options.update({ 'hwaccel': 'cuda', From c6a4258055c9694c2cd19a6d3b4e55c6510d843f Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Fri, 26 Sep 2025 01:42:30 +0700 Subject: [PATCH 16/55] refactor: enhance error logging in RTSPReader for better debugging of frame capture issues --- core/streaming/readers.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/core/streaming/readers.py b/core/streaming/readers.py index 32a424a..78a3d45 100644 --- a/core/streaming/readers.py +++ b/core/streaming/readers.py @@ -94,8 +94,17 @@ class RTSPReader: if not ret or frame is None: consecutive_errors += 1 + # Verbose logging to see actual errors + logger.error(f"Camera {self.camera_id}: cap.read() failed - ret={ret}, frame={frame is not None}") + + # Try to get more info from the capture + if self.cap.isOpened(): + logger.debug(f"Camera {self.camera_id}: Capture still open, backend: {self.cap.getBackendName()}") + else: + logger.error(f"Camera {self.camera_id}: Capture is closed!") + if consecutive_errors >= self.max_consecutive_errors: - logger.error(f"Camera {self.camera_id}: Too many consecutive errors, reinitializing") + logger.error(f"Camera {self.camera_id}: Too many consecutive errors ({consecutive_errors}), reinitializing") self._reinitialize_capture() consecutive_errors = 0 time.sleep(self.error_recovery_delay) From a1e7c42fb35db7f2bbf43b53769f0f149e7dfaa7 Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Fri, 26 Sep 2025 01:44:46 +0700 Subject: [PATCH 17/55] refactor: improve error handling and logging in RTSPReader for frame capture failures --- core/streaming/readers.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/core/streaming/readers.py b/core/streaming/readers.py index 78a3d45..59db84b 100644 --- a/core/streaming/readers.py +++ b/core/streaming/readers.py @@ -89,7 +89,11 @@ class RTSPReader: continue # Read frame immediately without rate limiting for minimum latency - ret, frame = self.cap.read() + try: + ret, frame = self.cap.read() + except Exception as read_error: + logger.error(f"Camera {self.camera_id}: cap.read() threw exception: {type(read_error).__name__}: {read_error}") + ret, frame = False, None if not ret or frame is None: consecutive_errors += 1 @@ -98,10 +102,14 @@ class RTSPReader: logger.error(f"Camera {self.camera_id}: cap.read() failed - ret={ret}, frame={frame is not None}") # Try to get more info from the capture - if self.cap.isOpened(): - logger.debug(f"Camera {self.camera_id}: Capture still open, backend: {self.cap.getBackendName()}") - else: - logger.error(f"Camera {self.camera_id}: Capture is closed!") + try: + if self.cap.isOpened(): + backend = self.cap.getBackendName() + logger.debug(f"Camera {self.camera_id}: Capture still open, backend: {backend}") + else: + logger.error(f"Camera {self.camera_id}: Capture is closed!") + except Exception as info_error: + logger.error(f"Camera {self.camera_id}: Error getting capture info: {type(info_error).__name__}: {info_error}") if consecutive_errors >= self.max_consecutive_errors: logger.error(f"Camera {self.camera_id}: Too many consecutive errors ({consecutive_errors}), reinitializing") From 65b7573fed5a0fcaf4d10003c1b10fb9cd655afc Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Fri, 26 Sep 2025 01:52:50 +0700 Subject: [PATCH 18/55] refactor: remove unnecessary buffer size setting for RTSP stream to improve latency --- core/streaming/readers.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/streaming/readers.py b/core/streaming/readers.py index 59db84b..ef89724 100644 --- a/core/streaming/readers.py +++ b/core/streaming/readers.py @@ -269,9 +269,6 @@ class RTSPReader: # Don't force resolution/fps - let the stream determine its natural specs # The camera will provide whatever resolution/fps it supports - # Set minimal buffer for lowest latency - single frame buffer - # This ensures we always get the most recent frame - self.cap.set(cv2.CAP_PROP_BUFFERSIZE, 1) # Set FFMPEG options for better H.264 handling self.cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*'H264')) From 08cb4eafc40758cf0e652fbfc834e4052ddd452d Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Fri, 26 Sep 2025 01:58:50 +0700 Subject: [PATCH 19/55] refactor: enhance error handling and logging in RTSPReader for improved frame retrieval diagnostics --- core/streaming/readers.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/core/streaming/readers.py b/core/streaming/readers.py index ef89724..6f31cf1 100644 --- a/core/streaming/readers.py +++ b/core/streaming/readers.py @@ -90,24 +90,30 @@ class RTSPReader: # Read frame immediately without rate limiting for minimum latency try: - ret, frame = self.cap.read() + # Force grab then retrieve for better error handling + ret = self.cap.grab() + if ret: + ret, frame = self.cap.retrieve() + else: + frame = None except Exception as read_error: - logger.error(f"Camera {self.camera_id}: cap.read() threw exception: {type(read_error).__name__}: {read_error}") + logger.error(f"Camera {self.camera_id}: cap.grab/retrieve threw exception: {type(read_error).__name__}: {read_error}") ret, frame = False, None if not ret or frame is None: consecutive_errors += 1 - # Verbose logging to see actual errors + # Enhanced logging to diagnose the issue logger.error(f"Camera {self.camera_id}: cap.read() failed - ret={ret}, frame={frame is not None}") # Try to get more info from the capture try: - if self.cap.isOpened(): + if self.cap and self.cap.isOpened(): backend = self.cap.getBackendName() - logger.debug(f"Camera {self.camera_id}: Capture still open, backend: {backend}") + pos_frames = self.cap.get(cv2.CAP_PROP_POS_FRAMES) + logger.error(f"Camera {self.camera_id}: Capture open, backend: {backend}, pos_frames: {pos_frames}") else: - logger.error(f"Camera {self.camera_id}: Capture is closed!") + logger.error(f"Camera {self.camera_id}: Capture is closed or None!") except Exception as info_error: logger.error(f"Camera {self.camera_id}: Error getting capture info: {type(info_error).__name__}: {info_error}") From c38b58e34c7928ed7a2b7750e947f8e3aed83c3d Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Fri, 26 Sep 2025 02:07:17 +0700 Subject: [PATCH 20/55] refactor: add FFmpegRTSPReader for enhanced RTSP stream handling with CUDA acceleration --- core/streaming/__init__.py | 3 +- core/streaming/manager.py | 8 +- core/streaming/readers.py | 150 +++++++++++++++++++++++++++++++++++-- 3 files changed, 149 insertions(+), 12 deletions(-) diff --git a/core/streaming/__init__.py b/core/streaming/__init__.py index c4c40dc..d878aac 100644 --- a/core/streaming/__init__.py +++ b/core/streaming/__init__.py @@ -2,7 +2,7 @@ Streaming system for RTSP and HTTP camera feeds. Provides modular frame readers, buffers, and stream management. """ -from .readers import RTSPReader, HTTPSnapshotReader +from .readers import RTSPReader, HTTPSnapshotReader, FFmpegRTSPReader from .buffers import FrameBuffer, CacheBuffer, shared_frame_buffer, shared_cache_buffer from .manager import StreamManager, StreamConfig, SubscriptionInfo, shared_stream_manager, initialize_stream_manager @@ -10,6 +10,7 @@ __all__ = [ # Readers 'RTSPReader', 'HTTPSnapshotReader', + 'FFmpegRTSPReader', # Buffers 'FrameBuffer', diff --git a/core/streaming/manager.py b/core/streaming/manager.py index 1e3719f..156daf1 100644 --- a/core/streaming/manager.py +++ b/core/streaming/manager.py @@ -9,7 +9,7 @@ from typing import Dict, Set, Optional, List, Any from dataclasses import dataclass from collections import defaultdict -from .readers import RTSPReader, HTTPSnapshotReader +from .readers import RTSPReader, HTTPSnapshotReader, FFmpegRTSPReader from .buffers import shared_cache_buffer from ..tracking.integration import TrackingPipelineIntegration @@ -129,8 +129,8 @@ class StreamManager: """Start a stream for the given camera.""" try: if stream_config.rtsp_url: - # RTSP stream - reader = RTSPReader( + # RTSP stream using FFmpeg subprocess with CUDA acceleration + reader = FFmpegRTSPReader( camera_id=camera_id, rtsp_url=stream_config.rtsp_url, max_retries=stream_config.max_retries @@ -138,7 +138,7 @@ class StreamManager: reader.set_frame_callback(self._frame_callback) reader.start() self._streams[camera_id] = reader - logger.info(f"Started RTSP stream for camera {camera_id}") + logger.info(f"Started FFmpeg RTSP stream for camera {camera_id}") elif stream_config.snapshot_url: # HTTP snapshot stream diff --git a/core/streaming/readers.py b/core/streaming/readers.py index 6f31cf1..243f088 100644 --- a/core/streaming/readers.py +++ b/core/streaming/readers.py @@ -9,6 +9,7 @@ import threading import requests import numpy as np import os +import subprocess from typing import Optional, Callable # Suppress FFMPEG/H.264 error messages if needed @@ -19,6 +20,143 @@ os.environ["OPENCV_FFMPEG_LOGLEVEL"] = "-8" # Suppress FFMPEG warnings logger = logging.getLogger(__name__) +class FFmpegRTSPReader: + """RTSP stream reader using subprocess FFmpeg with CUDA hardware acceleration.""" + + def __init__(self, camera_id: str, rtsp_url: str, max_retries: int = 3): + self.camera_id = camera_id + self.rtsp_url = rtsp_url + self.max_retries = max_retries + self.process = None + self.stop_event = threading.Event() + self.thread = None + self.frame_callback: Optional[Callable] = None + + # Stream specs + self.width = 1280 + self.height = 720 + + def set_frame_callback(self, callback: Callable[[str, np.ndarray], None]): + """Set callback function to handle captured frames.""" + self.frame_callback = callback + + def start(self): + """Start the FFmpeg subprocess reader.""" + if self.thread and self.thread.is_alive(): + logger.warning(f"FFmpeg reader for {self.camera_id} already running") + return + + self.stop_event.clear() + self.thread = threading.Thread(target=self._read_frames, daemon=True) + self.thread.start() + logger.info(f"Started FFmpeg reader for camera {self.camera_id}") + + def stop(self): + """Stop the FFmpeg subprocess reader.""" + self.stop_event.set() + if self.process: + self.process.terminate() + try: + self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + self.process.kill() + if self.thread: + self.thread.join(timeout=5.0) + logger.info(f"Stopped FFmpeg reader for camera {self.camera_id}") + + def _start_ffmpeg_process(self): + """Start FFmpeg subprocess with CUDA hardware acceleration.""" + cmd = [ + 'ffmpeg', + '-hwaccel', 'cuda', + '-hwaccel_device', '0', + '-rtsp_transport', 'tcp', + '-i', self.rtsp_url, + '-f', 'rawvideo', + '-pix_fmt', 'bgr24', + '-an', # No audio + '-' # Output to stdout + ] + + try: + self.process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=0 + ) + logger.info(f"Started FFmpeg process for camera {self.camera_id}") + return True + except Exception as e: + logger.error(f"Failed to start FFmpeg for camera {self.camera_id}: {e}") + return False + + def _read_frames(self): + """Read frames from FFmpeg stdout pipe.""" + consecutive_errors = 0 + frame_count = 0 + last_log_time = time.time() + bytes_per_frame = self.width * self.height * 3 # BGR = 3 bytes per pixel + + while not self.stop_event.is_set(): + try: + # Start/restart FFmpeg process if needed + if not self.process or self.process.poll() is not None: + if not self._start_ffmpeg_process(): + time.sleep(5.0) + continue + + # Read one frame worth of data + frame_data = self.process.stdout.read(bytes_per_frame) + + if len(frame_data) != bytes_per_frame: + consecutive_errors += 1 + if consecutive_errors >= 30: + logger.error(f"Camera {self.camera_id}: Too many read errors, restarting FFmpeg") + if self.process: + self.process.terminate() + consecutive_errors = 0 + continue + + # Convert raw bytes to numpy array + frame = np.frombuffer(frame_data, dtype=np.uint8) + frame = frame.reshape((self.height, self.width, 3)) + + # Frame is valid + consecutive_errors = 0 + frame_count += 1 + + # Call frame callback + if self.frame_callback: + try: + self.frame_callback(self.camera_id, frame) + except Exception as e: + logger.error(f"Camera {self.camera_id}: Frame callback error: {e}") + + # Log progress + current_time = time.time() + if current_time - last_log_time >= 30: + logger.info(f"Camera {self.camera_id}: {frame_count} frames processed via FFmpeg") + last_log_time = current_time + + except Exception as e: + logger.error(f"Camera {self.camera_id}: FFmpeg read error: {e}") + consecutive_errors += 1 + if consecutive_errors >= 30: + if self.process: + self.process.terminate() + consecutive_errors = 0 + time.sleep(1.0) + + # Cleanup + if self.process: + self.process.terminate() + logger.info(f"FFmpeg reader thread ended for camera {self.camera_id}") + + +logger = logging.getLogger(__name__) + + class RTSPReader: """RTSP stream frame reader optimized for 1280x720 @ 6fps streams.""" @@ -90,14 +228,12 @@ class RTSPReader: # Read frame immediately without rate limiting for minimum latency try: - # Force grab then retrieve for better error handling - ret = self.cap.grab() - if ret: - ret, frame = self.cap.retrieve() - else: - frame = None + ret, frame = self.cap.read() + if ret and frame is None: + # Grab succeeded but retrieve failed - decoder issue + logger.error(f"Camera {self.camera_id}: Frame grab OK but decode failed") except Exception as read_error: - logger.error(f"Camera {self.camera_id}: cap.grab/retrieve threw exception: {type(read_error).__name__}: {read_error}") + logger.error(f"Camera {self.camera_id}: cap.read() threw exception: {type(read_error).__name__}: {read_error}") ret, frame = False, None if not ret or frame is None: From 79a1189675e430e093d971565776b5ad01809eb0 Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Fri, 26 Sep 2025 02:15:06 +0700 Subject: [PATCH 21/55] refactor: update FFmpegRTSPReader to use a temporary file for frame reading and improve error handling --- core/streaming/readers.py | 112 +++++++++++++++++++++++++++----------- 1 file changed, 81 insertions(+), 31 deletions(-) diff --git a/core/streaming/readers.py b/core/streaming/readers.py index 243f088..7478e38 100644 --- a/core/streaming/readers.py +++ b/core/streaming/readers.py @@ -65,7 +65,12 @@ class FFmpegRTSPReader: logger.info(f"Stopped FFmpeg reader for camera {self.camera_id}") def _start_ffmpeg_process(self): - """Start FFmpeg subprocess with CUDA hardware acceleration.""" + """Start FFmpeg subprocess with CUDA hardware acceleration writing to temp file.""" + # Create temp file path for this camera + import tempfile + self.temp_file = f"/tmp/claude/camera_{self.camera_id.replace(' ', '_')}.raw" + os.makedirs("/tmp/claude", exist_ok=True) + cmd = [ 'ffmpeg', '-hwaccel', 'cuda', @@ -75,7 +80,8 @@ class FFmpegRTSPReader: '-f', 'rawvideo', '-pix_fmt', 'bgr24', '-an', # No audio - '-' # Output to stdout + '-y', # Overwrite output file + self.temp_file ] try: @@ -85,18 +91,22 @@ class FFmpegRTSPReader: stderr=subprocess.PIPE, bufsize=0 ) - logger.info(f"Started FFmpeg process for camera {self.camera_id}") + logger.info(f"Started FFmpeg process for camera {self.camera_id} writing to {self.temp_file}") + + # Don't check process immediately - FFmpeg takes time to initialize + logger.info(f"Waiting for FFmpeg to initialize for camera {self.camera_id}...") return True except Exception as e: logger.error(f"Failed to start FFmpeg for camera {self.camera_id}: {e}") return False def _read_frames(self): - """Read frames from FFmpeg stdout pipe.""" + """Read frames from FFmpeg temp file.""" consecutive_errors = 0 frame_count = 0 last_log_time = time.time() bytes_per_frame = self.width * self.height * 3 # BGR = 3 bytes per pixel + last_file_size = 0 while not self.stop_event.is_set(): try: @@ -106,38 +116,72 @@ class FFmpegRTSPReader: time.sleep(5.0) continue - # Read one frame worth of data - frame_data = self.process.stdout.read(bytes_per_frame) - - if len(frame_data) != bytes_per_frame: - consecutive_errors += 1 - if consecutive_errors >= 30: - logger.error(f"Camera {self.camera_id}: Too many read errors, restarting FFmpeg") - if self.process: - self.process.terminate() - consecutive_errors = 0 + # Wait for temp file to exist and have content + if not os.path.exists(self.temp_file): + time.sleep(0.1) continue - # Convert raw bytes to numpy array - frame = np.frombuffer(frame_data, dtype=np.uint8) - frame = frame.reshape((self.height, self.width, 3)) + # Check if file size changed (new frame available) + try: + current_file_size = os.path.getsize(self.temp_file) + if current_file_size <= last_file_size and current_file_size > 0: + # File size didn't increase, wait for next frame + time.sleep(0.05) # ~20 FPS max + continue + last_file_size = current_file_size + except OSError: + time.sleep(0.1) + continue - # Frame is valid - consecutive_errors = 0 - frame_count += 1 + # Read the latest frame from the end of file + try: + with open(self.temp_file, 'rb') as f: + # Seek to last complete frame + file_size = f.seek(0, 2) # Seek to end + if file_size < bytes_per_frame: + time.sleep(0.1) + continue - # Call frame callback - if self.frame_callback: - try: - self.frame_callback(self.camera_id, frame) - except Exception as e: - logger.error(f"Camera {self.camera_id}: Frame callback error: {e}") + # Read last complete frame + last_frame_offset = (file_size // bytes_per_frame - 1) * bytes_per_frame + f.seek(last_frame_offset) + frame_data = f.read(bytes_per_frame) - # Log progress - current_time = time.time() - if current_time - last_log_time >= 30: - logger.info(f"Camera {self.camera_id}: {frame_count} frames processed via FFmpeg") - last_log_time = current_time + if len(frame_data) != bytes_per_frame: + consecutive_errors += 1 + if consecutive_errors >= 30: + logger.error(f"Camera {self.camera_id}: Too many read errors, restarting FFmpeg") + if self.process: + self.process.terminate() + consecutive_errors = 0 + time.sleep(0.1) + continue + + # Convert raw bytes to numpy array + frame = np.frombuffer(frame_data, dtype=np.uint8) + frame = frame.reshape((self.height, self.width, 3)) + + # Frame is valid + consecutive_errors = 0 + frame_count += 1 + + # Call frame callback + if self.frame_callback: + try: + self.frame_callback(self.camera_id, frame) + except Exception as e: + logger.error(f"Camera {self.camera_id}: Frame callback error: {e}") + + # Log progress + current_time = time.time() + if current_time - last_log_time >= 30: + logger.info(f"Camera {self.camera_id}: {frame_count} frames processed via temp file") + last_log_time = current_time + + except IOError as e: + logger.debug(f"Camera {self.camera_id}: File read error: {e}") + time.sleep(0.1) + continue except Exception as e: logger.error(f"Camera {self.camera_id}: FFmpeg read error: {e}") @@ -151,6 +195,12 @@ class FFmpegRTSPReader: # Cleanup if self.process: self.process.terminate() + # Clean up temp file + try: + if hasattr(self, 'temp_file') and os.path.exists(self.temp_file): + os.remove(self.temp_file) + except: + pass logger.info(f"FFmpeg reader thread ended for camera {self.camera_id}") From cb31633cc107a5156b4c81d975823989f42e416c Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Fri, 26 Sep 2025 02:18:20 +0700 Subject: [PATCH 22/55] refactor: enhance FFmpegRTSPReader with file watching and reactive frame reading --- .claude/settings.local.json | 3 +- core/streaming/readers.py | 179 ++++++++++++++++++++---------------- 2 files changed, 101 insertions(+), 81 deletions(-) diff --git a/.claude/settings.local.json b/.claude/settings.local.json index b06024d..97cf5c1 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -1,7 +1,8 @@ { "permissions": { "allow": [ - "Bash(dir:*)" + "Bash(dir:*)", + "WebSearch" ], "deny": [], "ask": [] diff --git a/core/streaming/readers.py b/core/streaming/readers.py index 7478e38..e221c4a 100644 --- a/core/streaming/readers.py +++ b/core/streaming/readers.py @@ -11,6 +11,8 @@ import numpy as np import os import subprocess from typing import Optional, Callable +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler # Suppress FFMPEG/H.264 error messages if needed # Set this environment variable to reduce noise from decoder errors @@ -20,8 +22,25 @@ os.environ["OPENCV_FFMPEG_LOGLEVEL"] = "-8" # Suppress FFMPEG warnings logger = logging.getLogger(__name__) +class FrameFileHandler(FileSystemEventHandler): + """File system event handler for frame file changes.""" + + def __init__(self, callback): + self.callback = callback + self.last_modified = 0 + + def on_modified(self, event): + if event.is_directory: + return + # Debounce rapid file changes + current_time = time.time() + if current_time - self.last_modified > 0.01: # 10ms debounce + self.last_modified = current_time + self.callback() + + class FFmpegRTSPReader: - """RTSP stream reader using subprocess FFmpeg with CUDA hardware acceleration.""" + """RTSP stream reader using subprocess FFmpeg with CUDA hardware acceleration and file watching.""" def __init__(self, camera_id: str, rtsp_url: str, max_retries: int = 3): self.camera_id = camera_id @@ -31,6 +50,8 @@ class FFmpegRTSPReader: self.stop_event = threading.Event() self.thread = None self.frame_callback: Optional[Callable] = None + self.observer = None + self.frame_ready_event = threading.Event() # Stream specs self.width = 1280 @@ -67,7 +88,6 @@ class FFmpegRTSPReader: def _start_ffmpeg_process(self): """Start FFmpeg subprocess with CUDA hardware acceleration writing to temp file.""" # Create temp file path for this camera - import tempfile self.temp_file = f"/tmp/claude/camera_{self.camera_id.replace(' ', '_')}.raw" os.makedirs("/tmp/claude", exist_ok=True) @@ -85,114 +105,113 @@ class FFmpegRTSPReader: ] try: + # Start FFmpeg detached - we don't need to communicate with it self.process = subprocess.Popen( cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - bufsize=0 + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL ) - logger.info(f"Started FFmpeg process for camera {self.camera_id} writing to {self.temp_file}") - - # Don't check process immediately - FFmpeg takes time to initialize - logger.info(f"Waiting for FFmpeg to initialize for camera {self.camera_id}...") + logger.info(f"Started FFmpeg process PID {self.process.pid} for camera {self.camera_id} -> {self.temp_file}") return True except Exception as e: logger.error(f"Failed to start FFmpeg for camera {self.camera_id}: {e}") return False + def _setup_file_watcher(self): + """Setup file system watcher for temp file.""" + if not os.path.exists(self.temp_file): + return + + # Setup file watcher + handler = FrameFileHandler(self._on_file_changed) + self.observer = Observer() + self.observer.schedule(handler, os.path.dirname(self.temp_file), recursive=False) + self.observer.start() + logger.info(f"Started file watcher for {self.temp_file}") + + def _on_file_changed(self): + """Called when temp file is modified.""" + if os.path.basename(self.temp_file) in str(self.temp_file): + self.frame_ready_event.set() + def _read_frames(self): - """Read frames from FFmpeg temp file.""" - consecutive_errors = 0 + """Reactively read frames when file changes.""" frame_count = 0 last_log_time = time.time() - bytes_per_frame = self.width * self.height * 3 # BGR = 3 bytes per pixel - last_file_size = 0 + bytes_per_frame = self.width * self.height * 3 + restart_check_interval = 10 # Check FFmpeg status every 10 seconds while not self.stop_event.is_set(): try: - # Start/restart FFmpeg process if needed + # Start FFmpeg if not running if not self.process or self.process.poll() is not None: + if self.process and self.process.poll() is not None: + logger.warning(f"FFmpeg process died for camera {self.camera_id}, restarting...") + if not self._start_ffmpeg_process(): time.sleep(5.0) continue - # Wait for temp file to exist and have content - if not os.path.exists(self.temp_file): - time.sleep(0.1) - continue + # Wait for temp file to be created + wait_count = 0 + while not os.path.exists(self.temp_file) and wait_count < 30: + time.sleep(1.0) + wait_count += 1 - # Check if file size changed (new frame available) - try: - current_file_size = os.path.getsize(self.temp_file) - if current_file_size <= last_file_size and current_file_size > 0: - # File size didn't increase, wait for next frame - time.sleep(0.05) # ~20 FPS max - continue - last_file_size = current_file_size - except OSError: - time.sleep(0.1) - continue - - # Read the latest frame from the end of file - try: - with open(self.temp_file, 'rb') as f: - # Seek to last complete frame - file_size = f.seek(0, 2) # Seek to end - if file_size < bytes_per_frame: - time.sleep(0.1) - continue - - # Read last complete frame - last_frame_offset = (file_size // bytes_per_frame - 1) * bytes_per_frame - f.seek(last_frame_offset) - frame_data = f.read(bytes_per_frame) - - if len(frame_data) != bytes_per_frame: - consecutive_errors += 1 - if consecutive_errors >= 30: - logger.error(f"Camera {self.camera_id}: Too many read errors, restarting FFmpeg") - if self.process: - self.process.terminate() - consecutive_errors = 0 - time.sleep(0.1) + if not os.path.exists(self.temp_file): + logger.error(f"Temp file not created after 30s for {self.camera_id}") continue - # Convert raw bytes to numpy array - frame = np.frombuffer(frame_data, dtype=np.uint8) - frame = frame.reshape((self.height, self.width, 3)) + # Setup file watcher + self._setup_file_watcher() - # Frame is valid - consecutive_errors = 0 - frame_count += 1 + # Wait for file change event (or timeout for health check) + if self.frame_ready_event.wait(timeout=restart_check_interval): + self.frame_ready_event.clear() - # Call frame callback - if self.frame_callback: - try: - self.frame_callback(self.camera_id, frame) - except Exception as e: - logger.error(f"Camera {self.camera_id}: Frame callback error: {e}") + # Read latest frame + try: + with open(self.temp_file, 'rb') as f: + # Get file size + f.seek(0, 2) + file_size = f.tell() - # Log progress - current_time = time.time() - if current_time - last_log_time >= 30: - logger.info(f"Camera {self.camera_id}: {frame_count} frames processed via temp file") - last_log_time = current_time + if file_size < bytes_per_frame: + continue - except IOError as e: - logger.debug(f"Camera {self.camera_id}: File read error: {e}") - time.sleep(0.1) - continue + # Read last complete frame + last_frame_offset = (file_size // bytes_per_frame - 1) * bytes_per_frame + f.seek(last_frame_offset) + frame_data = f.read(bytes_per_frame) + + if len(frame_data) == bytes_per_frame: + # Convert to numpy array + frame = np.frombuffer(frame_data, dtype=np.uint8) + frame = frame.reshape((self.height, self.width, 3)) + + # Call frame callback + if self.frame_callback: + self.frame_callback(self.camera_id, frame) + + frame_count += 1 + + # Log progress + current_time = time.time() + if current_time - last_log_time >= 30: + logger.info(f"Camera {self.camera_id}: {frame_count} frames processed reactively") + last_log_time = current_time + + except (IOError, OSError) as e: + logger.debug(f"Camera {self.camera_id}: File read error: {e}") except Exception as e: - logger.error(f"Camera {self.camera_id}: FFmpeg read error: {e}") - consecutive_errors += 1 - if consecutive_errors >= 30: - if self.process: - self.process.terminate() - consecutive_errors = 0 + logger.error(f"Camera {self.camera_id}: Error in reactive frame reading: {e}") time.sleep(1.0) # Cleanup + if self.observer: + self.observer.stop() + self.observer.join() if self.process: self.process.terminate() # Clean up temp file @@ -201,7 +220,7 @@ class FFmpegRTSPReader: os.remove(self.temp_file) except: pass - logger.info(f"FFmpeg reader thread ended for camera {self.camera_id}") + logger.info(f"Reactive FFmpeg reader ended for camera {self.camera_id}") logger = logging.getLogger(__name__) From 84144a295542752f64b9ef1a940ca95b6fc6dd73 Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Fri, 26 Sep 2025 02:20:14 +0700 Subject: [PATCH 23/55] refactor: update FFmpegRTSPReader to read and update a single frame in place for improved efficiency --- core/streaming/readers.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/core/streaming/readers.py b/core/streaming/readers.py index e221c4a..d6a1272 100644 --- a/core/streaming/readers.py +++ b/core/streaming/readers.py @@ -100,6 +100,7 @@ class FFmpegRTSPReader: '-f', 'rawvideo', '-pix_fmt', 'bgr24', '-an', # No audio + '-update', '1', # Update single frame in place '-y', # Overwrite output file self.temp_file ] @@ -169,19 +170,9 @@ class FFmpegRTSPReader: if self.frame_ready_event.wait(timeout=restart_check_interval): self.frame_ready_event.clear() - # Read latest frame + # Read current frame (file is always exactly one frame) try: with open(self.temp_file, 'rb') as f: - # Get file size - f.seek(0, 2) - file_size = f.tell() - - if file_size < bytes_per_frame: - continue - - # Read last complete frame - last_frame_offset = (file_size // bytes_per_frame - 1) * bytes_per_frame - f.seek(last_frame_offset) frame_data = f.read(bytes_per_frame) if len(frame_data) == bytes_per_frame: From 2742b86961f98832d2f734e19ea9eb2413dc4e39 Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Fri, 26 Sep 2025 02:26:44 +0700 Subject: [PATCH 24/55] refactor: enhance FFmpegRTSPReader to improve frame reading reliability with retry logic --- core/streaming/readers.py | 49 ++++++++++++++++++++++++++------------- requirements.txt | 3 ++- 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/core/streaming/readers.py b/core/streaming/readers.py index d6a1272..b68a15b 100644 --- a/core/streaming/readers.py +++ b/core/streaming/readers.py @@ -170,27 +170,44 @@ class FFmpegRTSPReader: if self.frame_ready_event.wait(timeout=restart_check_interval): self.frame_ready_event.clear() - # Read current frame (file is always exactly one frame) + # Read current frame with concurrency safety try: - with open(self.temp_file, 'rb') as f: - frame_data = f.read(bytes_per_frame) + # Try to read frame multiple times to handle race conditions + frame_data = None + for attempt in range(3): + try: + with open(self.temp_file, 'rb') as f: + frame_data = f.read(bytes_per_frame) - if len(frame_data) == bytes_per_frame: - # Convert to numpy array - frame = np.frombuffer(frame_data, dtype=np.uint8) - frame = frame.reshape((self.height, self.width, 3)) + # Validate we got a complete frame + if len(frame_data) == bytes_per_frame: + break + else: + logger.debug(f"Camera {self.camera_id}: Partial read {len(frame_data)}/{bytes_per_frame}, attempt {attempt+1}") + time.sleep(0.01) # Brief wait before retry - # Call frame callback - if self.frame_callback: - self.frame_callback(self.camera_id, frame) + except (IOError, OSError) as e: + logger.debug(f"Camera {self.camera_id}: Read error on attempt {attempt+1}: {e}") + time.sleep(0.01) - frame_count += 1 + if frame_data and len(frame_data) == bytes_per_frame: + # Convert to numpy array + frame = np.frombuffer(frame_data, dtype=np.uint8) + frame = frame.reshape((self.height, self.width, 3)) - # Log progress - current_time = time.time() - if current_time - last_log_time >= 30: - logger.info(f"Camera {self.camera_id}: {frame_count} frames processed reactively") - last_log_time = current_time + # Call frame callback directly - trust the retry logic caught corruption + if self.frame_callback: + self.frame_callback(self.camera_id, frame) + + frame_count += 1 + + # Log progress + current_time = time.time() + if current_time - last_log_time >= 30: + logger.info(f"Camera {self.camera_id}: {frame_count} frames processed reactively") + last_log_time = current_time + else: + logger.debug(f"Camera {self.camera_id}: Failed to read complete frame after retries") except (IOError, OSError) as e: logger.debug(f"Camera {self.camera_id}: File read error: {e}") diff --git a/requirements.txt b/requirements.txt index 034d18e..2afeb0e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ fastapi[standard] redis urllib3<2.0.0 numpy -requests \ No newline at end of file +requests +watchdog \ No newline at end of file From 95c39a008f14b1795844e25fab42619a9b2211ee Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Fri, 26 Sep 2025 02:35:27 +0700 Subject: [PATCH 25/55] refactor: suppress noisy watchdog debug logs for cleaner output --- core/streaming/readers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/streaming/readers.py b/core/streaming/readers.py index b68a15b..f9df506 100644 --- a/core/streaming/readers.py +++ b/core/streaming/readers.py @@ -21,6 +21,9 @@ os.environ["OPENCV_FFMPEG_LOGLEVEL"] = "-8" # Suppress FFMPEG warnings logger = logging.getLogger(__name__) +# Suppress noisy watchdog debug logs +logging.getLogger('watchdog.observers.inotify_buffer').setLevel(logging.CRITICAL) + class FrameFileHandler(FileSystemEventHandler): """File system event handler for frame file changes.""" From 73c33676811c1c3e15abc468faab6394fdded6fe Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Fri, 26 Sep 2025 02:51:30 +0700 Subject: [PATCH 26/55] refactor: update FFmpegRTSPReader to use JPG format for single frame updates and improve image quality --- core/streaming/readers.py | 42 +++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/core/streaming/readers.py b/core/streaming/readers.py index f9df506..b623c49 100644 --- a/core/streaming/readers.py +++ b/core/streaming/readers.py @@ -94,16 +94,19 @@ class FFmpegRTSPReader: self.temp_file = f"/tmp/claude/camera_{self.camera_id.replace(' ', '_')}.raw" os.makedirs("/tmp/claude", exist_ok=True) + # Change to JPG format which properly supports -update 1 + self.temp_file = f"/tmp/claude/camera_{self.camera_id.replace(' ', '_')}.jpg" + cmd = [ 'ffmpeg', '-hwaccel', 'cuda', '-hwaccel_device', '0', '-rtsp_transport', 'tcp', '-i', self.rtsp_url, - '-f', 'rawvideo', - '-pix_fmt', 'bgr24', + '-f', 'image2', + '-update', '1', # This actually works with image2 format + '-q:v', '2', # High quality JPEG '-an', # No audio - '-update', '1', # Update single frame in place '-y', # Overwrite output file self.temp_file ] @@ -173,32 +176,27 @@ class FFmpegRTSPReader: if self.frame_ready_event.wait(timeout=restart_check_interval): self.frame_ready_event.clear() - # Read current frame with concurrency safety + # Read JPEG frame with concurrency safety try: - # Try to read frame multiple times to handle race conditions - frame_data = None + # Try to read JPEG multiple times to handle race conditions + frame = None for attempt in range(3): try: - with open(self.temp_file, 'rb') as f: - frame_data = f.read(bytes_per_frame) + # Read and decode JPEG directly + frame = cv2.imread(self.temp_file) - # Validate we got a complete frame - if len(frame_data) == bytes_per_frame: - break - else: - logger.debug(f"Camera {self.camera_id}: Partial read {len(frame_data)}/{bytes_per_frame}, attempt {attempt+1}") - time.sleep(0.01) # Brief wait before retry + if frame is not None and frame.shape == (self.height, self.width, 3): + break + else: + logger.debug(f"Camera {self.camera_id}: Invalid frame shape or None, attempt {attempt+1}") + time.sleep(0.01) # Brief wait before retry except (IOError, OSError) as e: logger.debug(f"Camera {self.camera_id}: Read error on attempt {attempt+1}: {e}") time.sleep(0.01) - if frame_data and len(frame_data) == bytes_per_frame: - # Convert to numpy array - frame = np.frombuffer(frame_data, dtype=np.uint8) - frame = frame.reshape((self.height, self.width, 3)) - - # Call frame callback directly - trust the retry logic caught corruption + if frame is not None: + # Call frame callback directly if self.frame_callback: self.frame_callback(self.camera_id, frame) @@ -207,10 +205,10 @@ class FFmpegRTSPReader: # Log progress current_time = time.time() if current_time - last_log_time >= 30: - logger.info(f"Camera {self.camera_id}: {frame_count} frames processed reactively") + logger.info(f"Camera {self.camera_id}: {frame_count} JPEG frames processed reactively") last_log_time = current_time else: - logger.debug(f"Camera {self.camera_id}: Failed to read complete frame after retries") + logger.debug(f"Camera {self.camera_id}: Failed to read valid JPEG after retries") except (IOError, OSError) as e: logger.debug(f"Camera {self.camera_id}: File read error: {e}") From fe0da18d0fefac3a0177a8bc8a319c2f7556593a Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Fri, 26 Sep 2025 02:55:26 +0700 Subject: [PATCH 27/55] refactor: change temporary file format from JPG to PPM for improved frame reading --- core/streaming/readers.py | 53 ++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 32 deletions(-) diff --git a/core/streaming/readers.py b/core/streaming/readers.py index b623c49..e6eed55 100644 --- a/core/streaming/readers.py +++ b/core/streaming/readers.py @@ -94,8 +94,8 @@ class FFmpegRTSPReader: self.temp_file = f"/tmp/claude/camera_{self.camera_id.replace(' ', '_')}.raw" os.makedirs("/tmp/claude", exist_ok=True) - # Change to JPG format which properly supports -update 1 - self.temp_file = f"/tmp/claude/camera_{self.camera_id.replace(' ', '_')}.jpg" + # Use PPM format - uncompressed with header, supports -update 1 + self.temp_file = f"/tmp/claude/camera_{self.camera_id.replace(' ', '_')}.ppm" cmd = [ 'ffmpeg', @@ -104,8 +104,8 @@ class FFmpegRTSPReader: '-rtsp_transport', 'tcp', '-i', self.rtsp_url, '-f', 'image2', - '-update', '1', # This actually works with image2 format - '-q:v', '2', # High quality JPEG + '-update', '1', # Works with image2 format + '-pix_fmt', 'rgb24', # PPM uses RGB not BGR '-an', # No audio '-y', # Overwrite output file self.temp_file @@ -176,39 +176,28 @@ class FFmpegRTSPReader: if self.frame_ready_event.wait(timeout=restart_check_interval): self.frame_ready_event.clear() - # Read JPEG frame with concurrency safety + # Read PPM frame (uncompressed with header) try: - # Try to read JPEG multiple times to handle race conditions - frame = None - for attempt in range(3): - try: - # Read and decode JPEG directly - frame = cv2.imread(self.temp_file) + if os.path.exists(self.temp_file): + # Read PPM with OpenCV (handles RGB->BGR conversion automatically) + frame = cv2.imread(self.temp_file) - if frame is not None and frame.shape == (self.height, self.width, 3): - break - else: - logger.debug(f"Camera {self.camera_id}: Invalid frame shape or None, attempt {attempt+1}") - time.sleep(0.01) # Brief wait before retry + if frame is not None and frame.shape == (self.height, self.width, 3): + # Call frame callback directly + if self.frame_callback: + self.frame_callback(self.camera_id, frame) - except (IOError, OSError) as e: - logger.debug(f"Camera {self.camera_id}: Read error on attempt {attempt+1}: {e}") - time.sleep(0.01) + frame_count += 1 - if frame is not None: - # Call frame callback directly - if self.frame_callback: - self.frame_callback(self.camera_id, frame) - - frame_count += 1 - - # Log progress - current_time = time.time() - if current_time - last_log_time >= 30: - logger.info(f"Camera {self.camera_id}: {frame_count} JPEG frames processed reactively") - last_log_time = current_time + # Log progress + current_time = time.time() + if current_time - last_log_time >= 30: + logger.info(f"Camera {self.camera_id}: {frame_count} PPM frames processed reactively") + last_log_time = current_time + else: + logger.debug(f"Camera {self.camera_id}: Invalid PPM frame") else: - logger.debug(f"Camera {self.camera_id}: Failed to read valid JPEG after retries") + logger.debug(f"Camera {self.camera_id}: PPM file not found yet") except (IOError, OSError) as e: logger.debug(f"Camera {self.camera_id}: File read error: {e}") From a12e3efa1282d23c305a0b8d6f8b96cd1083cc5f Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Fri, 26 Sep 2025 03:04:53 +0700 Subject: [PATCH 28/55] refactor: enhance FFmpegRTSPReader to implement persistent file locking for PPM frame reading --- core/streaming/readers.py | 61 ++++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/core/streaming/readers.py b/core/streaming/readers.py index e6eed55..35a7213 100644 --- a/core/streaming/readers.py +++ b/core/streaming/readers.py @@ -10,6 +10,7 @@ import requests import numpy as np import os import subprocess +import fcntl from typing import Optional, Callable from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler @@ -94,7 +95,7 @@ class FFmpegRTSPReader: self.temp_file = f"/tmp/claude/camera_{self.camera_id.replace(' ', '_')}.raw" os.makedirs("/tmp/claude", exist_ok=True) - # Use PPM format - uncompressed with header, supports -update 1 + # Use PPM format with single file (will use file locking for concurrency) self.temp_file = f"/tmp/claude/camera_{self.camera_id.replace(' ', '_')}.ppm" cmd = [ @@ -176,31 +177,51 @@ class FFmpegRTSPReader: if self.frame_ready_event.wait(timeout=restart_check_interval): self.frame_ready_event.clear() - # Read PPM frame (uncompressed with header) + # Read PPM frame with persistent lock attempts until new inotify try: if os.path.exists(self.temp_file): - # Read PPM with OpenCV (handles RGB->BGR conversion automatically) - frame = cv2.imread(self.temp_file) + # Keep trying to acquire lock until new inotify event or success + max_attempts = 50 # ~500ms worth of attempts + for attempt in range(max_attempts): + # Check if new inotify event arrived (cancel current attempt) + if self.frame_ready_event.is_set(): + break - if frame is not None and frame.shape == (self.height, self.width, 3): - # Call frame callback directly - if self.frame_callback: - self.frame_callback(self.camera_id, frame) + try: + with open(self.temp_file, 'rb') as f: + # Try to acquire shared lock (non-blocking) + fcntl.flock(f.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB) - frame_count += 1 + # Success! File is locked, safe to read + frame = cv2.imread(self.temp_file) - # Log progress - current_time = time.time() - if current_time - last_log_time >= 30: - logger.info(f"Camera {self.camera_id}: {frame_count} PPM frames processed reactively") - last_log_time = current_time - else: - logger.debug(f"Camera {self.camera_id}: Invalid PPM frame") - else: - logger.debug(f"Camera {self.camera_id}: PPM file not found yet") + if frame is not None and frame.shape == (self.height, self.width, 3): + # Call frame callback directly + if self.frame_callback: + self.frame_callback(self.camera_id, frame) - except (IOError, OSError) as e: - logger.debug(f"Camera {self.camera_id}: File read error: {e}") + frame_count += 1 + + # Log progress + current_time = time.time() + if current_time - last_log_time >= 30: + logger.info(f"Camera {self.camera_id}: {frame_count} PPM frames processed with persistent locking") + last_log_time = current_time + # Invalid frame - just skip, no logging needed + + # Successfully processed frame + break + + except (OSError, IOError): + # File is still locked, wait a bit and try again + time.sleep(0.01) # 10ms wait between attempts + continue + + # If we get here, exhausted attempts or file not ready - just continue + + except (IOError, OSError): + # File errors are routine, just continue + pass except Exception as e: logger.error(f"Camera {self.camera_id}: Error in reactive frame reading: {e}") From f5c6da80140198ad8656e406d738f1cb984eed3c Mon Sep 17 00:00:00 2001 From: ziesorx Date: Fri, 26 Sep 2025 10:18:44 +0700 Subject: [PATCH 29/55] change: temp_file path --- core/streaming/readers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/streaming/readers.py b/core/streaming/readers.py index 35a7213..44fee34 100644 --- a/core/streaming/readers.py +++ b/core/streaming/readers.py @@ -92,11 +92,11 @@ class FFmpegRTSPReader: def _start_ffmpeg_process(self): """Start FFmpeg subprocess with CUDA hardware acceleration writing to temp file.""" # Create temp file path for this camera - self.temp_file = f"/tmp/claude/camera_{self.camera_id.replace(' ', '_')}.raw" - os.makedirs("/tmp/claude", exist_ok=True) + self.temp_file = f"/tmp/frame/camera_{self.camera_id.replace(' ', '_')}.raw" + os.makedirs("/tmp/frame", exist_ok=True) # Use PPM format with single file (will use file locking for concurrency) - self.temp_file = f"/tmp/claude/camera_{self.camera_id.replace(' ', '_')}.ppm" + self.temp_file = f"/tmp/frame/camera_{self.camera_id.replace(' ', '_')}.ppm" cmd = [ 'ffmpeg', From 83aaf95f594c83180353f37f305490a08c890524 Mon Sep 17 00:00:00 2001 From: ziesorx Date: Fri, 26 Sep 2025 11:24:48 +0700 Subject: [PATCH 30/55] fix: can read, track, and detect frame --- core/streaming/readers.py | 144 +++++++++++++++++++++----------------- 1 file changed, 79 insertions(+), 65 deletions(-) diff --git a/core/streaming/readers.py b/core/streaming/readers.py index 44fee34..d8d4b4d 100644 --- a/core/streaming/readers.py +++ b/core/streaming/readers.py @@ -10,7 +10,7 @@ import requests import numpy as np import os import subprocess -import fcntl +# import fcntl # No longer needed with atomic file operations from typing import Optional, Callable from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler @@ -24,6 +24,8 @@ logger = logging.getLogger(__name__) # Suppress noisy watchdog debug logs logging.getLogger('watchdog.observers.inotify_buffer').setLevel(logging.CRITICAL) +logging.getLogger('watchdog.observers.fsevents').setLevel(logging.CRITICAL) +logging.getLogger('fsevents').setLevel(logging.CRITICAL) class FrameFileHandler(FileSystemEventHandler): @@ -90,63 +92,68 @@ class FFmpegRTSPReader: logger.info(f"Stopped FFmpeg reader for camera {self.camera_id}") def _start_ffmpeg_process(self): - """Start FFmpeg subprocess with CUDA hardware acceleration writing to temp file.""" - # Create temp file path for this camera - self.temp_file = f"/tmp/frame/camera_{self.camera_id.replace(' ', '_')}.raw" - os.makedirs("/tmp/frame", exist_ok=True) + """Start FFmpeg subprocess writing timestamped frames for atomic reads.""" + # Create temp file paths for this camera + self.frame_dir = "/tmp/frame" + os.makedirs(self.frame_dir, exist_ok=True) - # Use PPM format with single file (will use file locking for concurrency) - self.temp_file = f"/tmp/frame/camera_{self.camera_id.replace(' ', '_')}.ppm" + # Use strftime pattern - FFmpeg writes each frame with unique timestamp + # This ensures each file is complete when written + camera_id_safe = self.camera_id.replace(' ', '_') + self.frame_prefix = f"camera_{camera_id_safe}" + # Using strftime pattern with microseconds for unique filenames + self.frame_pattern = f"{self.frame_dir}/{self.frame_prefix}_%Y%m%d_%H%M%S_%f.ppm" cmd = [ 'ffmpeg', + # DO NOT REMOVE '-hwaccel', 'cuda', '-hwaccel_device', '0', '-rtsp_transport', 'tcp', '-i', self.rtsp_url, '-f', 'image2', - '-update', '1', # Works with image2 format + '-strftime', '1', # Enable strftime pattern expansion '-pix_fmt', 'rgb24', # PPM uses RGB not BGR '-an', # No audio '-y', # Overwrite output file - self.temp_file + self.frame_pattern # Write timestamped frames ] try: + # Log the FFmpeg command for debugging + logger.info(f"Starting FFmpeg for camera {self.camera_id} with command: {' '.join(cmd)}") + # Start FFmpeg detached - we don't need to communicate with it self.process = subprocess.Popen( cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) - logger.info(f"Started FFmpeg process PID {self.process.pid} for camera {self.camera_id} -> {self.temp_file}") + logger.info(f"Started FFmpeg process PID {self.process.pid} for camera {self.camera_id} -> {self.frame_pattern}") return True except Exception as e: logger.error(f"Failed to start FFmpeg for camera {self.camera_id}: {e}") return False def _setup_file_watcher(self): - """Setup file system watcher for temp file.""" - if not os.path.exists(self.temp_file): - return - - # Setup file watcher - handler = FrameFileHandler(self._on_file_changed) + """Setup file system watcher for frame directory.""" + # Setup file watcher for the frame directory + handler = FrameFileHandler(lambda: self._on_file_changed()) self.observer = Observer() - self.observer.schedule(handler, os.path.dirname(self.temp_file), recursive=False) + self.observer.schedule(handler, self.frame_dir, recursive=False) self.observer.start() - logger.info(f"Started file watcher for {self.temp_file}") + logger.info(f"Started file watcher for {self.frame_dir} with pattern {self.frame_prefix}*.ppm") def _on_file_changed(self): - """Called when temp file is modified.""" - if os.path.basename(self.temp_file) in str(self.temp_file): - self.frame_ready_event.set() + """Called when a new frame file is created.""" + # Signal that a new frame might be available + self.frame_ready_event.set() def _read_frames(self): """Reactively read frames when file changes.""" frame_count = 0 last_log_time = time.time() - bytes_per_frame = self.width * self.height * 3 + # Remove unused variable: bytes_per_frame = self.width * self.height * 3 restart_check_interval = 10 # Check FFmpeg status every 10 seconds while not self.stop_event.is_set(): @@ -160,14 +167,21 @@ class FFmpegRTSPReader: time.sleep(5.0) continue - # Wait for temp file to be created + # Wait for FFmpeg to start writing frame files wait_count = 0 - while not os.path.exists(self.temp_file) and wait_count < 30: + while wait_count < 30: + # Check if any frame files exist + import glob + frame_files = glob.glob(f"{self.frame_dir}/{self.frame_prefix}*.ppm") + if frame_files: + logger.info(f"Found {len(frame_files)} initial frame files for {self.camera_id}") + break time.sleep(1.0) wait_count += 1 - if not os.path.exists(self.temp_file): - logger.error(f"Temp file not created after 30s for {self.camera_id}") + if wait_count >= 30: + logger.error(f"No frame files created after 30s for {self.camera_id}") + logger.error(f"Expected pattern: {self.frame_dir}/{self.frame_prefix}*.ppm") continue # Setup file watcher @@ -177,50 +191,44 @@ class FFmpegRTSPReader: if self.frame_ready_event.wait(timeout=restart_check_interval): self.frame_ready_event.clear() - # Read PPM frame with persistent lock attempts until new inotify + # Read latest complete frame file try: - if os.path.exists(self.temp_file): - # Keep trying to acquire lock until new inotify event or success - max_attempts = 50 # ~500ms worth of attempts - for attempt in range(max_attempts): - # Check if new inotify event arrived (cancel current attempt) - if self.frame_ready_event.is_set(): - break + import glob + # Find all frame files for this camera + frame_files = glob.glob(f"{self.frame_dir}/{self.frame_prefix}*.ppm") - try: - with open(self.temp_file, 'rb') as f: - # Try to acquire shared lock (non-blocking) - fcntl.flock(f.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB) + if frame_files: + # Sort by filename (which includes timestamp) and get the latest + frame_files.sort() + latest_frame = frame_files[-1] - # Success! File is locked, safe to read - frame = cv2.imread(self.temp_file) + # Read the latest frame (it's complete since FFmpeg wrote it atomically) + frame = cv2.imread(latest_frame) - if frame is not None and frame.shape == (self.height, self.width, 3): - # Call frame callback directly - if self.frame_callback: - self.frame_callback(self.camera_id, frame) + if frame is not None and frame.shape == (self.height, self.width, 3): + # Call frame callback directly + if self.frame_callback: + self.frame_callback(self.camera_id, frame) - frame_count += 1 + frame_count += 1 - # Log progress - current_time = time.time() - if current_time - last_log_time >= 30: - logger.info(f"Camera {self.camera_id}: {frame_count} PPM frames processed with persistent locking") - last_log_time = current_time - # Invalid frame - just skip, no logging needed + # Log progress + current_time = time.time() + if current_time - last_log_time >= 30: + logger.info(f"Camera {self.camera_id}: {frame_count} frames processed") + last_log_time = current_time - # Successfully processed frame - break + # Clean up old frame files to prevent disk filling + # Keep only the latest 5 frames + if len(frame_files) > 5: + for old_file in frame_files[:-5]: + try: + os.remove(old_file) + except: + pass - except (OSError, IOError): - # File is still locked, wait a bit and try again - time.sleep(0.01) # 10ms wait between attempts - continue - - # If we get here, exhausted attempts or file not ready - just continue - - except (IOError, OSError): - # File errors are routine, just continue + except Exception as e: + logger.debug(f"Camera {self.camera_id}: Error reading frames: {e}") pass except Exception as e: @@ -233,10 +241,16 @@ class FFmpegRTSPReader: self.observer.join() if self.process: self.process.terminate() - # Clean up temp file + # Clean up all frame files for this camera try: - if hasattr(self, 'temp_file') and os.path.exists(self.temp_file): - os.remove(self.temp_file) + if hasattr(self, 'frame_prefix') and hasattr(self, 'frame_dir'): + import glob + frame_files = glob.glob(f"{self.frame_dir}/{self.frame_prefix}*.ppm") + for frame_file in frame_files: + try: + os.remove(frame_file) + except: + pass except: pass logger.info(f"Reactive FFmpeg reader ended for camera {self.camera_id}") From 519e073f7f03e0f7d2fe5340404b12845c1f1c8c Mon Sep 17 00:00:00 2001 From: ziesorx Date: Fri, 26 Sep 2025 13:05:58 +0700 Subject: [PATCH 31/55] fix: camera api endpoint --- app.py | 56 ++++++++++++++++++--------------- core/communication/websocket.py | 2 ++ core/streaming/buffers.py | 44 +++++++------------------- core/streaming/manager.py | 16 +++++++--- core/streaming/readers.py | 19 +++++++---- 5 files changed, 69 insertions(+), 68 deletions(-) diff --git a/app.py b/app.py index 6338401..2e6a0c5 100644 --- a/app.py +++ b/app.py @@ -6,8 +6,9 @@ import json import logging import os import time +import cv2 from contextlib import asynccontextmanager -from fastapi import FastAPI, WebSocket, HTTPException, Request +from fastapi import FastAPI, WebSocket, HTTPException from fastapi.responses import Response # Import new modular communication system @@ -27,8 +28,8 @@ logging.basicConfig( logger = logging.getLogger("detector_worker") logger.setLevel(logging.DEBUG) -# Store cached frames for REST API access (temporary storage) -latest_frames = {} +# Frames are now stored in the shared cache buffer from core.streaming.buffers +# latest_frames = {} # Deprecated - using shared_cache_buffer instead # Lifespan event handler (modern FastAPI approach) @asynccontextmanager @@ -49,7 +50,7 @@ async def lifespan(app: FastAPI): worker_state.set_subscriptions([]) worker_state.session_ids.clear() worker_state.progression_stages.clear() - latest_frames.clear() + # latest_frames.clear() # No longer needed - frames are in shared_cache_buffer logger.info("Detector Worker shutdown complete") # Create FastAPI application with detailed WebSocket logging @@ -90,8 +91,8 @@ from core.streaming import initialize_stream_manager initialize_stream_manager(max_streams=config.get('max_streams', 10)) logger.info(f"Initialized stream manager with max_streams={config.get('max_streams', 10)}") -# Store cached frames for REST API access (temporary storage) -latest_frames = {} +# Frames are now stored in the shared cache buffer from core.streaming.buffers +# latest_frames = {} # Deprecated - using shared_cache_buffer instead logger.info("Starting detector worker application (refactored)") logger.info(f"Configuration: Target FPS: {config.get('target_fps', 10)}, " @@ -150,31 +151,36 @@ async def get_camera_image(camera_id: str): detail=f"Camera {camera_id} not found or not active" ) - # Check if we have a cached frame for this camera - if camera_id not in latest_frames: - logger.warning(f"No cached frame available for camera '{camera_id}'") + # Extract actual camera_id from subscription identifier (displayId;cameraId) + # Frames are stored using just the camera_id part + actual_camera_id = camera_id.split(';')[-1] if ';' in camera_id else camera_id + + # Get frame from the shared cache buffer + from core.streaming.buffers import shared_cache_buffer + + # Debug: Log available cameras in buffer + available_cameras = shared_cache_buffer.frame_buffer.get_camera_list() + logger.debug(f"Available cameras in buffer: {available_cameras}") + logger.debug(f"Looking for camera: '{actual_camera_id}'") + + frame = shared_cache_buffer.get_frame(actual_camera_id) + if frame is None: + logger.warning(f"No cached frame available for camera '{actual_camera_id}' (from subscription '{camera_id}')") + logger.warning(f"Available cameras in buffer: {available_cameras}") raise HTTPException( status_code=404, - detail=f"No frame available for camera {camera_id}" + detail=f"No frame available for camera {actual_camera_id}" ) - frame = latest_frames[camera_id] - logger.debug(f"Retrieved cached frame for camera '{camera_id}', shape: {frame.shape}") + logger.debug(f"Retrieved cached frame for camera '{actual_camera_id}' (from subscription '{camera_id}'), shape: {frame.shape}") - # TODO: This import will be replaced in Phase 3 (Streaming System) - # For now, we need to handle the case where OpenCV is not available - try: - import cv2 - # Encode frame as JPEG - success, buffer_img = cv2.imencode('.jpg', frame, [cv2.IMWRITE_JPEG_QUALITY, 85]) - if not success: - raise HTTPException(status_code=500, detail="Failed to encode image as JPEG") + # Encode frame as JPEG + success, buffer_img = cv2.imencode('.jpg', frame, [cv2.IMWRITE_JPEG_QUALITY, 85]) + if not success: + raise HTTPException(status_code=500, detail="Failed to encode image as JPEG") - # Return image as binary response - return Response(content=buffer_img.tobytes(), media_type="image/jpeg") - except ImportError: - logger.error("OpenCV not available for image encoding") - raise HTTPException(status_code=500, detail="Image processing not available") + # Return image as binary response + return Response(content=buffer_img.tobytes(), media_type="image/jpeg") except HTTPException: raise diff --git a/core/communication/websocket.py b/core/communication/websocket.py index 813350e..077c6dc 100644 --- a/core/communication/websocket.py +++ b/core/communication/websocket.py @@ -377,6 +377,8 @@ class WebSocketHandler: camera_id = subscription_id.split(';')[-1] model_id = payload['modelId'] + logger.info(f"[SUBSCRIPTION_MAPPING] subscription_id='{subscription_id}' → camera_id='{camera_id}'") + # Get tracking integration for this model tracking_integration = tracking_integrations.get(model_id) diff --git a/core/streaming/buffers.py b/core/streaming/buffers.py index fd29fbb..f2c5787 100644 --- a/core/streaming/buffers.py +++ b/core/streaming/buffers.py @@ -46,13 +46,7 @@ class FrameBuffer: frame_data = self._frames[camera_id] - # Check if frame is too old - age = time.time() - frame_data['timestamp'] - if age > self.max_age_seconds: - logger.debug(f"Frame for camera {camera_id} is {age:.1f}s old, discarding") - del self._frames[camera_id] - return None - + # Return frame regardless of age - frames persist until replaced return frame_data['frame'].copy() def get_frame_info(self, camera_id: str) -> Optional[Dict[str, Any]]: @@ -64,10 +58,7 @@ class FrameBuffer: frame_data = self._frames[camera_id] age = time.time() - frame_data['timestamp'] - if age > self.max_age_seconds: - del self._frames[camera_id] - return None - + # Return frame info regardless of age - frames persist until replaced return { 'timestamp': frame_data['timestamp'], 'age': age, @@ -95,24 +86,10 @@ class FrameBuffer: logger.debug(f"Cleared all frames ({count} cameras)") def get_camera_list(self) -> list: - """Get list of cameras with valid frames.""" + """Get list of cameras with frames - all frames persist until replaced.""" with self._lock: - current_time = time.time() - valid_cameras = [] - expired_cameras = [] - - for camera_id, frame_data in self._frames.items(): - age = current_time - frame_data['timestamp'] - if age <= self.max_age_seconds: - valid_cameras.append(camera_id) - else: - expired_cameras.append(camera_id) - - # Clean up expired frames - for camera_id in expired_cameras: - del self._frames[camera_id] - - return valid_cameras + # Return all cameras that have frames - no age-based filtering + return list(self._frames.keys()) def get_stats(self) -> Dict[str, Any]: """Get buffer statistics.""" @@ -120,8 +97,8 @@ class FrameBuffer: current_time = time.time() stats = { 'total_cameras': len(self._frames), - 'valid_cameras': 0, - 'expired_cameras': 0, + 'recent_cameras': 0, + 'stale_cameras': 0, 'total_memory_mb': 0, 'cameras': {} } @@ -130,16 +107,17 @@ class FrameBuffer: age = current_time - frame_data['timestamp'] size_mb = frame_data.get('size_mb', 0) + # All frames are valid/available, but categorize by freshness for monitoring if age <= self.max_age_seconds: - stats['valid_cameras'] += 1 + stats['recent_cameras'] += 1 else: - stats['expired_cameras'] += 1 + stats['stale_cameras'] += 1 stats['total_memory_mb'] += size_mb stats['cameras'][camera_id] = { 'age': age, - 'valid': age <= self.max_age_seconds, + 'recent': age <= self.max_age_seconds, # Recent but all frames available 'shape': frame_data['shape'], 'dtype': frame_data['dtype'], 'size_mb': size_mb diff --git a/core/streaming/manager.py b/core/streaming/manager.py index 156daf1..0c172ac 100644 --- a/core/streaming/manager.py +++ b/core/streaming/manager.py @@ -130,6 +130,7 @@ class StreamManager: try: if stream_config.rtsp_url: # RTSP stream using FFmpeg subprocess with CUDA acceleration + logger.info(f"[STREAM_START] Starting FFmpeg RTSP stream for camera_id='{camera_id}' URL={stream_config.rtsp_url}") reader = FFmpegRTSPReader( camera_id=camera_id, rtsp_url=stream_config.rtsp_url, @@ -138,10 +139,11 @@ class StreamManager: reader.set_frame_callback(self._frame_callback) reader.start() self._streams[camera_id] = reader - logger.info(f"Started FFmpeg RTSP stream for camera {camera_id}") + logger.info(f"[STREAM_START] ✅ Started FFmpeg RTSP stream for camera_id='{camera_id}'") elif stream_config.snapshot_url: # HTTP snapshot stream + logger.info(f"[STREAM_START] Starting HTTP snapshot stream for camera_id='{camera_id}' URL={stream_config.snapshot_url}") reader = HTTPSnapshotReader( camera_id=camera_id, snapshot_url=stream_config.snapshot_url, @@ -151,7 +153,7 @@ class StreamManager: reader.set_frame_callback(self._frame_callback) reader.start() self._streams[camera_id] = reader - logger.info(f"Started HTTP snapshot stream for camera {camera_id}") + logger.info(f"[STREAM_START] ✅ Started HTTP snapshot stream for camera_id='{camera_id}'") else: logger.error(f"No valid URL provided for camera {camera_id}") @@ -169,8 +171,9 @@ class StreamManager: try: self._streams[camera_id].stop() del self._streams[camera_id] - shared_cache_buffer.clear_camera(camera_id) - logger.info(f"Stopped stream for camera {camera_id}") + # DON'T clear frames - they should persist until replaced + # shared_cache_buffer.clear_camera(camera_id) # REMOVED - frames should persist + logger.info(f"Stopped stream for camera {camera_id} (frames preserved in buffer)") except Exception as e: logger.error(f"Error stopping stream for camera {camera_id}: {e}") @@ -179,6 +182,11 @@ class StreamManager: try: # Store frame in shared buffer shared_cache_buffer.put_frame(camera_id, frame) + logger.info(f"[FRAME_CALLBACK] Stored frame for camera_id='{camera_id}' in shared_cache_buffer, shape={frame.shape}") + + # Log current buffer state + available_cameras = shared_cache_buffer.frame_buffer.get_camera_list() + logger.info(f"[FRAME_CALLBACK] Buffer now contains {len(available_cameras)} cameras: {available_cameras}") # Process tracking for subscriptions with tracking integration self._process_tracking_for_camera(camera_id, frame) diff --git a/core/streaming/readers.py b/core/streaming/readers.py index d8d4b4d..4b5c8ba 100644 --- a/core/streaming/readers.py +++ b/core/streaming/readers.py @@ -101,14 +101,14 @@ class FFmpegRTSPReader: # This ensures each file is complete when written camera_id_safe = self.camera_id.replace(' ', '_') self.frame_prefix = f"camera_{camera_id_safe}" - # Using strftime pattern with microseconds for unique filenames - self.frame_pattern = f"{self.frame_dir}/{self.frame_prefix}_%Y%m%d_%H%M%S_%f.ppm" + # Using strftime pattern with seconds for unique filenames (avoid %f which may not work) + self.frame_pattern = f"{self.frame_dir}/{self.frame_prefix}_%Y%m%d_%H%M%S.ppm" cmd = [ 'ffmpeg', # DO NOT REMOVE - '-hwaccel', 'cuda', - '-hwaccel_device', '0', + # '-hwaccel', 'cuda', + # '-hwaccel_device', '0', '-rtsp_transport', 'tcp', '-i', self.rtsp_url, '-f', 'image2', @@ -201,14 +201,17 @@ class FFmpegRTSPReader: # Sort by filename (which includes timestamp) and get the latest frame_files.sort() latest_frame = frame_files[-1] + logger.debug(f"Camera {self.camera_id}: Found {len(frame_files)} frames, processing latest: {latest_frame}") # Read the latest frame (it's complete since FFmpeg wrote it atomically) frame = cv2.imread(latest_frame) - if frame is not None and frame.shape == (self.height, self.width, 3): - # Call frame callback directly + if frame is not None: + logger.debug(f"Camera {self.camera_id}: Successfully read frame {frame.shape} from {latest_frame}") + # Accept any frame dimensions initially for debugging if self.frame_callback: self.frame_callback(self.camera_id, frame) + logger.debug(f"Camera {self.camera_id}: Called frame callback") frame_count += 1 @@ -217,6 +220,8 @@ class FFmpegRTSPReader: if current_time - last_log_time >= 30: logger.info(f"Camera {self.camera_id}: {frame_count} frames processed") last_log_time = current_time + else: + logger.warning(f"Camera {self.camera_id}: Failed to read frame from {latest_frame}") # Clean up old frame files to prevent disk filling # Keep only the latest 5 frames @@ -226,6 +231,8 @@ class FFmpegRTSPReader: os.remove(old_file) except: pass + else: + logger.warning(f"Camera {self.camera_id}: No frame files found in {self.frame_dir} with pattern {self.frame_prefix}*.ppm") except Exception as e: logger.debug(f"Camera {self.camera_id}: Error reading frames: {e}") From bd201acac1e942920611d329408fea7dc3d7ad88 Mon Sep 17 00:00:00 2001 From: ziesorx Date: Fri, 26 Sep 2025 13:16:37 +0700 Subject: [PATCH 32/55] fix: cameras buffer --- core/streaming/readers.py | 270 ++++++++++++++++++-------------------- 1 file changed, 127 insertions(+), 143 deletions(-) diff --git a/core/streaming/readers.py b/core/streaming/readers.py index 4b5c8ba..d17a229 100644 --- a/core/streaming/readers.py +++ b/core/streaming/readers.py @@ -12,8 +12,7 @@ import os import subprocess # import fcntl # No longer needed with atomic file operations from typing import Optional, Callable -from watchdog.observers import Observer -from watchdog.events import FileSystemEventHandler +# Removed watchdog imports - no longer using file watching # Suppress FFMPEG/H.264 error messages if needed # Set this environment variable to reduce noise from decoder errors @@ -22,31 +21,14 @@ os.environ["OPENCV_FFMPEG_LOGLEVEL"] = "-8" # Suppress FFMPEG warnings logger = logging.getLogger(__name__) -# Suppress noisy watchdog debug logs -logging.getLogger('watchdog.observers.inotify_buffer').setLevel(logging.CRITICAL) -logging.getLogger('watchdog.observers.fsevents').setLevel(logging.CRITICAL) -logging.getLogger('fsevents').setLevel(logging.CRITICAL) +# Removed watchdog logging configuration - no longer using file watching -class FrameFileHandler(FileSystemEventHandler): - """File system event handler for frame file changes.""" - - def __init__(self, callback): - self.callback = callback - self.last_modified = 0 - - def on_modified(self, event): - if event.is_directory: - return - # Debounce rapid file changes - current_time = time.time() - if current_time - self.last_modified > 0.01: # 10ms debounce - self.last_modified = current_time - self.callback() +# Removed FrameFileHandler - no longer using file watching class FFmpegRTSPReader: - """RTSP stream reader using subprocess FFmpeg with CUDA hardware acceleration and file watching.""" + """RTSP stream reader using subprocess FFmpeg piping frames directly to buffer.""" def __init__(self, camera_id: str, rtsp_url: str, max_retries: int = 3): self.camera_id = camera_id @@ -56,10 +38,8 @@ class FFmpegRTSPReader: self.stop_event = threading.Event() self.thread = None self.frame_callback: Optional[Callable] = None - self.observer = None - self.frame_ready_event = threading.Event() - # Stream specs + # Expected stream specs (for reference, actual dimensions read from PPM header) self.width = 1280 self.height = 720 @@ -91,18 +71,58 @@ class FFmpegRTSPReader: self.thread.join(timeout=5.0) logger.info(f"Stopped FFmpeg reader for camera {self.camera_id}") - def _start_ffmpeg_process(self): - """Start FFmpeg subprocess writing timestamped frames for atomic reads.""" - # Create temp file paths for this camera - self.frame_dir = "/tmp/frame" - os.makedirs(self.frame_dir, exist_ok=True) + def _probe_stream_info(self): + """Probe stream to get resolution and other info.""" + try: + cmd = [ + 'ffprobe', + '-v', 'quiet', + '-print_format', 'json', + '-show_streams', + '-select_streams', 'v:0', # First video stream + '-rtsp_transport', 'tcp', + self.rtsp_url + ] - # Use strftime pattern - FFmpeg writes each frame with unique timestamp - # This ensures each file is complete when written - camera_id_safe = self.camera_id.replace(' ', '_') - self.frame_prefix = f"camera_{camera_id_safe}" - # Using strftime pattern with seconds for unique filenames (avoid %f which may not work) - self.frame_pattern = f"{self.frame_dir}/{self.frame_prefix}_%Y%m%d_%H%M%S.ppm" + result = subprocess.run(cmd, capture_output=True, text=True, timeout=10) + if result.returncode != 0: + logger.error(f"Camera {self.camera_id}: ffprobe failed (code {result.returncode})") + if result.stderr: + logger.error(f"Camera {self.camera_id}: ffprobe stderr: {result.stderr}") + if result.stdout: + logger.debug(f"Camera {self.camera_id}: ffprobe stdout: {result.stdout}") + return None + + import json + data = json.loads(result.stdout) + if not data.get('streams'): + logger.error(f"Camera {self.camera_id}: No video streams found") + return None + + stream = data['streams'][0] + width = stream.get('width') + height = stream.get('height') + + if not width or not height: + logger.error(f"Camera {self.camera_id}: Could not determine resolution") + return None + + logger.info(f"Camera {self.camera_id}: Detected resolution {width}x{height}") + return width, height + + except Exception as e: + logger.error(f"Camera {self.camera_id}: Error probing stream: {e}") + return None + + def _start_ffmpeg_process(self): + """Start FFmpeg subprocess outputting raw RGB frames to stdout pipe.""" + # First probe the stream to get resolution + probe_result = self._probe_stream_info() + if not probe_result: + logger.error(f"Camera {self.camera_id}: Failed to probe stream info") + return False + + self.actual_width, self.actual_height = probe_result cmd = [ 'ffmpeg', @@ -111,50 +131,69 @@ class FFmpegRTSPReader: # '-hwaccel_device', '0', '-rtsp_transport', 'tcp', '-i', self.rtsp_url, - '-f', 'image2', - '-strftime', '1', # Enable strftime pattern expansion - '-pix_fmt', 'rgb24', # PPM uses RGB not BGR - '-an', # No audio - '-y', # Overwrite output file - self.frame_pattern # Write timestamped frames + '-f', 'rawvideo', # Raw video output instead of PPM + '-pix_fmt', 'rgb24', # Raw RGB24 format + # Use native stream resolution and framerate + '-an', # No audio + '-' # Output to stdout ] try: # Log the FFmpeg command for debugging logger.info(f"Starting FFmpeg for camera {self.camera_id} with command: {' '.join(cmd)}") - # Start FFmpeg detached - we don't need to communicate with it + # Start FFmpeg with stdout pipe to read frames directly self.process = subprocess.Popen( cmd, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL + stdout=subprocess.PIPE, # Capture stdout for frame data + stderr=subprocess.DEVNULL, + bufsize=0 # Unbuffered for real-time processing ) - logger.info(f"Started FFmpeg process PID {self.process.pid} for camera {self.camera_id} -> {self.frame_pattern}") + logger.info(f"Started FFmpeg process PID {self.process.pid} for camera {self.camera_id} -> stdout pipe (resolution: {self.actual_width}x{self.actual_height})") return True except Exception as e: logger.error(f"Failed to start FFmpeg for camera {self.camera_id}: {e}") return False - def _setup_file_watcher(self): - """Setup file system watcher for frame directory.""" - # Setup file watcher for the frame directory - handler = FrameFileHandler(lambda: self._on_file_changed()) - self.observer = Observer() - self.observer.schedule(handler, self.frame_dir, recursive=False) - self.observer.start() - logger.info(f"Started file watcher for {self.frame_dir} with pattern {self.frame_prefix}*.ppm") + def _read_raw_frame(self, pipe): + """Read raw RGB frame data from pipe with proper buffering.""" + try: + # Calculate frame size using actual detected dimensions + frame_size = self.actual_width * self.actual_height * 3 - def _on_file_changed(self): - """Called when a new frame file is created.""" - # Signal that a new frame might be available - self.frame_ready_event.set() + # Read frame data in chunks until we have the complete frame + frame_data = b'' + bytes_remaining = frame_size + + while bytes_remaining > 0: + chunk = pipe.read(bytes_remaining) + if not chunk: # EOF + if len(frame_data) == 0: + logger.debug(f"Camera {self.camera_id}: No more data (stream ended)") + else: + logger.warning(f"Camera {self.camera_id}: Stream ended mid-frame: {len(frame_data)}/{frame_size} bytes") + return None + + frame_data += chunk + bytes_remaining -= len(chunk) + + # Convert raw RGB data to numpy array using actual dimensions + frame_array = np.frombuffer(frame_data, dtype=np.uint8) + frame_rgb = frame_array.reshape((self.actual_height, self.actual_width, 3)) + + # Convert RGB to BGR for OpenCV compatibility + frame_bgr = cv2.cvtColor(frame_rgb, cv2.COLOR_RGB2BGR) + + return frame_bgr + + except Exception as e: + logger.error(f"Camera {self.camera_id}: Error reading raw frame: {e}") + return None def _read_frames(self): - """Reactively read frames when file changes.""" + """Read frames directly from FFmpeg stdout pipe.""" frame_count = 0 last_log_time = time.time() - # Remove unused variable: bytes_per_frame = self.width * self.height * 3 - restart_check_interval = 10 # Check FFmpeg status every 10 seconds while not self.stop_event.is_set(): try: @@ -167,100 +206,45 @@ class FFmpegRTSPReader: time.sleep(5.0) continue - # Wait for FFmpeg to start writing frame files - wait_count = 0 - while wait_count < 30: - # Check if any frame files exist - import glob - frame_files = glob.glob(f"{self.frame_dir}/{self.frame_prefix}*.ppm") - if frame_files: - logger.info(f"Found {len(frame_files)} initial frame files for {self.camera_id}") - break - time.sleep(1.0) - wait_count += 1 + logger.info(f"FFmpeg started for camera {self.camera_id}, reading frames from pipe...") - if wait_count >= 30: - logger.error(f"No frame files created after 30s for {self.camera_id}") - logger.error(f"Expected pattern: {self.frame_dir}/{self.frame_prefix}*.ppm") - continue + # Read frames directly from FFmpeg stdout + try: + if self.process and self.process.stdout: + # Read raw frame data + frame = self._read_raw_frame(self.process.stdout) + if frame is None: + continue - # Setup file watcher - self._setup_file_watcher() + # Call frame callback + if self.frame_callback: + self.frame_callback(self.camera_id, frame) + logger.debug(f"Camera {self.camera_id}: Called frame callback with shape {frame.shape}") - # Wait for file change event (or timeout for health check) - if self.frame_ready_event.wait(timeout=restart_check_interval): - self.frame_ready_event.clear() + frame_count += 1 - # Read latest complete frame file - try: - import glob - # Find all frame files for this camera - frame_files = glob.glob(f"{self.frame_dir}/{self.frame_prefix}*.ppm") + # Log progress + current_time = time.time() + if current_time - last_log_time >= 30: + logger.info(f"Camera {self.camera_id}: {frame_count} frames processed via pipe") + last_log_time = current_time - if frame_files: - # Sort by filename (which includes timestamp) and get the latest - frame_files.sort() - latest_frame = frame_files[-1] - logger.debug(f"Camera {self.camera_id}: Found {len(frame_files)} frames, processing latest: {latest_frame}") - - # Read the latest frame (it's complete since FFmpeg wrote it atomically) - frame = cv2.imread(latest_frame) - - if frame is not None: - logger.debug(f"Camera {self.camera_id}: Successfully read frame {frame.shape} from {latest_frame}") - # Accept any frame dimensions initially for debugging - if self.frame_callback: - self.frame_callback(self.camera_id, frame) - logger.debug(f"Camera {self.camera_id}: Called frame callback") - - frame_count += 1 - - # Log progress - current_time = time.time() - if current_time - last_log_time >= 30: - logger.info(f"Camera {self.camera_id}: {frame_count} frames processed") - last_log_time = current_time - else: - logger.warning(f"Camera {self.camera_id}: Failed to read frame from {latest_frame}") - - # Clean up old frame files to prevent disk filling - # Keep only the latest 5 frames - if len(frame_files) > 5: - for old_file in frame_files[:-5]: - try: - os.remove(old_file) - except: - pass - else: - logger.warning(f"Camera {self.camera_id}: No frame files found in {self.frame_dir} with pattern {self.frame_prefix}*.ppm") - - except Exception as e: - logger.debug(f"Camera {self.camera_id}: Error reading frames: {e}") - pass + except Exception as e: + logger.error(f"Camera {self.camera_id}: Error reading from pipe: {e}") + # Process might have died, let it restart on next iteration + if self.process: + self.process.terminate() + self.process = None + time.sleep(1.0) except Exception as e: - logger.error(f"Camera {self.camera_id}: Error in reactive frame reading: {e}") + logger.error(f"Camera {self.camera_id}: Error in pipe frame reading: {e}") time.sleep(1.0) # Cleanup - if self.observer: - self.observer.stop() - self.observer.join() if self.process: self.process.terminate() - # Clean up all frame files for this camera - try: - if hasattr(self, 'frame_prefix') and hasattr(self, 'frame_dir'): - import glob - frame_files = glob.glob(f"{self.frame_dir}/{self.frame_prefix}*.ppm") - for frame_file in frame_files: - try: - os.remove(frame_file) - except: - pass - except: - pass - logger.info(f"Reactive FFmpeg reader ended for camera {self.camera_id}") + logger.info(f"FFmpeg pipe reader ended for camera {self.camera_id}") logger = logging.getLogger(__name__) From 791f611f7d36924bd1ce6f0776e0dc140f3c8096 Mon Sep 17 00:00:00 2001 From: ziesorx Date: Fri, 26 Sep 2025 14:22:38 +0700 Subject: [PATCH 33/55] feat: custom bot-sort based tracker --- app.py | 9 +- core/models/inference.py | 47 +--- core/streaming/manager.py | 21 +- core/streaming/readers.py | 184 ++++++-------- core/tracking/bot_sort_tracker.py | 408 ++++++++++++++++++++++++++++++ core/tracking/integration.py | 10 +- core/tracking/tracker.py | 233 ++++++++--------- core/tracking/validator.py | 19 +- 8 files changed, 649 insertions(+), 282 deletions(-) create mode 100644 core/tracking/bot_sort_tracker.py diff --git a/app.py b/app.py index 2e6a0c5..605aa0b 100644 --- a/app.py +++ b/app.py @@ -158,21 +158,18 @@ async def get_camera_image(camera_id: str): # Get frame from the shared cache buffer from core.streaming.buffers import shared_cache_buffer - # Debug: Log available cameras in buffer + # Only show buffer debug info if camera not found (to reduce log spam) available_cameras = shared_cache_buffer.frame_buffer.get_camera_list() - logger.debug(f"Available cameras in buffer: {available_cameras}") - logger.debug(f"Looking for camera: '{actual_camera_id}'") frame = shared_cache_buffer.get_frame(actual_camera_id) if frame is None: - logger.warning(f"No cached frame available for camera '{actual_camera_id}' (from subscription '{camera_id}')") - logger.warning(f"Available cameras in buffer: {available_cameras}") + logger.warning(f"\033[93m[API] No frame for '{actual_camera_id}' - Available: {available_cameras}\033[0m") raise HTTPException( status_code=404, detail=f"No frame available for camera {actual_camera_id}" ) - logger.debug(f"Retrieved cached frame for camera '{actual_camera_id}' (from subscription '{camera_id}'), shape: {frame.shape}") + # Successful frame retrieval - log only occasionally to avoid spam # Encode frame as JPEG success, buffer_img = cv2.imencode('.jpg', frame, [cv2.IMWRITE_JPEG_QUALITY, 85]) diff --git a/core/models/inference.py b/core/models/inference.py index 826061c..f96c0e8 100644 --- a/core/models/inference.py +++ b/core/models/inference.py @@ -60,6 +60,8 @@ class YOLOWrapper: self.model = None self._class_names = [] + + self._load_model() logger.info(f"Initialized YOLO wrapper for {model_id} on {self.device}") @@ -115,6 +117,7 @@ class YOLOWrapper: logger.error(f"Failed to extract class names: {str(e)}") self._class_names = {} + def infer( self, image: np.ndarray, @@ -222,55 +225,30 @@ class YOLOWrapper: return detections + def track( self, image: np.ndarray, confidence_threshold: float = 0.5, trigger_classes: Optional[List[str]] = None, - persist: bool = True + persist: bool = True, + camera_id: Optional[str] = None ) -> InferenceResult: """ - Run tracking on an image + Run detection (tracking will be handled by external tracker) Args: image: Input image as numpy array (BGR format) confidence_threshold: Minimum confidence for detections trigger_classes: List of class names to filter - persist: Whether to persist tracks across frames + persist: Ignored - tracking handled externally + camera_id: Ignored - tracking handled externally Returns: - InferenceResult containing detections with track IDs + InferenceResult containing detections (no track IDs from YOLO) """ - if self.model is None: - raise RuntimeError(f"Model {self.model_id} not loaded") - - try: - import time - start_time = time.time() - - # Run tracking - results = self.model.track( - image, - conf=confidence_threshold, - persist=persist, - verbose=False - ) - - inference_time = time.time() - start_time - - # Parse results - detections = self._parse_results(results[0], trigger_classes) - - return InferenceResult( - detections=detections, - image_shape=(image.shape[0], image.shape[1]), - inference_time=inference_time, - model_id=self.model_id - ) - - except Exception as e: - logger.error(f"Tracking failed for model {self.model_id}: {str(e)}", exc_info=True) - raise + # Just do detection - no YOLO tracking + return self.infer(image, confidence_threshold, trigger_classes) def predict_classification( self, @@ -350,6 +328,7 @@ class YOLOWrapper: """Get the number of classes the model can detect""" return len(self._class_names) + def clear_cache(self) -> None: """Clear the model cache""" with self._cache_lock: diff --git a/core/streaming/manager.py b/core/streaming/manager.py index 0c172ac..f6cfbda 100644 --- a/core/streaming/manager.py +++ b/core/streaming/manager.py @@ -130,7 +130,7 @@ class StreamManager: try: if stream_config.rtsp_url: # RTSP stream using FFmpeg subprocess with CUDA acceleration - logger.info(f"[STREAM_START] Starting FFmpeg RTSP stream for camera_id='{camera_id}' URL={stream_config.rtsp_url}") + logger.info(f"\033[94m[RTSP] Starting {camera_id}\033[0m") reader = FFmpegRTSPReader( camera_id=camera_id, rtsp_url=stream_config.rtsp_url, @@ -139,11 +139,11 @@ class StreamManager: reader.set_frame_callback(self._frame_callback) reader.start() self._streams[camera_id] = reader - logger.info(f"[STREAM_START] ✅ Started FFmpeg RTSP stream for camera_id='{camera_id}'") + logger.info(f"\033[92m[RTSP] {camera_id} connected\033[0m") elif stream_config.snapshot_url: # HTTP snapshot stream - logger.info(f"[STREAM_START] Starting HTTP snapshot stream for camera_id='{camera_id}' URL={stream_config.snapshot_url}") + logger.info(f"\033[95m[HTTP] Starting {camera_id}\033[0m") reader = HTTPSnapshotReader( camera_id=camera_id, snapshot_url=stream_config.snapshot_url, @@ -153,7 +153,7 @@ class StreamManager: reader.set_frame_callback(self._frame_callback) reader.start() self._streams[camera_id] = reader - logger.info(f"[STREAM_START] ✅ Started HTTP snapshot stream for camera_id='{camera_id}'") + logger.info(f"\033[92m[HTTP] {camera_id} connected\033[0m") else: logger.error(f"No valid URL provided for camera {camera_id}") @@ -182,11 +182,16 @@ class StreamManager: try: # Store frame in shared buffer shared_cache_buffer.put_frame(camera_id, frame) - logger.info(f"[FRAME_CALLBACK] Stored frame for camera_id='{camera_id}' in shared_cache_buffer, shape={frame.shape}") + # Quieter frame callback logging - only log occasionally + if hasattr(self, '_frame_log_count'): + self._frame_log_count += 1 + else: + self._frame_log_count = 1 - # Log current buffer state - available_cameras = shared_cache_buffer.frame_buffer.get_camera_list() - logger.info(f"[FRAME_CALLBACK] Buffer now contains {len(available_cameras)} cameras: {available_cameras}") + # Log every 100 frames to avoid spam + if self._frame_log_count % 100 == 0: + available_cameras = shared_cache_buffer.frame_buffer.get_camera_list() + logger.info(f"\033[96m[BUFFER] {len(available_cameras)} active cameras: {', '.join(available_cameras)}\033[0m") # Process tracking for subscriptions with tracking integration self._process_tracking_for_camera(camera_id, frame) diff --git a/core/streaming/readers.py b/core/streaming/readers.py index d17a229..d5635ba 100644 --- a/core/streaming/readers.py +++ b/core/streaming/readers.py @@ -21,6 +21,34 @@ os.environ["OPENCV_FFMPEG_LOGLEVEL"] = "-8" # Suppress FFMPEG warnings logger = logging.getLogger(__name__) +# Color codes for pretty logging +class Colors: + GREEN = '\033[92m' + YELLOW = '\033[93m' + RED = '\033[91m' + BLUE = '\033[94m' + PURPLE = '\033[95m' + CYAN = '\033[96m' + WHITE = '\033[97m' + BOLD = '\033[1m' + END = '\033[0m' + +def log_success(camera_id: str, message: str): + """Log success messages in green""" + logger.info(f"{Colors.GREEN}[{camera_id}] {message}{Colors.END}") + +def log_warning(camera_id: str, message: str): + """Log warnings in yellow""" + logger.warning(f"{Colors.YELLOW}[{camera_id}] {message}{Colors.END}") + +def log_error(camera_id: str, message: str): + """Log errors in red""" + logger.error(f"{Colors.RED}[{camera_id}] {message}{Colors.END}") + +def log_info(camera_id: str, message: str): + """Log info in cyan""" + logger.info(f"{Colors.CYAN}[{camera_id}] {message}{Colors.END}") + # Removed watchdog logging configuration - no longer using file watching @@ -56,7 +84,7 @@ class FFmpegRTSPReader: self.stop_event.clear() self.thread = threading.Thread(target=self._read_frames, daemon=True) self.thread.start() - logger.info(f"Started FFmpeg reader for camera {self.camera_id}") + log_success(self.camera_id, "Stream started") def stop(self): """Stop the FFmpeg subprocess reader.""" @@ -69,61 +97,12 @@ class FFmpegRTSPReader: self.process.kill() if self.thread: self.thread.join(timeout=5.0) - logger.info(f"Stopped FFmpeg reader for camera {self.camera_id}") + log_info(self.camera_id, "Stream stopped") - def _probe_stream_info(self): - """Probe stream to get resolution and other info.""" - try: - cmd = [ - 'ffprobe', - '-v', 'quiet', - '-print_format', 'json', - '-show_streams', - '-select_streams', 'v:0', # First video stream - '-rtsp_transport', 'tcp', - self.rtsp_url - ] - - result = subprocess.run(cmd, capture_output=True, text=True, timeout=10) - if result.returncode != 0: - logger.error(f"Camera {self.camera_id}: ffprobe failed (code {result.returncode})") - if result.stderr: - logger.error(f"Camera {self.camera_id}: ffprobe stderr: {result.stderr}") - if result.stdout: - logger.debug(f"Camera {self.camera_id}: ffprobe stdout: {result.stdout}") - return None - - import json - data = json.loads(result.stdout) - if not data.get('streams'): - logger.error(f"Camera {self.camera_id}: No video streams found") - return None - - stream = data['streams'][0] - width = stream.get('width') - height = stream.get('height') - - if not width or not height: - logger.error(f"Camera {self.camera_id}: Could not determine resolution") - return None - - logger.info(f"Camera {self.camera_id}: Detected resolution {width}x{height}") - return width, height - - except Exception as e: - logger.error(f"Camera {self.camera_id}: Error probing stream: {e}") - return None + # Removed _probe_stream_info - BMP headers contain dimensions def _start_ffmpeg_process(self): - """Start FFmpeg subprocess outputting raw RGB frames to stdout pipe.""" - # First probe the stream to get resolution - probe_result = self._probe_stream_info() - if not probe_result: - logger.error(f"Camera {self.camera_id}: Failed to probe stream info") - return False - - self.actual_width, self.actual_height = probe_result - + """Start FFmpeg subprocess outputting BMP frames to stdout pipe.""" cmd = [ 'ffmpeg', # DO NOT REMOVE @@ -131,17 +110,14 @@ class FFmpegRTSPReader: # '-hwaccel_device', '0', '-rtsp_transport', 'tcp', '-i', self.rtsp_url, - '-f', 'rawvideo', # Raw video output instead of PPM - '-pix_fmt', 'rgb24', # Raw RGB24 format + '-f', 'image2pipe', # Output images to pipe + '-vcodec', 'bmp', # BMP format with header containing dimensions # Use native stream resolution and framerate '-an', # No audio '-' # Output to stdout ] try: - # Log the FFmpeg command for debugging - logger.info(f"Starting FFmpeg for camera {self.camera_id} with command: {' '.join(cmd)}") - # Start FFmpeg with stdout pipe to read frames directly self.process = subprocess.Popen( cmd, @@ -149,46 +125,60 @@ class FFmpegRTSPReader: stderr=subprocess.DEVNULL, bufsize=0 # Unbuffered for real-time processing ) - logger.info(f"Started FFmpeg process PID {self.process.pid} for camera {self.camera_id} -> stdout pipe (resolution: {self.actual_width}x{self.actual_height})") return True except Exception as e: - logger.error(f"Failed to start FFmpeg for camera {self.camera_id}: {e}") + log_error(self.camera_id, f"FFmpeg startup failed: {e}") return False - def _read_raw_frame(self, pipe): - """Read raw RGB frame data from pipe with proper buffering.""" + def _read_bmp_frame(self, pipe): + """Read BMP frame from pipe - BMP header contains dimensions.""" try: - # Calculate frame size using actual detected dimensions - frame_size = self.actual_width * self.actual_height * 3 + # Read BMP header (14 bytes file header + 40 bytes info header = 54 bytes minimum) + header_data = b'' + bytes_to_read = 54 - # Read frame data in chunks until we have the complete frame - frame_data = b'' - bytes_remaining = frame_size + while len(header_data) < bytes_to_read: + chunk = pipe.read(bytes_to_read - len(header_data)) + if not chunk: + return None # Silent end of stream + header_data += chunk - while bytes_remaining > 0: - chunk = pipe.read(bytes_remaining) - if not chunk: # EOF - if len(frame_data) == 0: - logger.debug(f"Camera {self.camera_id}: No more data (stream ended)") - else: - logger.warning(f"Camera {self.camera_id}: Stream ended mid-frame: {len(frame_data)}/{frame_size} bytes") - return None + # Parse BMP header + if header_data[:2] != b'BM': + return None # Invalid format, skip frame silently - frame_data += chunk - bytes_remaining -= len(chunk) + # Extract file size from header (bytes 2-5) + import struct + file_size = struct.unpack('= 30: - logger.info(f"Camera {self.camera_id}: {frame_count} frames processed via pipe") + if current_time - last_log_time >= 60: + log_success(self.camera_id, f"{frame_count} frames captured ({frame.shape[1]}x{frame.shape[0]})") last_log_time = current_time - except Exception as e: - logger.error(f"Camera {self.camera_id}: Error reading from pipe: {e}") + except Exception: # Process might have died, let it restart on next iteration if self.process: self.process.terminate() self.process = None time.sleep(1.0) - except Exception as e: - logger.error(f"Camera {self.camera_id}: Error in pipe frame reading: {e}") + except Exception: time.sleep(1.0) # Cleanup if self.process: self.process.terminate() - logger.info(f"FFmpeg pipe reader ended for camera {self.camera_id}") logger = logging.getLogger(__name__) diff --git a/core/tracking/bot_sort_tracker.py b/core/tracking/bot_sort_tracker.py new file mode 100644 index 0000000..f487a6a --- /dev/null +++ b/core/tracking/bot_sort_tracker.py @@ -0,0 +1,408 @@ +""" +BoT-SORT Multi-Object Tracker with Camera Isolation +Based on BoT-SORT: Robust Associations Multi-Pedestrian Tracking +""" + +import logging +import time +import numpy as np +from typing import Dict, List, Optional, Tuple, Any +from dataclasses import dataclass +from scipy.optimize import linear_sum_assignment +from filterpy.kalman import KalmanFilter +import cv2 + +logger = logging.getLogger(__name__) + + +@dataclass +class TrackState: + """Track state enumeration""" + TENTATIVE = "tentative" # New track, not confirmed yet + CONFIRMED = "confirmed" # Confirmed track + DELETED = "deleted" # Track to be deleted + + +class Track: + """ + Individual track representation with Kalman filter for motion prediction + """ + + def __init__(self, detection, track_id: int, camera_id: str): + """ + Initialize a new track + + Args: + detection: Initial detection (bbox, confidence, class) + track_id: Unique track identifier within camera + camera_id: Camera identifier + """ + self.track_id = track_id + self.camera_id = camera_id + self.state = TrackState.TENTATIVE + + # Time tracking + self.start_time = time.time() + self.last_update_time = time.time() + + # Appearance and motion + self.bbox = detection.bbox # [x1, y1, x2, y2] + self.confidence = detection.confidence + self.class_name = detection.class_name + + # Track management + self.hit_streak = 1 + self.time_since_update = 0 + self.age = 1 + + # Kalman filter for motion prediction + self.kf = self._create_kalman_filter() + self._update_kalman_filter(detection.bbox) + + # Track history + self.history = [detection.bbox] + self.max_history = 10 + + def _create_kalman_filter(self) -> KalmanFilter: + """Create Kalman filter for bbox tracking (x, y, w, h, vx, vy, vw, vh)""" + kf = KalmanFilter(dim_x=8, dim_z=4) + + # State transition matrix (constant velocity model) + kf.F = np.array([ + [1, 0, 0, 0, 1, 0, 0, 0], + [0, 1, 0, 0, 0, 1, 0, 0], + [0, 0, 1, 0, 0, 0, 1, 0], + [0, 0, 0, 1, 0, 0, 0, 1], + [0, 0, 0, 0, 1, 0, 0, 0], + [0, 0, 0, 0, 0, 1, 0, 0], + [0, 0, 0, 0, 0, 0, 1, 0], + [0, 0, 0, 0, 0, 0, 0, 1] + ]) + + # Measurement matrix (observe x, y, w, h) + kf.H = np.array([ + [1, 0, 0, 0, 0, 0, 0, 0], + [0, 1, 0, 0, 0, 0, 0, 0], + [0, 0, 1, 0, 0, 0, 0, 0], + [0, 0, 0, 1, 0, 0, 0, 0] + ]) + + # Process noise + kf.Q *= 0.01 + + # Measurement noise + kf.R *= 10 + + # Initial covariance + kf.P *= 100 + + return kf + + def _update_kalman_filter(self, bbox: List[float]): + """Update Kalman filter with new bbox""" + # Convert [x1, y1, x2, y2] to [cx, cy, w, h] + x1, y1, x2, y2 = bbox + cx = (x1 + x2) / 2 + cy = (y1 + y2) / 2 + w = x2 - x1 + h = y2 - y1 + + # Properly assign to column vector + self.kf.x[:4, 0] = [cx, cy, w, h] + + def predict(self) -> np.ndarray: + """Predict next position using Kalman filter""" + self.kf.predict() + + # Convert back to [x1, y1, x2, y2] format + cx, cy, w, h = self.kf.x[:4, 0] # Extract from column vector + x1 = cx - w/2 + y1 = cy - h/2 + x2 = cx + w/2 + y2 = cy + h/2 + + return np.array([x1, y1, x2, y2]) + + def update(self, detection): + """Update track with new detection""" + self.last_update_time = time.time() + self.time_since_update = 0 + self.hit_streak += 1 + self.age += 1 + + # Update track properties + self.bbox = detection.bbox + self.confidence = detection.confidence + + # Update Kalman filter + x1, y1, x2, y2 = detection.bbox + cx = (x1 + x2) / 2 + cy = (y1 + y2) / 2 + w = x2 - x1 + h = y2 - y1 + + self.kf.update([cx, cy, w, h]) + + # Update history + self.history.append(detection.bbox) + if len(self.history) > self.max_history: + self.history.pop(0) + + # Update state + if self.state == TrackState.TENTATIVE and self.hit_streak >= 3: + self.state = TrackState.CONFIRMED + + def mark_missed(self): + """Mark track as missed in this frame""" + self.time_since_update += 1 + self.age += 1 + + if self.time_since_update > 5: # Delete after 5 missed frames + self.state = TrackState.DELETED + + def is_confirmed(self) -> bool: + """Check if track is confirmed""" + return self.state == TrackState.CONFIRMED + + def is_deleted(self) -> bool: + """Check if track should be deleted""" + return self.state == TrackState.DELETED + + +class CameraTracker: + """ + BoT-SORT tracker for a single camera + """ + + def __init__(self, camera_id: str, max_disappeared: int = 10): + """ + Initialize camera tracker + + Args: + camera_id: Unique camera identifier + max_disappeared: Maximum frames a track can be missed before deletion + """ + self.camera_id = camera_id + self.max_disappeared = max_disappeared + + # Track management + self.tracks: Dict[int, Track] = {} + self.next_id = 1 + self.frame_count = 0 + + logger.info(f"Initialized BoT-SORT tracker for camera {camera_id}") + + def update(self, detections: List) -> List[Track]: + """ + Update tracker with new detections + + Args: + detections: List of Detection objects + + Returns: + List of active confirmed tracks + """ + self.frame_count += 1 + + # Predict all existing tracks + for track in self.tracks.values(): + track.predict() + + # Associate detections to tracks + matched_tracks, unmatched_detections, unmatched_tracks = self._associate(detections) + + # Update matched tracks + for track_id, detection in matched_tracks: + self.tracks[track_id].update(detection) + + # Mark unmatched tracks as missed + for track_id in unmatched_tracks: + self.tracks[track_id].mark_missed() + + # Create new tracks for unmatched detections + for detection in unmatched_detections: + track = Track(detection, self.next_id, self.camera_id) + self.tracks[self.next_id] = track + self.next_id += 1 + + # Remove deleted tracks + tracks_to_remove = [tid for tid, track in self.tracks.items() if track.is_deleted()] + for tid in tracks_to_remove: + del self.tracks[tid] + + # Return confirmed tracks + confirmed_tracks = [track for track in self.tracks.values() if track.is_confirmed()] + + return confirmed_tracks + + def _associate(self, detections: List) -> Tuple[List[Tuple[int, Any]], List[Any], List[int]]: + """ + Associate detections to existing tracks using IoU distance + + Returns: + (matched_tracks, unmatched_detections, unmatched_tracks) + """ + if not detections or not self.tracks: + return [], detections, list(self.tracks.keys()) + + # Calculate IoU distance matrix + track_ids = list(self.tracks.keys()) + cost_matrix = np.zeros((len(track_ids), len(detections))) + + for i, track_id in enumerate(track_ids): + track = self.tracks[track_id] + predicted_bbox = track.predict() + + for j, detection in enumerate(detections): + iou = self._calculate_iou(predicted_bbox, detection.bbox) + cost_matrix[i, j] = 1 - iou # Convert IoU to distance + + # Solve assignment problem + row_indices, col_indices = linear_sum_assignment(cost_matrix) + + # Filter matches by IoU threshold + iou_threshold = 0.3 + matched_tracks = [] + matched_detection_indices = set() + matched_track_indices = set() + + for row, col in zip(row_indices, col_indices): + if cost_matrix[row, col] <= (1 - iou_threshold): + track_id = track_ids[row] + detection = detections[col] + matched_tracks.append((track_id, detection)) + matched_detection_indices.add(col) + matched_track_indices.add(row) + + # Find unmatched detections and tracks + unmatched_detections = [detections[i] for i in range(len(detections)) + if i not in matched_detection_indices] + unmatched_tracks = [track_ids[i] for i in range(len(track_ids)) + if i not in matched_track_indices] + + return matched_tracks, unmatched_detections, unmatched_tracks + + def _calculate_iou(self, bbox1: np.ndarray, bbox2: List[float]) -> float: + """Calculate IoU between two bounding boxes""" + x1_1, y1_1, x2_1, y2_1 = bbox1 + x1_2, y1_2, x2_2, y2_2 = bbox2 + + # Calculate intersection area + x1_i = max(x1_1, x1_2) + y1_i = max(y1_1, y1_2) + x2_i = min(x2_1, x2_2) + y2_i = min(y2_1, y2_2) + + if x2_i <= x1_i or y2_i <= y1_i: + return 0.0 + + intersection = (x2_i - x1_i) * (y2_i - y1_i) + + # Calculate union area + area1 = (x2_1 - x1_1) * (y2_1 - y1_1) + area2 = (x2_2 - x1_2) * (y2_2 - y1_2) + union = area1 + area2 - intersection + + return intersection / union if union > 0 else 0.0 + + +class MultiCameraBoTSORT: + """ + Multi-camera BoT-SORT tracker with complete camera isolation + """ + + def __init__(self, trigger_classes: List[str], min_confidence: float = 0.6): + """ + Initialize multi-camera tracker + + Args: + trigger_classes: List of class names to track + min_confidence: Minimum detection confidence threshold + """ + self.trigger_classes = trigger_classes + self.min_confidence = min_confidence + + # Camera-specific trackers + self.camera_trackers: Dict[str, CameraTracker] = {} + + logger.info(f"Initialized MultiCameraBoTSORT with classes={trigger_classes}, " + f"min_confidence={min_confidence}") + + def get_or_create_tracker(self, camera_id: str) -> CameraTracker: + """Get or create tracker for specific camera""" + if camera_id not in self.camera_trackers: + self.camera_trackers[camera_id] = CameraTracker(camera_id) + logger.info(f"Created new tracker for camera {camera_id}") + + return self.camera_trackers[camera_id] + + def update(self, camera_id: str, inference_result) -> List[Dict]: + """ + Update tracker for specific camera with detections + + Args: + camera_id: Camera identifier + inference_result: InferenceResult with detections + + Returns: + List of track information dictionaries + """ + # Filter detections by confidence and trigger classes + filtered_detections = [] + + if hasattr(inference_result, 'detections') and inference_result.detections: + for detection in inference_result.detections: + if (detection.confidence >= self.min_confidence and + detection.class_name in self.trigger_classes): + filtered_detections.append(detection) + + # Get camera tracker and update + tracker = self.get_or_create_tracker(camera_id) + confirmed_tracks = tracker.update(filtered_detections) + + # Convert tracks to output format + track_results = [] + for track in confirmed_tracks: + track_results.append({ + 'track_id': track.track_id, + 'camera_id': track.camera_id, + 'bbox': track.bbox, + 'confidence': track.confidence, + 'class_name': track.class_name, + 'hit_streak': track.hit_streak, + 'age': track.age + }) + + return track_results + + def get_statistics(self) -> Dict[str, Any]: + """Get tracking statistics across all cameras""" + stats = {} + total_tracks = 0 + + for camera_id, tracker in self.camera_trackers.items(): + camera_stats = { + 'active_tracks': len([t for t in tracker.tracks.values() if t.is_confirmed()]), + 'total_tracks': len(tracker.tracks), + 'frame_count': tracker.frame_count + } + stats[camera_id] = camera_stats + total_tracks += camera_stats['active_tracks'] + + stats['summary'] = { + 'total_cameras': len(self.camera_trackers), + 'total_active_tracks': total_tracks + } + + return stats + + def reset_camera(self, camera_id: str): + """Reset tracking for specific camera""" + if camera_id in self.camera_trackers: + del self.camera_trackers[camera_id] + logger.info(f"Reset tracking for camera {camera_id}") + + def reset_all(self): + """Reset all camera trackers""" + self.camera_trackers.clear() + logger.info("Reset all camera trackers") \ No newline at end of file diff --git a/core/tracking/integration.py b/core/tracking/integration.py index a10acf8..3f1ebe0 100644 --- a/core/tracking/integration.py +++ b/core/tracking/integration.py @@ -63,7 +63,7 @@ class TrackingPipelineIntegration: self.pending_processing_data: Dict[str, Dict] = {} # display_id -> processing data (waiting for session ID) # Additional validators for enhanced flow control - self.permanently_processed: Dict[int, float] = {} # track_id -> process_time (never process again) + self.permanently_processed: Dict[str, float] = {} # "camera_id:track_id" -> process_time (never process again) self.progression_stages: Dict[str, str] = {} # session_id -> current_stage self.last_detection_time: Dict[str, float] = {} # display_id -> last_detection_timestamp self.abandonment_timeout = 3.0 # seconds to wait before declaring car abandoned @@ -183,7 +183,7 @@ class TrackingPipelineIntegration: # Run tracking model if self.tracking_model: - # Run inference with tracking + # Run detection-only (tracking handled by our own tracker) tracking_results = self.tracking_model.track( frame, confidence_threshold=self.tracker.min_confidence, @@ -486,7 +486,10 @@ class TrackingPipelineIntegration: self.session_vehicles[session_id] = track_id # Mark vehicle as permanently processed (won't process again even after session clear) - self.permanently_processed[track_id] = time.time() + # Use composite key to distinguish same track IDs across different cameras + camera_id = display_id # Using display_id as camera_id for isolation + permanent_key = f"{camera_id}:{track_id}" + self.permanently_processed[permanent_key] = time.time() # Remove from pending del self.pending_vehicles[display_id] @@ -667,6 +670,7 @@ class TrackingPipelineIntegration: self.executor.shutdown(wait=False) self.reset_tracking() + # Cleanup detection pipeline if self.detection_pipeline: self.detection_pipeline.cleanup() diff --git a/core/tracking/tracker.py b/core/tracking/tracker.py index 6fa6ed9..63d0299 100644 --- a/core/tracking/tracker.py +++ b/core/tracking/tracker.py @@ -1,6 +1,6 @@ """ -Vehicle Tracking Module - Continuous tracking with front_rear_detection model -Implements vehicle identification, persistence, and motion analysis. +Vehicle Tracking Module - BoT-SORT based tracking with camera isolation +Implements vehicle identification, persistence, and motion analysis using external tracker. """ import logging import time @@ -10,6 +10,8 @@ from dataclasses import dataclass, field import numpy as np from threading import Lock +from .bot_sort_tracker import MultiCameraBoTSORT + logger = logging.getLogger(__name__) @@ -17,6 +19,7 @@ logger = logging.getLogger(__name__) class TrackedVehicle: """Represents a tracked vehicle with all its state information.""" track_id: int + camera_id: str first_seen: float last_seen: float session_id: Optional[str] = None @@ -30,6 +33,8 @@ class TrackedVehicle: processed_pipeline: bool = False last_position_history: List[Tuple[float, float]] = field(default_factory=list) avg_confidence: float = 0.0 + hit_streak: int = 0 + age: int = 0 def update_position(self, bbox: Tuple[int, int, int, int], confidence: float): """Update vehicle position and confidence.""" @@ -73,7 +78,7 @@ class TrackedVehicle: class VehicleTracker: """ - Main vehicle tracking implementation using YOLO tracking capabilities. + Main vehicle tracking implementation using BoT-SORT with camera isolation. Manages continuous tracking, vehicle identification, and state persistence. """ @@ -88,18 +93,19 @@ class VehicleTracker: self.trigger_classes = self.config.get('trigger_classes', self.config.get('triggerClasses', ['frontal'])) self.min_confidence = self.config.get('minConfidence', 0.6) - # Tracking state - self.tracked_vehicles: Dict[int, TrackedVehicle] = {} - self.next_track_id = 1 + # BoT-SORT multi-camera tracker + self.bot_sort = MultiCameraBoTSORT(self.trigger_classes, self.min_confidence) + + # Tracking state - maintain compatibility with existing code + self.tracked_vehicles: Dict[str, Dict[int, TrackedVehicle]] = {} # camera_id -> {track_id: vehicle} self.lock = Lock() # Tracking parameters self.stability_threshold = 0.7 self.min_stable_frames = 5 - self.position_tolerance = 50 # pixels self.timeout_seconds = 2.0 - logger.info(f"VehicleTracker initialized with trigger_classes={self.trigger_classes}, " + logger.info(f"VehicleTracker initialized with BoT-SORT: trigger_classes={self.trigger_classes}, " f"min_confidence={self.min_confidence}") def process_detections(self, @@ -107,10 +113,10 @@ class VehicleTracker: display_id: str, frame: np.ndarray) -> List[TrackedVehicle]: """ - Process YOLO detection results and update tracking state. + Process detection results using BoT-SORT tracking. Args: - results: YOLO detection results with tracking + results: Detection results (InferenceResult) display_id: Display identifier for this stream frame: Current frame being processed @@ -118,108 +124,67 @@ class VehicleTracker: List of currently tracked vehicles """ current_time = time.time() - active_tracks = [] + + # Extract camera_id from display_id for tracking isolation + camera_id = display_id # Using display_id as camera_id for isolation with self.lock: - # Clean up expired tracks - expired_ids = [ - track_id for track_id, vehicle in self.tracked_vehicles.items() - if vehicle.is_expired(self.timeout_seconds) - ] - for track_id in expired_ids: - logger.debug(f"Removing expired track {track_id}") - del self.tracked_vehicles[track_id] + # Update BoT-SORT tracker + track_results = self.bot_sort.update(camera_id, results) - # Process new detections from InferenceResult - if hasattr(results, 'detections') and results.detections: - # Process detections from InferenceResult - for detection in results.detections: - # Skip if confidence is too low - if detection.confidence < self.min_confidence: - continue + # Ensure camera tracking dict exists + if camera_id not in self.tracked_vehicles: + self.tracked_vehicles[camera_id] = {} - # Check if class is in trigger classes - if detection.class_name not in self.trigger_classes: - continue + # Update tracked vehicles based on BoT-SORT results + current_tracks = {} + active_tracks = [] - # Use track_id if available, otherwise generate one - track_id = detection.track_id if detection.track_id is not None else self.next_track_id - if detection.track_id is None: - self.next_track_id += 1 + for track_result in track_results: + track_id = track_result['track_id'] - # Get bounding box from Detection object - x1, y1, x2, y2 = detection.bbox - bbox = (int(x1), int(y1), int(x2), int(y2)) + # Create or update TrackedVehicle + if track_id in self.tracked_vehicles[camera_id]: + # Update existing vehicle + vehicle = self.tracked_vehicles[camera_id][track_id] + vehicle.update_position(track_result['bbox'], track_result['confidence']) + vehicle.hit_streak = track_result['hit_streak'] + vehicle.age = track_result['age'] - # Update or create tracked vehicle - confidence = detection.confidence - if track_id in self.tracked_vehicles: - # Update existing track - vehicle = self.tracked_vehicles[track_id] - vehicle.update_position(bbox, confidence) - vehicle.display_id = display_id + # Update stability based on hit_streak + if vehicle.hit_streak >= self.min_stable_frames: + vehicle.is_stable = True + vehicle.stable_frames = vehicle.hit_streak - # Check stability - stability = vehicle.calculate_stability() - if stability > self.stability_threshold: - vehicle.stable_frames += 1 - if vehicle.stable_frames >= self.min_stable_frames: - vehicle.is_stable = True - else: - vehicle.stable_frames = max(0, vehicle.stable_frames - 1) - if vehicle.stable_frames < self.min_stable_frames: - vehicle.is_stable = False + logger.debug(f"Updated track {track_id}: conf={vehicle.confidence:.2f}, " + f"stable={vehicle.is_stable}, hit_streak={vehicle.hit_streak}") + else: + # Create new vehicle + x1, y1, x2, y2 = track_result['bbox'] + vehicle = TrackedVehicle( + track_id=track_id, + camera_id=camera_id, + first_seen=current_time, + last_seen=current_time, + display_id=display_id, + confidence=track_result['confidence'], + bbox=tuple(track_result['bbox']), + center=((x1 + x2) / 2, (y1 + y2) / 2), + total_frames=1, + hit_streak=track_result['hit_streak'], + age=track_result['age'] + ) + vehicle.last_position_history.append(vehicle.center) + logger.info(f"New vehicle tracked: ID={track_id}, camera={camera_id}, display={display_id}") - logger.debug(f"Updated track {track_id}: conf={confidence:.2f}, " - f"stable={vehicle.is_stable}, stability={stability:.2f}") - else: - # Create new track - vehicle = TrackedVehicle( - track_id=track_id, - first_seen=current_time, - last_seen=current_time, - display_id=display_id, - confidence=confidence, - bbox=bbox, - center=((x1 + x2) / 2, (y1 + y2) / 2), - total_frames=1 - ) - vehicle.last_position_history.append(vehicle.center) - self.tracked_vehicles[track_id] = vehicle - logger.info(f"New vehicle tracked: ID={track_id}, display={display_id}") + current_tracks[track_id] = vehicle + active_tracks.append(vehicle) - active_tracks.append(self.tracked_vehicles[track_id]) + # Update the camera's tracked vehicles + self.tracked_vehicles[camera_id] = current_tracks return active_tracks - def _find_closest_track(self, center: Tuple[float, float]) -> Optional[TrackedVehicle]: - """ - Find the closest existing track to a given position. - - Args: - center: Center position to match - - Returns: - Closest tracked vehicle if within tolerance, None otherwise - """ - min_distance = float('inf') - closest_track = None - - for vehicle in self.tracked_vehicles.values(): - if vehicle.is_expired(0.5): # Shorter timeout for matching - continue - - distance = np.sqrt( - (center[0] - vehicle.center[0]) ** 2 + - (center[1] - vehicle.center[1]) ** 2 - ) - - if distance < min_distance and distance < self.position_tolerance: - min_distance = distance - closest_track = vehicle - - return closest_track - def get_stable_vehicles(self, display_id: Optional[str] = None) -> List[TrackedVehicle]: """ Get all stable vehicles, optionally filtered by display. @@ -231,11 +196,15 @@ class VehicleTracker: List of stable tracked vehicles """ with self.lock: - stable = [ - v for v in self.tracked_vehicles.values() - if v.is_stable and not v.is_expired(self.timeout_seconds) - and (display_id is None or v.display_id == display_id) - ] + stable = [] + camera_id = display_id # Using display_id as camera_id + + if camera_id in self.tracked_vehicles: + for vehicle in self.tracked_vehicles[camera_id].values(): + if (vehicle.is_stable and not vehicle.is_expired(self.timeout_seconds) and + (display_id is None or vehicle.display_id == display_id)): + stable.append(vehicle) + return stable def get_vehicle_by_session(self, session_id: str) -> Optional[TrackedVehicle]: @@ -249,9 +218,11 @@ class VehicleTracker: Tracked vehicle if found, None otherwise """ with self.lock: - for vehicle in self.tracked_vehicles.values(): - if vehicle.session_id == session_id: - return vehicle + # Search across all cameras + for camera_vehicles in self.tracked_vehicles.values(): + for vehicle in camera_vehicles.values(): + if vehicle.session_id == session_id: + return vehicle return None def mark_processed(self, track_id: int, session_id: str): @@ -263,11 +234,14 @@ class VehicleTracker: session_id: Session ID assigned to this vehicle """ with self.lock: - if track_id in self.tracked_vehicles: - vehicle = self.tracked_vehicles[track_id] - vehicle.processed_pipeline = True - vehicle.session_id = session_id - logger.info(f"Marked vehicle {track_id} as processed with session {session_id}") + # Search across all cameras for the track_id + for camera_vehicles in self.tracked_vehicles.values(): + if track_id in camera_vehicles: + vehicle = camera_vehicles[track_id] + vehicle.processed_pipeline = True + vehicle.session_id = session_id + logger.info(f"Marked vehicle {track_id} as processed with session {session_id}") + return def clear_session(self, session_id: str): """ @@ -277,30 +251,43 @@ class VehicleTracker: session_id: Session ID to clear """ with self.lock: - for vehicle in self.tracked_vehicles.values(): - if vehicle.session_id == session_id: - logger.info(f"Clearing session {session_id} from vehicle {vehicle.track_id}") - vehicle.session_id = None - # Keep processed_pipeline=True to prevent re-processing + # Search across all cameras + for camera_vehicles in self.tracked_vehicles.values(): + for vehicle in camera_vehicles.values(): + if vehicle.session_id == session_id: + logger.info(f"Clearing session {session_id} from vehicle {vehicle.track_id}") + vehicle.session_id = None + # Keep processed_pipeline=True to prevent re-processing def reset_tracking(self): """Reset all tracking state.""" with self.lock: self.tracked_vehicles.clear() - self.next_track_id = 1 + self.bot_sort.reset_all() logger.info("Vehicle tracking state reset") def get_statistics(self) -> Dict: """Get tracking statistics.""" with self.lock: - total = len(self.tracked_vehicles) - stable = sum(1 for v in self.tracked_vehicles.values() if v.is_stable) - processed = sum(1 for v in self.tracked_vehicles.values() if v.processed_pipeline) + total = 0 + stable = 0 + processed = 0 + all_confidences = [] + + # Aggregate stats across all cameras + for camera_vehicles in self.tracked_vehicles.values(): + total += len(camera_vehicles) + for vehicle in camera_vehicles.values(): + if vehicle.is_stable: + stable += 1 + if vehicle.processed_pipeline: + processed += 1 + all_confidences.append(vehicle.avg_confidence) return { 'total_tracked': total, 'stable_vehicles': stable, 'processed_vehicles': processed, - 'avg_confidence': np.mean([v.avg_confidence for v in self.tracked_vehicles.values()]) - if self.tracked_vehicles else 0.0 + 'avg_confidence': np.mean(all_confidences) if all_confidences else 0.0, + 'bot_sort_stats': self.bot_sort.get_statistics() } \ No newline at end of file diff --git a/core/tracking/validator.py b/core/tracking/validator.py index d90d4ec..c20987f 100644 --- a/core/tracking/validator.py +++ b/core/tracking/validator.py @@ -354,25 +354,28 @@ class StableCarValidator: def should_skip_same_car(self, vehicle: TrackedVehicle, session_cleared: bool = False, - permanently_processed: Dict[int, float] = None) -> bool: + permanently_processed: Dict[str, float] = None) -> bool: """ Determine if we should skip processing for the same car after session clear. Args: vehicle: The tracked vehicle session_cleared: Whether the session was recently cleared - permanently_processed: Dict of permanently processed vehicles + permanently_processed: Dict of permanently processed vehicles (camera_id:track_id -> time) Returns: True if we should skip this vehicle """ # Check if this vehicle was permanently processed (never process again) - if permanently_processed and vehicle.track_id in permanently_processed: - process_time = permanently_processed[vehicle.track_id] - time_since = time.time() - process_time - logger.debug(f"Skipping permanently processed vehicle {vehicle.track_id} " - f"(processed {time_since:.1f}s ago)") - return True + if permanently_processed: + # Create composite key using camera_id and track_id + permanent_key = f"{vehicle.camera_id}:{vehicle.track_id}" + if permanent_key in permanently_processed: + process_time = permanently_processed[permanent_key] + time_since = time.time() - process_time + logger.debug(f"Skipping permanently processed vehicle {vehicle.track_id} on camera {vehicle.camera_id} " + f"(processed {time_since:.1f}s ago)") + return True # If vehicle has a session_id but it was cleared, skip for a period if vehicle.session_id is None and vehicle.processed_pipeline and session_cleared: From 61ac39b4f353e9bdb4411ea430b50743f59f37d3 Mon Sep 17 00:00:00 2001 From: ziesorx Date: Fri, 26 Sep 2025 14:50:45 +0700 Subject: [PATCH 34/55] fix: validator --- core/communication/websocket.py | 42 ++++++------ core/streaming/manager.py | 43 ++++++++++-- core/tracking/validator.py | 116 +++++++++++++------------------- 3 files changed, 106 insertions(+), 95 deletions(-) diff --git a/core/communication/websocket.py b/core/communication/websocket.py index 077c6dc..7394280 100644 --- a/core/communication/websocket.py +++ b/core/communication/websocket.py @@ -297,31 +297,31 @@ class WebSocketHandler: async def _reconcile_subscriptions_with_tracking(self, target_subscriptions) -> dict: """Reconcile subscriptions with tracking integration.""" try: - # First, we need to create tracking integrations for each unique model + # Create separate tracking integrations for each subscription (camera isolation) tracking_integrations = {} for subscription_payload in target_subscriptions: + subscription_id = subscription_payload['subscriptionIdentifier'] model_id = subscription_payload['modelId'] - # Create tracking integration if not already created - if model_id not in tracking_integrations: - # Get pipeline configuration for this model - pipeline_parser = model_manager.get_pipeline_config(model_id) - if pipeline_parser: - # Create tracking integration with message sender - tracking_integration = TrackingPipelineIntegration( - pipeline_parser, model_manager, model_id, self._send_message - ) + # Create separate tracking integration per subscription for camera isolation + # Get pipeline configuration for this model + pipeline_parser = model_manager.get_pipeline_config(model_id) + if pipeline_parser: + # Create tracking integration with message sender (separate instance per camera) + tracking_integration = TrackingPipelineIntegration( + pipeline_parser, model_manager, model_id, self._send_message + ) - # Initialize tracking model - success = await tracking_integration.initialize_tracking_model() - if success: - tracking_integrations[model_id] = tracking_integration - logger.info(f"[Tracking] Created tracking integration for model {model_id}") - else: - logger.warning(f"[Tracking] Failed to initialize tracking for model {model_id}") + # Initialize tracking model + success = await tracking_integration.initialize_tracking_model() + if success: + tracking_integrations[subscription_id] = tracking_integration + logger.info(f"[Tracking] Created isolated tracking integration for subscription {subscription_id} (model {model_id})") else: - logger.warning(f"[Tracking] No pipeline config found for model {model_id}") + logger.warning(f"[Tracking] Failed to initialize tracking for subscription {subscription_id} (model {model_id})") + else: + logger.warning(f"[Tracking] No pipeline config found for model {model_id} in subscription {subscription_id}") # Now reconcile with StreamManager, adding tracking integrations current_subscription_ids = set() @@ -379,8 +379,8 @@ class WebSocketHandler: logger.info(f"[SUBSCRIPTION_MAPPING] subscription_id='{subscription_id}' → camera_id='{camera_id}'") - # Get tracking integration for this model - tracking_integration = tracking_integrations.get(model_id) + # Get tracking integration for this subscription (camera-isolated) + tracking_integration = tracking_integrations.get(subscription_id) # Extract crop coordinates if present crop_coords = None @@ -412,7 +412,7 @@ class WebSocketHandler: ) if success and tracking_integration: - logger.info(f"[Tracking] Subscription {subscription_id} configured with tracking for model {model_id}") + logger.info(f"[Tracking] Subscription {subscription_id} configured with isolated tracking for model {model_id}") return success diff --git a/core/streaming/manager.py b/core/streaming/manager.py index f6cfbda..0c026e7 100644 --- a/core/streaming/manager.py +++ b/core/streaming/manager.py @@ -389,20 +389,51 @@ class StreamManager: logger.debug(f"Set session {session_id} for display {display_id}") def clear_session_id(self, session_id: str): - """Clear session ID from tracking integrations.""" + """Clear session ID from the specific tracking integration handling this session.""" with self._lock: + # Find the subscription that's handling this session + session_subscription = None for subscription_info in self._subscriptions.values(): if subscription_info.tracking_integration: - subscription_info.tracking_integration.clear_session_id(session_id) - logger.debug(f"Cleared session {session_id}") + # Check if this integration is handling the given session_id + integration = subscription_info.tracking_integration + if session_id in integration.session_vehicles: + session_subscription = subscription_info + break + + if session_subscription and session_subscription.tracking_integration: + session_subscription.tracking_integration.clear_session_id(session_id) + logger.debug(f"Cleared session {session_id} from subscription {session_subscription.subscription_id}") + else: + logger.warning(f"No tracking integration found for session {session_id}, broadcasting to all subscriptions") + # Fallback: broadcast to all (original behavior) + for subscription_info in self._subscriptions.values(): + if subscription_info.tracking_integration: + subscription_info.tracking_integration.clear_session_id(session_id) def set_progression_stage(self, session_id: str, stage: str): - """Set progression stage for tracking integrations.""" + """Set progression stage for the specific tracking integration handling this session.""" with self._lock: + # Find the subscription that's handling this session + session_subscription = None for subscription_info in self._subscriptions.values(): if subscription_info.tracking_integration: - subscription_info.tracking_integration.set_progression_stage(session_id, stage) - logger.debug(f"Set progression stage for session {session_id}: {stage}") + # Check if this integration is handling the given session_id + # We need to check the integration's active sessions + integration = subscription_info.tracking_integration + if session_id in integration.session_vehicles: + session_subscription = subscription_info + break + + if session_subscription and session_subscription.tracking_integration: + session_subscription.tracking_integration.set_progression_stage(session_id, stage) + logger.debug(f"Set progression stage for session {session_id}: {stage} on subscription {session_subscription.subscription_id}") + else: + logger.warning(f"No tracking integration found for session {session_id}, broadcasting to all subscriptions") + # Fallback: broadcast to all (original behavior) + for subscription_info in self._subscriptions.values(): + if subscription_info.tracking_integration: + subscription_info.tracking_integration.set_progression_stage(session_id, stage) def get_tracking_stats(self) -> Dict[str, Any]: """Get tracking statistics from all subscriptions.""" diff --git a/core/tracking/validator.py b/core/tracking/validator.py index c20987f..d86a3f6 100644 --- a/core/tracking/validator.py +++ b/core/tracking/validator.py @@ -36,8 +36,14 @@ class ValidationResult: class StableCarValidator: """ - Validates whether a tracked vehicle is stable (fueling) or just passing by. - Uses multiple criteria including position stability, duration, and movement patterns. + Validates whether a tracked vehicle should be processed through the pipeline. + + Updated for BoT-SORT integration: Trusts the sophisticated BoT-SORT tracking algorithm + for stability determination and focuses on business logic validation: + - Duration requirements for processing + - Confidence thresholds + - Session management and cooldowns + - Camera isolation with composite keys """ def __init__(self, config: Optional[Dict] = None): @@ -169,7 +175,10 @@ class StableCarValidator: def _determine_vehicle_state(self, vehicle: TrackedVehicle) -> VehicleState: """ - Determine the current state of the vehicle based on movement patterns. + Determine the current state of the vehicle based on BoT-SORT tracking results. + + BoT-SORT provides sophisticated tracking, so we trust its stability determination + and focus on business logic validation. Args: vehicle: The tracked vehicle @@ -177,53 +186,44 @@ class StableCarValidator: Returns: Current vehicle state """ - # Not enough data - if len(vehicle.last_position_history) < 3: - return VehicleState.UNKNOWN - - # Calculate velocity - velocity = self._calculate_velocity(vehicle) - - # Get position zones - x_position = vehicle.center[0] / self.frame_width - y_position = vehicle.center[1] / self.frame_height - - # Check if vehicle is stable - stability = vehicle.calculate_stability() - if stability > 0.7 and velocity < self.velocity_threshold: - # Check if it's been stable long enough + # Trust BoT-SORT's stability determination + if vehicle.is_stable: + # Check if it's been stable long enough for processing duration = time.time() - vehicle.first_seen - if duration > self.min_stable_duration and vehicle.stable_frames >= self.min_stable_frames: + if duration >= self.min_stable_duration: return VehicleState.STABLE else: return VehicleState.ENTERING - # Check if vehicle is entering or leaving + # For non-stable vehicles, use simplified state determination + if len(vehicle.last_position_history) < 2: + return VehicleState.UNKNOWN + + # Calculate velocity for movement classification + velocity = self._calculate_velocity(vehicle) + + # Basic movement classification if velocity > self.velocity_threshold: - # Determine direction based on position history - positions = np.array(vehicle.last_position_history) - if len(positions) >= 2: - direction = positions[-1] - positions[0] + # Vehicle is moving - classify as passing by or entering/leaving + x_position = vehicle.center[0] / self.frame_width - # Entering: moving towards center - if x_position < self.entering_zone_ratio or x_position > (1 - self.entering_zone_ratio): - if abs(direction[0]) > abs(direction[1]): # Horizontal movement - if (x_position < 0.5 and direction[0] > 0) or (x_position > 0.5 and direction[0] < 0): - return VehicleState.ENTERING + # Simple heuristic: vehicles near edges are entering/leaving, center vehicles are passing + if x_position < 0.2 or x_position > 0.8: + return VehicleState.ENTERING + else: + return VehicleState.PASSING_BY - # Leaving: moving away from center - if 0.3 < x_position < 0.7: # In center zone - if abs(direction[0]) > abs(direction[1]): # Horizontal movement - if abs(direction[0]) > 10: # Significant movement - return VehicleState.LEAVING - - return VehicleState.PASSING_BY - - return VehicleState.UNKNOWN + # Low velocity but not marked stable by tracker - likely entering + return VehicleState.ENTERING def _validate_stable_vehicle(self, vehicle: TrackedVehicle) -> ValidationResult: """ - Perform detailed validation of a stable vehicle. + Perform business logic validation of a stable vehicle. + + Since BoT-SORT already determined the vehicle is stable, we focus on: + - Duration requirements for processing + - Confidence thresholds + - Business logic constraints Args: vehicle: The stable vehicle to validate @@ -231,7 +231,7 @@ class StableCarValidator: Returns: Detailed validation result """ - # Check duration + # Check duration (business requirement) duration = time.time() - vehicle.first_seen if duration < self.min_stable_duration: return ValidationResult( @@ -243,18 +243,7 @@ class StableCarValidator: track_id=vehicle.track_id ) - # Check frame count - if vehicle.stable_frames < self.min_stable_frames: - return ValidationResult( - is_valid=False, - state=VehicleState.STABLE, - confidence=0.6, - reason=f"Not enough stable frames ({vehicle.stable_frames} < {self.min_stable_frames})", - should_process=False, - track_id=vehicle.track_id - ) - - # Check confidence + # Check confidence (business requirement) if vehicle.avg_confidence < self.min_confidence: return ValidationResult( is_valid=False, @@ -265,28 +254,19 @@ class StableCarValidator: track_id=vehicle.track_id ) - # Check position variance - variance = self._calculate_position_variance(vehicle) - if variance > self.position_variance_threshold: - return ValidationResult( - is_valid=False, - state=VehicleState.STABLE, - confidence=0.7, - reason=f"Position variance too high ({variance:.1f} > {self.position_variance_threshold})", - should_process=False, - track_id=vehicle.track_id - ) + # Trust BoT-SORT's stability determination - skip position variance check + # BoT-SORT's sophisticated tracking already ensures consistent positioning - # Check state history consistency + # Simplified state history check - just ensure recent stability if vehicle.track_id in self.validation_history: - history = self.validation_history[vehicle.track_id][-5:] # Last 5 states + history = self.validation_history[vehicle.track_id][-3:] # Last 3 states stable_count = sum(1 for s in history if s == VehicleState.STABLE) - if stable_count < 3: + if len(history) >= 2 and stable_count == 0: # Only fail if clear instability return ValidationResult( is_valid=False, state=VehicleState.STABLE, confidence=0.7, - reason="Inconsistent state history", + reason="Recent state history shows instability", should_process=False, track_id=vehicle.track_id ) @@ -298,7 +278,7 @@ class StableCarValidator: is_valid=True, state=VehicleState.STABLE, confidence=vehicle.avg_confidence, - reason="Vehicle is stable and ready for processing", + reason="Vehicle is stable and ready for processing (BoT-SORT validated)", should_process=True, track_id=vehicle.track_id ) From 9f8372d8445024813acc5b185241f2d2a440ba41 Mon Sep 17 00:00:00 2001 From: ziesorx Date: Fri, 26 Sep 2025 15:00:24 +0700 Subject: [PATCH 35/55] fix: change save image logic --- core/communication/websocket.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/communication/websocket.py b/core/communication/websocket.py index 7394280..4e40d2a 100644 --- a/core/communication/websocket.py +++ b/core/communication/websocket.py @@ -549,10 +549,6 @@ class WebSocketHandler: # Update tracking integrations with session ID shared_stream_manager.set_session_id(display_identifier, session_id) - # Save snapshot image after getting sessionId - if session_id: - await self._save_snapshot(display_identifier, session_id) - async def _handle_set_progression_stage(self, message: SetProgressionStageMessage) -> None: """Handle setProgressionStage message.""" display_identifier = message.payload.displayIdentifier @@ -568,6 +564,10 @@ class WebSocketHandler: if session_id: shared_stream_manager.set_progression_stage(session_id, stage) + # Save snapshot image when progression stage is car_fueling + if stage == 'car_fueling' and session_id: + await self._save_snapshot(display_identifier, session_id) + # If stage indicates session is cleared/finished, clear from tracking if stage in ['finished', 'cleared', 'idle']: # Get session ID for this display and clear it From cd1359f5d227d29d3b576649b3d31c3c3b5307b8 Mon Sep 17 00:00:00 2001 From: ziesorx Date: Fri, 26 Sep 2025 15:06:12 +0700 Subject: [PATCH 36/55] fix: enable hardward acceleration --- core/streaming/readers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/streaming/readers.py b/core/streaming/readers.py index d5635ba..6a1dab8 100644 --- a/core/streaming/readers.py +++ b/core/streaming/readers.py @@ -106,8 +106,8 @@ class FFmpegRTSPReader: cmd = [ 'ffmpeg', # DO NOT REMOVE - # '-hwaccel', 'cuda', - # '-hwaccel_device', '0', + '-hwaccel', 'cuda', + '-hwaccel_device', '0', '-rtsp_transport', 'tcp', '-i', self.rtsp_url, '-f', 'image2pipe', # Output images to pipe From 2808316e94f09db23ef3a922b95aae97a9aec847 Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Fri, 26 Sep 2025 19:42:41 +0700 Subject: [PATCH 37/55] fix: remove unused RTSPReader import and related code --- core/streaming/__init__.py | 3 +- core/streaming/manager.py | 2 +- core/streaming/readers.py | 444 +++++++++---------------------------- 3 files changed, 112 insertions(+), 337 deletions(-) diff --git a/core/streaming/__init__.py b/core/streaming/__init__.py index d878aac..93005ab 100644 --- a/core/streaming/__init__.py +++ b/core/streaming/__init__.py @@ -2,13 +2,12 @@ Streaming system for RTSP and HTTP camera feeds. Provides modular frame readers, buffers, and stream management. """ -from .readers import RTSPReader, HTTPSnapshotReader, FFmpegRTSPReader +from .readers import HTTPSnapshotReader, FFmpegRTSPReader from .buffers import FrameBuffer, CacheBuffer, shared_frame_buffer, shared_cache_buffer from .manager import StreamManager, StreamConfig, SubscriptionInfo, shared_stream_manager, initialize_stream_manager __all__ = [ # Readers - 'RTSPReader', 'HTTPSnapshotReader', 'FFmpegRTSPReader', diff --git a/core/streaming/manager.py b/core/streaming/manager.py index 0c026e7..5b4637c 100644 --- a/core/streaming/manager.py +++ b/core/streaming/manager.py @@ -9,7 +9,7 @@ from typing import Dict, Set, Optional, List, Any from dataclasses import dataclass from collections import defaultdict -from .readers import RTSPReader, HTTPSnapshotReader, FFmpegRTSPReader +from .readers import HTTPSnapshotReader, FFmpegRTSPReader from .buffers import shared_cache_buffer from ..tracking.integration import TrackingPipelineIntegration diff --git a/core/streaming/readers.py b/core/streaming/readers.py index 6a1dab8..5684997 100644 --- a/core/streaming/readers.py +++ b/core/streaming/readers.py @@ -8,16 +8,10 @@ import time import threading import requests import numpy as np -import os import subprocess -# import fcntl # No longer needed with atomic file operations from typing import Optional, Callable -# Removed watchdog imports - no longer using file watching -# Suppress FFMPEG/H.264 error messages if needed -# Set this environment variable to reduce noise from decoder errors -os.environ["OPENCV_LOG_LEVEL"] = "ERROR" -os.environ["OPENCV_FFMPEG_LOGLEVEL"] = "-8" # Suppress FFMPEG warnings + logger = logging.getLogger(__name__) @@ -65,12 +59,20 @@ class FFmpegRTSPReader: self.process = None self.stop_event = threading.Event() self.thread = None + self.stderr_thread = None self.frame_callback: Optional[Callable] = None # Expected stream specs (for reference, actual dimensions read from PPM header) self.width = 1280 self.height = 720 + # Watchdog timers for stream reliability + self.process_start_time = None + self.last_frame_time = None + self.is_restart = False # Track if this is a restart (shorter timeout) + self.first_start_timeout = 30.0 # 30s timeout on first start + self.restart_timeout = 15.0 # 15s timeout after restart + def set_frame_callback(self, callback: Callable[[str, np.ndarray], None]): """Set callback function to handle captured frames.""" self.frame_callback = callback @@ -97,6 +99,8 @@ class FFmpegRTSPReader: self.process.kill() if self.thread: self.thread.join(timeout=5.0) + if self.stderr_thread: + self.stderr_thread.join(timeout=2.0) log_info(self.camera_id, "Stream stopped") # Removed _probe_stream_info - BMP headers contain dimensions @@ -122,9 +126,30 @@ class FFmpegRTSPReader: self.process = subprocess.Popen( cmd, stdout=subprocess.PIPE, # Capture stdout for frame data - stderr=subprocess.DEVNULL, + stderr=subprocess.PIPE, # Capture stderr for error logging bufsize=0 # Unbuffered for real-time processing ) + + # Start stderr reading thread + if self.stderr_thread and self.stderr_thread.is_alive(): + # Stop previous stderr thread + try: + self.stderr_thread.join(timeout=1.0) + except: + pass + + self.stderr_thread = threading.Thread(target=self._read_stderr, daemon=True) + self.stderr_thread.start() + + # Set process start time for watchdog + self.process_start_time = time.time() + self.last_frame_time = None # Reset frame time + + # After successful restart, next timeout will be back to 30s + if self.is_restart: + log_info(self.camera_id, f"FFmpeg restarted successfully, next timeout: {self.first_start_timeout}s") + self.is_restart = False + return True except Exception as e: log_error(self.camera_id, f"FFmpeg startup failed: {e}") @@ -180,6 +205,74 @@ class FFmpegRTSPReader: except Exception: return None # Error reading frame silently + def _read_stderr(self): + """Read and log FFmpeg stderr output in background thread.""" + if not self.process or not self.process.stderr: + return + + try: + while self.process and self.process.poll() is None: + try: + line = self.process.stderr.readline() + if line: + error_msg = line.decode('utf-8', errors='ignore').strip() + if error_msg and not self.stop_event.is_set(): + # Filter out common noise but log actual errors + if any(keyword in error_msg.lower() for keyword in ['error', 'failed', 'cannot', 'invalid']): + log_error(self.camera_id, f"FFmpeg: {error_msg}") + elif 'warning' in error_msg.lower(): + log_warning(self.camera_id, f"FFmpeg: {error_msg}") + except Exception: + break + except Exception: + pass + + def _check_watchdog_timeout(self) -> bool: + """Check if watchdog timeout has been exceeded.""" + if not self.process_start_time: + return False + + current_time = time.time() + time_since_start = current_time - self.process_start_time + + # Determine timeout based on whether this is a restart + timeout = self.restart_timeout if self.is_restart else self.first_start_timeout + + # If no frames received yet, check against process start time + if not self.last_frame_time: + if time_since_start > timeout: + log_warning(self.camera_id, f"Watchdog timeout: No frames for {time_since_start:.1f}s (limit: {timeout}s)") + return True + else: + # Check time since last frame + time_since_frame = current_time - self.last_frame_time + if time_since_frame > timeout: + log_warning(self.camera_id, f"Watchdog timeout: No frames for {time_since_frame:.1f}s (limit: {timeout}s)") + return True + + return False + + def _restart_ffmpeg_process(self): + """Restart FFmpeg process due to watchdog timeout.""" + log_warning(self.camera_id, "Watchdog triggered FFmpeg restart") + + # Terminate current process + if self.process: + try: + self.process.terminate() + self.process.wait(timeout=3) + except subprocess.TimeoutExpired: + self.process.kill() + except Exception: + pass + self.process = None + + # Mark as restart for shorter timeout + self.is_restart = True + + # Small delay before restart + time.sleep(1.0) + def _read_frames(self): """Read frames directly from FFmpeg stdout pipe.""" frame_count = 0 @@ -187,6 +280,12 @@ class FFmpegRTSPReader: while not self.stop_event.is_set(): try: + # Check watchdog timeout if process is running + if self.process and self.process.poll() is None: + if self._check_watchdog_timeout(): + self._restart_ffmpeg_process() + continue + # Start FFmpeg if not running if not self.process or self.process.poll() is not None: if self.process and self.process.poll() is not None: @@ -204,6 +303,9 @@ class FFmpegRTSPReader: if frame is None: continue + # Update watchdog - we got a frame + self.last_frame_time = time.time() + # Call frame callback if self.frame_callback: self.frame_callback(self.camera_id, frame) @@ -234,332 +336,6 @@ class FFmpegRTSPReader: logger = logging.getLogger(__name__) -class RTSPReader: - """RTSP stream frame reader optimized for 1280x720 @ 6fps streams.""" - - def __init__(self, camera_id: str, rtsp_url: str, max_retries: int = 3): - self.camera_id = camera_id - self.rtsp_url = rtsp_url - self.max_retries = max_retries - self.cap = None - self.stop_event = threading.Event() - self.thread = None - self.frame_callback: Optional[Callable] = None - - # Expected stream specifications - self.expected_width = 1280 - self.expected_height = 720 - self.expected_fps = 6 - - # Frame processing parameters - self.error_recovery_delay = 5.0 # Increased from 2.0 for stability - self.max_consecutive_errors = 30 # Increased from 10 to handle network jitter - self.stream_timeout = 30.0 - - def set_frame_callback(self, callback: Callable[[str, np.ndarray], None]): - """Set callback function to handle captured frames.""" - self.frame_callback = callback - - def start(self): - """Start the RTSP reader thread.""" - if self.thread and self.thread.is_alive(): - logger.warning(f"RTSP reader for {self.camera_id} already running") - return - - self.stop_event.clear() - self.thread = threading.Thread(target=self._read_frames, daemon=True) - self.thread.start() - logger.info(f"Started RTSP reader for camera {self.camera_id}") - - def stop(self): - """Stop the RTSP reader thread.""" - self.stop_event.set() - if self.thread: - self.thread.join(timeout=5.0) - if self.cap: - self.cap.release() - logger.info(f"Stopped RTSP reader for camera {self.camera_id}") - - def _read_frames(self): - """Main frame reading loop with H.264 error recovery.""" - consecutive_errors = 0 - frame_count = 0 - last_log_time = time.time() - last_successful_frame_time = time.time() - - while not self.stop_event.is_set(): - try: - # Initialize/reinitialize capture if needed - if not self.cap or not self.cap.isOpened(): - if not self._initialize_capture(): - time.sleep(self.error_recovery_delay) - continue - last_successful_frame_time = time.time() - - # Check for stream timeout - if time.time() - last_successful_frame_time > self.stream_timeout: - logger.warning(f"Camera {self.camera_id}: Stream timeout, reinitializing") - self._reinitialize_capture() - last_successful_frame_time = time.time() - continue - - # Read frame immediately without rate limiting for minimum latency - try: - ret, frame = self.cap.read() - if ret and frame is None: - # Grab succeeded but retrieve failed - decoder issue - logger.error(f"Camera {self.camera_id}: Frame grab OK but decode failed") - except Exception as read_error: - logger.error(f"Camera {self.camera_id}: cap.read() threw exception: {type(read_error).__name__}: {read_error}") - ret, frame = False, None - - if not ret or frame is None: - consecutive_errors += 1 - - # Enhanced logging to diagnose the issue - logger.error(f"Camera {self.camera_id}: cap.read() failed - ret={ret}, frame={frame is not None}") - - # Try to get more info from the capture - try: - if self.cap and self.cap.isOpened(): - backend = self.cap.getBackendName() - pos_frames = self.cap.get(cv2.CAP_PROP_POS_FRAMES) - logger.error(f"Camera {self.camera_id}: Capture open, backend: {backend}, pos_frames: {pos_frames}") - else: - logger.error(f"Camera {self.camera_id}: Capture is closed or None!") - except Exception as info_error: - logger.error(f"Camera {self.camera_id}: Error getting capture info: {type(info_error).__name__}: {info_error}") - - if consecutive_errors >= self.max_consecutive_errors: - logger.error(f"Camera {self.camera_id}: Too many consecutive errors ({consecutive_errors}), reinitializing") - self._reinitialize_capture() - consecutive_errors = 0 - time.sleep(self.error_recovery_delay) - else: - # Skip corrupted frame and continue with exponential backoff - if consecutive_errors <= 5: - logger.debug(f"Camera {self.camera_id}: Frame read failed (error {consecutive_errors})") - elif consecutive_errors % 10 == 0: # Log every 10th error after 5 - logger.warning(f"Camera {self.camera_id}: Continuing frame read failures (error {consecutive_errors})") - - # Exponential backoff with cap at 1 second - sleep_time = min(0.1 * (1.5 ** min(consecutive_errors, 10)), 1.0) - time.sleep(sleep_time) - continue - - # Accept any valid frame dimensions - don't force specific resolution - if frame.shape[1] <= 0 or frame.shape[0] <= 0: - consecutive_errors += 1 - continue - - # Check for corrupted frames (all black, all white, excessive noise) - if self._is_frame_corrupted(frame): - logger.debug(f"Camera {self.camera_id}: Corrupted frame detected, skipping") - consecutive_errors += 1 - continue - - # Frame is valid - consecutive_errors = 0 - frame_count += 1 - last_successful_frame_time = time.time() - - # Call frame callback - if self.frame_callback: - try: - self.frame_callback(self.camera_id, frame) - except Exception as e: - logger.error(f"Camera {self.camera_id}: Frame callback error: {e}") - - # Log progress every 30 seconds - current_time = time.time() - if current_time - last_log_time >= 30: - logger.info(f"Camera {self.camera_id}: {frame_count} frames processed") - last_log_time = current_time - - except Exception as e: - logger.error(f"Camera {self.camera_id}: Error in frame reading loop: {e}") - consecutive_errors += 1 - if consecutive_errors >= self.max_consecutive_errors: - self._reinitialize_capture() - consecutive_errors = 0 - time.sleep(self.error_recovery_delay) - - # Cleanup - if self.cap: - self.cap.release() - logger.info(f"RTSP reader thread ended for camera {self.camera_id}") - - def _initialize_capture(self) -> bool: - """Initialize video capture with FFmpeg hardware acceleration (CUVID/NVDEC) for 1280x720@6fps.""" - try: - # Release previous capture if exists - if self.cap: - self.cap.release() - time.sleep(0.5) - - logger.info(f"Initializing capture for camera {self.camera_id} with FFmpeg hardware acceleration") - hw_accel_success = False - - # Method 1: Try OpenCV CUDA VideoReader (if built with CUVID support) - if not hw_accel_success: - try: - # Check if OpenCV was built with CUDA codec support - build_info = cv2.getBuildInformation() - if 'cudacodec' in build_info or 'CUVID' in build_info: - logger.info(f"Attempting OpenCV CUDA VideoReader for camera {self.camera_id}") - - # Use OpenCV's CUDA backend - self.cap = cv2.VideoCapture(self.rtsp_url, cv2.CAP_FFMPEG, [ - cv2.CAP_PROP_HW_ACCELERATION, cv2.VIDEO_ACCELERATION_ANY - ]) - - if self.cap.isOpened(): - hw_accel_success = True - logger.info(f"Camera {self.camera_id}: Using OpenCV CUDA hardware acceleration") - else: - logger.debug(f"Camera {self.camera_id}: OpenCV not built with CUDA codec support") - except Exception as e: - logger.debug(f"Camera {self.camera_id}: OpenCV CUDA not available: {e}") - - # Method 2: Try FFmpeg with optimal hardware acceleration (CUVID/NVDEC) - if not hw_accel_success: - try: - from core.utils.ffmpeg_detector import get_optimal_rtsp_options - import os - - # Get optimal FFmpeg options based on detected capabilities - optimal_options = get_optimal_rtsp_options(self.rtsp_url) - os.environ['OPENCV_FFMPEG_CAPTURE_OPTIONS'] = optimal_options - - logger.info(f"Attempting FFmpeg with detected hardware acceleration for camera {self.camera_id}") - logger.debug(f"Camera {self.camera_id}: Using FFmpeg options: {optimal_options}") - - self.cap = cv2.VideoCapture(self.rtsp_url, cv2.CAP_FFMPEG) - - if self.cap.isOpened(): - hw_accel_success = True - # Try to get backend info to confirm hardware acceleration - backend = self.cap.getBackendName() - logger.info(f"Camera {self.camera_id}: Using FFmpeg hardware acceleration (backend: {backend})") - except Exception as e: - logger.debug(f"Camera {self.camera_id}: FFmpeg optimal hardware acceleration not available: {e}") - - # Method 3: Try FFmpeg with NVIDIA NVDEC (better for RTX 3060) - if not hw_accel_success: - try: - import os - os.environ['OPENCV_FFMPEG_CAPTURE_OPTIONS'] = 'hwaccel;cuda|hwaccel_device;0|rtsp_transport;tcp' - - logger.info(f"Attempting FFmpeg with NVDEC hardware acceleration for camera {self.camera_id}") - self.cap = cv2.VideoCapture(self.rtsp_url, cv2.CAP_FFMPEG) - - if self.cap.isOpened(): - hw_accel_success = True - logger.info(f"Camera {self.camera_id}: Using FFmpeg NVDEC hardware acceleration") - except Exception as e: - logger.debug(f"Camera {self.camera_id}: FFmpeg NVDEC not available: {e}") - - # Method 4: Try FFmpeg with VAAPI (Intel/AMD GPUs) - if not hw_accel_success: - try: - import os - os.environ['OPENCV_FFMPEG_CAPTURE_OPTIONS'] = 'hwaccel;vaapi|hwaccel_device;/dev/dri/renderD128|video_codec;h264|rtsp_transport;tcp' - - logger.info(f"Attempting FFmpeg with VAAPI for camera {self.camera_id}") - self.cap = cv2.VideoCapture(self.rtsp_url, cv2.CAP_FFMPEG) - - if self.cap.isOpened(): - hw_accel_success = True - logger.info(f"Camera {self.camera_id}: Using FFmpeg VAAPI hardware acceleration") - except Exception as e: - logger.debug(f"Camera {self.camera_id}: FFmpeg VAAPI not available: {e}") - - # Fallback: Standard FFmpeg with software decoding - if not hw_accel_success: - logger.warning(f"Camera {self.camera_id}: Hardware acceleration not available, falling back to software decoding") - import os - os.environ['OPENCV_FFMPEG_CAPTURE_OPTIONS'] = 'rtsp_transport;tcp' - self.cap = cv2.VideoCapture(self.rtsp_url, cv2.CAP_FFMPEG) - - if not self.cap.isOpened(): - logger.error(f"Failed to open stream for camera {self.camera_id}") - return False - - # Don't force resolution/fps - let the stream determine its natural specs - # The camera will provide whatever resolution/fps it supports - - - # Set FFMPEG options for better H.264 handling - self.cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*'H264')) - - # Verify stream properties - actual_width = int(self.cap.get(cv2.CAP_PROP_FRAME_WIDTH)) - actual_height = int(self.cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) - actual_fps = self.cap.get(cv2.CAP_PROP_FPS) - - logger.info(f"Camera {self.camera_id} initialized: {actual_width}x{actual_height} @ {actual_fps}fps") - - # Read and discard first few frames to stabilize stream - for _ in range(5): - ret, _ = self.cap.read() - if not ret: - logger.warning(f"Camera {self.camera_id}: Failed to read initial frames") - time.sleep(0.1) - - return True - - except Exception as e: - logger.error(f"Error initializing capture for camera {self.camera_id}: {e}") - return False - - def _reinitialize_capture(self): - """Reinitialize capture after errors with retry logic.""" - logger.info(f"Reinitializing capture for camera {self.camera_id}") - if self.cap: - self.cap.release() - self.cap = None - - # Longer delay before reconnection to avoid rapid reconnect loops - time.sleep(3.0) - - # Retry initialization up to 3 times - for attempt in range(3): - if self._initialize_capture(): - logger.info(f"Successfully reinitialized camera {self.camera_id} on attempt {attempt + 1}") - break - else: - logger.warning(f"Failed to reinitialize camera {self.camera_id} on attempt {attempt + 1}") - time.sleep(2.0) - - def _is_frame_corrupted(self, frame: np.ndarray) -> bool: - """Check if frame is corrupted (all black, all white, or excessive noise).""" - if frame is None or frame.size == 0: - return True - - # Check mean and standard deviation - mean = np.mean(frame) - std = np.std(frame) - - # All black or all white - if mean < 5 or mean > 250: - return True - - # No variation (stuck frame) - if std < 1: - return True - - # Excessive noise (corrupted H.264 decode) - # Calculate edge density as corruption indicator - edges = cv2.Canny(frame, 50, 150) - edge_density = np.sum(edges > 0) / edges.size - - # Too many edges indicate corruption - if edge_density > 0.5: - return True - - return False - - class HTTPSnapshotReader: """HTTP snapshot reader optimized for 2560x1440 (2K) high quality images.""" From 33d738b31b353433d104ff0104c6bb49ffe8ac7e Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Fri, 26 Sep 2025 19:42:57 +0700 Subject: [PATCH 38/55] fix: remove unused watchdog logging configuration and FrameFileHandler --- core/streaming/readers.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/core/streaming/readers.py b/core/streaming/readers.py index 5684997..c8c0ec3 100644 --- a/core/streaming/readers.py +++ b/core/streaming/readers.py @@ -43,11 +43,6 @@ def log_info(camera_id: str, message: str): """Log info in cyan""" logger.info(f"{Colors.CYAN}[{camera_id}] {message}{Colors.END}") -# Removed watchdog logging configuration - no longer using file watching - - -# Removed FrameFileHandler - no longer using file watching - class FFmpegRTSPReader: """RTSP stream reader using subprocess FFmpeg piping frames directly to buffer.""" From d8d1b33cd86490cc075a4ca8a208dd68099f86e5 Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Fri, 26 Sep 2025 19:47:13 +0700 Subject: [PATCH 39/55] feat: add GPU accelerated libraries --- requirements.base.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/requirements.base.txt b/requirements.base.txt index 3511dd4..722962f 100644 --- a/requirements.base.txt +++ b/requirements.base.txt @@ -7,4 +7,7 @@ filterpy psycopg2-binary lap>=0.5.12 pynvml -PyTurboJPEG \ No newline at end of file +PyTurboJPEG +PyNvVideoCodec +pycuda +cupy-cuda12x \ No newline at end of file From 2b382210eb702a0ff87a5ad64e721f2881deffec Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Fri, 26 Sep 2025 20:03:09 +0700 Subject: [PATCH 40/55] Refactor streaming readers: Split into modular files and implement base class - Removed the existing `readers.py` file and created separate modules for `FFmpegRTSPReader`, `HTTPSnapshotReader`, and utility functions. - Introduced an abstract base class `VideoReader` to standardize the interface for video stream readers. - Updated `FFmpegRTSPReader` and `HTTPSnapshotReader` to inherit from `VideoReader` and implement required methods. - Enhanced logging utilities for better readability and maintainability. - Removed `pycuda` from requirements as it is no longer needed. --- core/streaming/readers.py | 557 ------------------------ core/streaming/readers/__init__.py | 18 + core/streaming/readers/base.py | 65 +++ core/streaming/readers/ffmpeg_rtsp.py | 302 +++++++++++++ core/streaming/readers/http_snapshot.py | 249 +++++++++++ core/streaming/readers/utils.py | 38 ++ requirements.base.txt | 1 - 7 files changed, 672 insertions(+), 558 deletions(-) delete mode 100644 core/streaming/readers.py create mode 100644 core/streaming/readers/__init__.py create mode 100644 core/streaming/readers/base.py create mode 100644 core/streaming/readers/ffmpeg_rtsp.py create mode 100644 core/streaming/readers/http_snapshot.py create mode 100644 core/streaming/readers/utils.py diff --git a/core/streaming/readers.py b/core/streaming/readers.py deleted file mode 100644 index c8c0ec3..0000000 --- a/core/streaming/readers.py +++ /dev/null @@ -1,557 +0,0 @@ -""" -Frame readers for RTSP streams and HTTP snapshots. -Optimized for 1280x720@6fps RTSP and 2560x1440 HTTP snapshots. -""" -import cv2 -import logging -import time -import threading -import requests -import numpy as np -import subprocess -from typing import Optional, Callable - - - -logger = logging.getLogger(__name__) - -# Color codes for pretty logging -class Colors: - GREEN = '\033[92m' - YELLOW = '\033[93m' - RED = '\033[91m' - BLUE = '\033[94m' - PURPLE = '\033[95m' - CYAN = '\033[96m' - WHITE = '\033[97m' - BOLD = '\033[1m' - END = '\033[0m' - -def log_success(camera_id: str, message: str): - """Log success messages in green""" - logger.info(f"{Colors.GREEN}[{camera_id}] {message}{Colors.END}") - -def log_warning(camera_id: str, message: str): - """Log warnings in yellow""" - logger.warning(f"{Colors.YELLOW}[{camera_id}] {message}{Colors.END}") - -def log_error(camera_id: str, message: str): - """Log errors in red""" - logger.error(f"{Colors.RED}[{camera_id}] {message}{Colors.END}") - -def log_info(camera_id: str, message: str): - """Log info in cyan""" - logger.info(f"{Colors.CYAN}[{camera_id}] {message}{Colors.END}") - - -class FFmpegRTSPReader: - """RTSP stream reader using subprocess FFmpeg piping frames directly to buffer.""" - - def __init__(self, camera_id: str, rtsp_url: str, max_retries: int = 3): - self.camera_id = camera_id - self.rtsp_url = rtsp_url - self.max_retries = max_retries - self.process = None - self.stop_event = threading.Event() - self.thread = None - self.stderr_thread = None - self.frame_callback: Optional[Callable] = None - - # Expected stream specs (for reference, actual dimensions read from PPM header) - self.width = 1280 - self.height = 720 - - # Watchdog timers for stream reliability - self.process_start_time = None - self.last_frame_time = None - self.is_restart = False # Track if this is a restart (shorter timeout) - self.first_start_timeout = 30.0 # 30s timeout on first start - self.restart_timeout = 15.0 # 15s timeout after restart - - def set_frame_callback(self, callback: Callable[[str, np.ndarray], None]): - """Set callback function to handle captured frames.""" - self.frame_callback = callback - - def start(self): - """Start the FFmpeg subprocess reader.""" - if self.thread and self.thread.is_alive(): - logger.warning(f"FFmpeg reader for {self.camera_id} already running") - return - - self.stop_event.clear() - self.thread = threading.Thread(target=self._read_frames, daemon=True) - self.thread.start() - log_success(self.camera_id, "Stream started") - - def stop(self): - """Stop the FFmpeg subprocess reader.""" - self.stop_event.set() - if self.process: - self.process.terminate() - try: - self.process.wait(timeout=5) - except subprocess.TimeoutExpired: - self.process.kill() - if self.thread: - self.thread.join(timeout=5.0) - if self.stderr_thread: - self.stderr_thread.join(timeout=2.0) - log_info(self.camera_id, "Stream stopped") - - # Removed _probe_stream_info - BMP headers contain dimensions - - def _start_ffmpeg_process(self): - """Start FFmpeg subprocess outputting BMP frames to stdout pipe.""" - cmd = [ - 'ffmpeg', - # DO NOT REMOVE - '-hwaccel', 'cuda', - '-hwaccel_device', '0', - '-rtsp_transport', 'tcp', - '-i', self.rtsp_url, - '-f', 'image2pipe', # Output images to pipe - '-vcodec', 'bmp', # BMP format with header containing dimensions - # Use native stream resolution and framerate - '-an', # No audio - '-' # Output to stdout - ] - - try: - # Start FFmpeg with stdout pipe to read frames directly - self.process = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, # Capture stdout for frame data - stderr=subprocess.PIPE, # Capture stderr for error logging - bufsize=0 # Unbuffered for real-time processing - ) - - # Start stderr reading thread - if self.stderr_thread and self.stderr_thread.is_alive(): - # Stop previous stderr thread - try: - self.stderr_thread.join(timeout=1.0) - except: - pass - - self.stderr_thread = threading.Thread(target=self._read_stderr, daemon=True) - self.stderr_thread.start() - - # Set process start time for watchdog - self.process_start_time = time.time() - self.last_frame_time = None # Reset frame time - - # After successful restart, next timeout will be back to 30s - if self.is_restart: - log_info(self.camera_id, f"FFmpeg restarted successfully, next timeout: {self.first_start_timeout}s") - self.is_restart = False - - return True - except Exception as e: - log_error(self.camera_id, f"FFmpeg startup failed: {e}") - return False - - def _read_bmp_frame(self, pipe): - """Read BMP frame from pipe - BMP header contains dimensions.""" - try: - # Read BMP header (14 bytes file header + 40 bytes info header = 54 bytes minimum) - header_data = b'' - bytes_to_read = 54 - - while len(header_data) < bytes_to_read: - chunk = pipe.read(bytes_to_read - len(header_data)) - if not chunk: - return None # Silent end of stream - header_data += chunk - - # Parse BMP header - if header_data[:2] != b'BM': - return None # Invalid format, skip frame silently - - # Extract file size from header (bytes 2-5) - import struct - file_size = struct.unpack(' bool: - """Check if watchdog timeout has been exceeded.""" - if not self.process_start_time: - return False - - current_time = time.time() - time_since_start = current_time - self.process_start_time - - # Determine timeout based on whether this is a restart - timeout = self.restart_timeout if self.is_restart else self.first_start_timeout - - # If no frames received yet, check against process start time - if not self.last_frame_time: - if time_since_start > timeout: - log_warning(self.camera_id, f"Watchdog timeout: No frames for {time_since_start:.1f}s (limit: {timeout}s)") - return True - else: - # Check time since last frame - time_since_frame = current_time - self.last_frame_time - if time_since_frame > timeout: - log_warning(self.camera_id, f"Watchdog timeout: No frames for {time_since_frame:.1f}s (limit: {timeout}s)") - return True - - return False - - def _restart_ffmpeg_process(self): - """Restart FFmpeg process due to watchdog timeout.""" - log_warning(self.camera_id, "Watchdog triggered FFmpeg restart") - - # Terminate current process - if self.process: - try: - self.process.terminate() - self.process.wait(timeout=3) - except subprocess.TimeoutExpired: - self.process.kill() - except Exception: - pass - self.process = None - - # Mark as restart for shorter timeout - self.is_restart = True - - # Small delay before restart - time.sleep(1.0) - - def _read_frames(self): - """Read frames directly from FFmpeg stdout pipe.""" - frame_count = 0 - last_log_time = time.time() - - while not self.stop_event.is_set(): - try: - # Check watchdog timeout if process is running - if self.process and self.process.poll() is None: - if self._check_watchdog_timeout(): - self._restart_ffmpeg_process() - continue - - # Start FFmpeg if not running - if not self.process or self.process.poll() is not None: - if self.process and self.process.poll() is not None: - log_warning(self.camera_id, "Stream disconnected, reconnecting...") - - if not self._start_ffmpeg_process(): - time.sleep(5.0) - continue - - # Read frames directly from FFmpeg stdout - try: - if self.process and self.process.stdout: - # Read BMP frame data - frame = self._read_bmp_frame(self.process.stdout) - if frame is None: - continue - - # Update watchdog - we got a frame - self.last_frame_time = time.time() - - # Call frame callback - if self.frame_callback: - self.frame_callback(self.camera_id, frame) - - frame_count += 1 - - # Log progress every 60 seconds (quieter) - current_time = time.time() - if current_time - last_log_time >= 60: - log_success(self.camera_id, f"{frame_count} frames captured ({frame.shape[1]}x{frame.shape[0]})") - last_log_time = current_time - - except Exception: - # Process might have died, let it restart on next iteration - if self.process: - self.process.terminate() - self.process = None - time.sleep(1.0) - - except Exception: - time.sleep(1.0) - - # Cleanup - if self.process: - self.process.terminate() - - -logger = logging.getLogger(__name__) - - -class HTTPSnapshotReader: - """HTTP snapshot reader optimized for 2560x1440 (2K) high quality images.""" - - def __init__(self, camera_id: str, snapshot_url: str, interval_ms: int = 5000, max_retries: int = 3): - self.camera_id = camera_id - self.snapshot_url = snapshot_url - self.interval_ms = interval_ms - self.max_retries = max_retries - self.stop_event = threading.Event() - self.thread = None - self.frame_callback: Optional[Callable] = None - - # Expected snapshot specifications - self.expected_width = 2560 - self.expected_height = 1440 - self.max_file_size = 10 * 1024 * 1024 # 10MB max for 2K image - - def set_frame_callback(self, callback: Callable[[str, np.ndarray], None]): - """Set callback function to handle captured frames.""" - self.frame_callback = callback - - def start(self): - """Start the snapshot reader thread.""" - if self.thread and self.thread.is_alive(): - logger.warning(f"Snapshot reader for {self.camera_id} already running") - return - - self.stop_event.clear() - self.thread = threading.Thread(target=self._read_snapshots, daemon=True) - self.thread.start() - logger.info(f"Started snapshot reader for camera {self.camera_id}") - - def stop(self): - """Stop the snapshot reader thread.""" - self.stop_event.set() - if self.thread: - self.thread.join(timeout=5.0) - logger.info(f"Stopped snapshot reader for camera {self.camera_id}") - - def _read_snapshots(self): - """Main snapshot reading loop for high quality 2K images.""" - retries = 0 - frame_count = 0 - last_log_time = time.time() - interval_seconds = self.interval_ms / 1000.0 - - logger.info(f"Snapshot interval for camera {self.camera_id}: {interval_seconds}s") - - while not self.stop_event.is_set(): - try: - start_time = time.time() - frame = self._fetch_snapshot() - - if frame is None: - retries += 1 - logger.warning(f"Failed to fetch snapshot for camera {self.camera_id}, retry {retries}/{self.max_retries}") - - if self.max_retries != -1 and retries > self.max_retries: - logger.error(f"Max retries reached for snapshot camera {self.camera_id}") - break - - time.sleep(min(2.0, interval_seconds)) - continue - - # Accept any valid image dimensions - don't force specific resolution - if frame.shape[1] <= 0 or frame.shape[0] <= 0: - logger.warning(f"Camera {self.camera_id}: Invalid frame dimensions {frame.shape[1]}x{frame.shape[0]}") - continue - - # Reset retry counter on successful fetch - retries = 0 - frame_count += 1 - - # Call frame callback - if self.frame_callback: - try: - self.frame_callback(self.camera_id, frame) - except Exception as e: - logger.error(f"Camera {self.camera_id}: Frame callback error: {e}") - - # Log progress every 30 seconds - current_time = time.time() - if current_time - last_log_time >= 30: - logger.info(f"Camera {self.camera_id}: {frame_count} snapshots processed") - last_log_time = current_time - - # Wait for next interval - elapsed = time.time() - start_time - sleep_time = max(0, interval_seconds - elapsed) - if sleep_time > 0: - self.stop_event.wait(sleep_time) - - except Exception as e: - logger.error(f"Error in snapshot loop for camera {self.camera_id}: {e}") - retries += 1 - if self.max_retries != -1 and retries > self.max_retries: - break - time.sleep(min(2.0, interval_seconds)) - - logger.info(f"Snapshot reader thread ended for camera {self.camera_id}") - - def _fetch_snapshot(self) -> Optional[np.ndarray]: - """Fetch a single high quality snapshot from HTTP URL.""" - try: - # Parse URL for authentication - from urllib.parse import urlparse - parsed_url = urlparse(self.snapshot_url) - - headers = { - 'User-Agent': 'Python-Detector-Worker/1.0', - 'Accept': 'image/jpeg, image/png, image/*' - } - auth = None - - if parsed_url.username and parsed_url.password: - from requests.auth import HTTPBasicAuth, HTTPDigestAuth - auth = HTTPBasicAuth(parsed_url.username, parsed_url.password) - - # Reconstruct URL without credentials - clean_url = f"{parsed_url.scheme}://{parsed_url.hostname}" - if parsed_url.port: - clean_url += f":{parsed_url.port}" - clean_url += parsed_url.path - if parsed_url.query: - clean_url += f"?{parsed_url.query}" - - # Try Basic Auth first - response = requests.get(clean_url, auth=auth, timeout=15, headers=headers, - stream=True, verify=False) - - # If Basic Auth fails, try Digest Auth - if response.status_code == 401: - auth = HTTPDigestAuth(parsed_url.username, parsed_url.password) - response = requests.get(clean_url, auth=auth, timeout=15, headers=headers, - stream=True, verify=False) - else: - response = requests.get(self.snapshot_url, timeout=15, headers=headers, - stream=True, verify=False) - - if response.status_code == 200: - # Check content size - content_length = int(response.headers.get('content-length', 0)) - if content_length > self.max_file_size: - logger.warning(f"Snapshot too large for camera {self.camera_id}: {content_length} bytes") - return None - - # Read content - content = response.content - - # Convert to numpy array - image_array = np.frombuffer(content, np.uint8) - - # Decode as high quality image - frame = cv2.imdecode(image_array, cv2.IMREAD_COLOR) - - if frame is None: - logger.error(f"Failed to decode snapshot for camera {self.camera_id}") - return None - - logger.debug(f"Fetched snapshot for camera {self.camera_id}: {frame.shape[1]}x{frame.shape[0]}") - return frame - else: - logger.warning(f"HTTP {response.status_code} from {self.camera_id}") - return None - - except requests.RequestException as e: - logger.error(f"Request error fetching snapshot for {self.camera_id}: {e}") - return None - except Exception as e: - logger.error(f"Error decoding snapshot for {self.camera_id}: {e}") - return None - - def fetch_single_snapshot(self) -> Optional[np.ndarray]: - """ - Fetch a single high-quality snapshot on demand for pipeline processing. - This method is for one-time fetch from HTTP URL, not continuous streaming. - - Returns: - High quality 2K snapshot frame or None if failed - """ - logger.info(f"[SNAPSHOT] Fetching snapshot for {self.camera_id} from {self.snapshot_url}") - - # Try to fetch snapshot with retries - for attempt in range(self.max_retries): - frame = self._fetch_snapshot() - - if frame is not None: - logger.info(f"[SNAPSHOT] Successfully fetched {frame.shape[1]}x{frame.shape[0]} snapshot for {self.camera_id}") - return frame - - if attempt < self.max_retries - 1: - logger.warning(f"[SNAPSHOT] Attempt {attempt + 1}/{self.max_retries} failed for {self.camera_id}, retrying...") - time.sleep(0.5) - - logger.error(f"[SNAPSHOT] Failed to fetch snapshot for {self.camera_id} after {self.max_retries} attempts") - return None - - def _resize_maintain_aspect(self, frame: np.ndarray, target_width: int, target_height: int) -> np.ndarray: - """Resize image while maintaining aspect ratio for high quality.""" - h, w = frame.shape[:2] - aspect = w / h - target_aspect = target_width / target_height - - if aspect > target_aspect: - # Image is wider - new_width = target_width - new_height = int(target_width / aspect) - else: - # Image is taller - new_height = target_height - new_width = int(target_height * aspect) - - # Use INTER_LANCZOS4 for high quality downsampling - resized = cv2.resize(frame, (new_width, new_height), interpolation=cv2.INTER_LANCZOS4) - - # Pad to target size if needed - if new_width < target_width or new_height < target_height: - top = (target_height - new_height) // 2 - bottom = target_height - new_height - top - left = (target_width - new_width) // 2 - right = target_width - new_width - left - resized = cv2.copyMakeBorder(resized, top, bottom, left, right, cv2.BORDER_CONSTANT, value=[0, 0, 0]) - - return resized \ No newline at end of file diff --git a/core/streaming/readers/__init__.py b/core/streaming/readers/__init__.py new file mode 100644 index 0000000..0903d6d --- /dev/null +++ b/core/streaming/readers/__init__.py @@ -0,0 +1,18 @@ +""" +Stream readers for RTSP and HTTP camera feeds. +""" +from .base import VideoReader +from .ffmpeg_rtsp import FFmpegRTSPReader +from .http_snapshot import HTTPSnapshotReader +from .utils import log_success, log_warning, log_error, log_info, Colors + +__all__ = [ + 'VideoReader', + 'FFmpegRTSPReader', + 'HTTPSnapshotReader', + 'log_success', + 'log_warning', + 'log_error', + 'log_info', + 'Colors' +] \ No newline at end of file diff --git a/core/streaming/readers/base.py b/core/streaming/readers/base.py new file mode 100644 index 0000000..56c41cb --- /dev/null +++ b/core/streaming/readers/base.py @@ -0,0 +1,65 @@ +""" +Abstract base class for video stream readers. +""" +from abc import ABC, abstractmethod +from typing import Optional, Callable +import numpy as np + + +class VideoReader(ABC): + """Abstract base class for video stream readers.""" + + def __init__(self, camera_id: str, source_url: str, max_retries: int = 3): + """ + Initialize the video reader. + + Args: + camera_id: Unique identifier for the camera + source_url: URL or path to the video source + max_retries: Maximum number of retry attempts + """ + self.camera_id = camera_id + self.source_url = source_url + self.max_retries = max_retries + self.frame_callback: Optional[Callable[[str, np.ndarray], None]] = None + + @abstractmethod + def start(self) -> None: + """Start the video reader.""" + pass + + @abstractmethod + def stop(self) -> None: + """Stop the video reader.""" + pass + + @abstractmethod + def set_frame_callback(self, callback: Callable[[str, np.ndarray], None]) -> None: + """ + Set callback function to handle captured frames. + + Args: + callback: Function that takes (camera_id, frame) as arguments + """ + pass + + @property + @abstractmethod + def is_running(self) -> bool: + """Check if the reader is currently running.""" + pass + + @property + @abstractmethod + def reader_type(self) -> str: + """Get the type of reader (e.g., 'rtsp', 'http_snapshot').""" + pass + + def __enter__(self): + """Context manager entry.""" + self.start() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """Context manager exit.""" + self.stop() \ No newline at end of file diff --git a/core/streaming/readers/ffmpeg_rtsp.py b/core/streaming/readers/ffmpeg_rtsp.py new file mode 100644 index 0000000..8641495 --- /dev/null +++ b/core/streaming/readers/ffmpeg_rtsp.py @@ -0,0 +1,302 @@ +""" +FFmpeg RTSP stream reader using subprocess piping frames directly to buffer. +""" +import cv2 +import time +import threading +import numpy as np +import subprocess +import struct +from typing import Optional, Callable + +from .base import VideoReader +from .utils import log_success, log_warning, log_error, log_info + + +class FFmpegRTSPReader(VideoReader): + """RTSP stream reader using subprocess FFmpeg piping frames directly to buffer.""" + + def __init__(self, camera_id: str, rtsp_url: str, max_retries: int = 3): + super().__init__(camera_id, rtsp_url, max_retries) + self.rtsp_url = rtsp_url + self.process = None + self.stop_event = threading.Event() + self.thread = None + self.stderr_thread = None + + # Expected stream specs (for reference, actual dimensions read from PPM header) + self.width = 1280 + self.height = 720 + + # Watchdog timers for stream reliability + self.process_start_time = None + self.last_frame_time = None + self.is_restart = False # Track if this is a restart (shorter timeout) + self.first_start_timeout = 30.0 # 30s timeout on first start + self.restart_timeout = 15.0 # 15s timeout after restart + + @property + def is_running(self) -> bool: + """Check if the reader is currently running.""" + return self.thread is not None and self.thread.is_alive() + + @property + def reader_type(self) -> str: + """Get the type of reader.""" + return "rtsp_ffmpeg" + + def set_frame_callback(self, callback: Callable[[str, np.ndarray], None]): + """Set callback function to handle captured frames.""" + self.frame_callback = callback + + def start(self): + """Start the FFmpeg subprocess reader.""" + if self.thread and self.thread.is_alive(): + log_warning(self.camera_id, "FFmpeg reader already running") + return + + self.stop_event.clear() + self.thread = threading.Thread(target=self._read_frames, daemon=True) + self.thread.start() + log_success(self.camera_id, "Stream started") + + def stop(self): + """Stop the FFmpeg subprocess reader.""" + self.stop_event.set() + if self.process: + self.process.terminate() + try: + self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + self.process.kill() + if self.thread: + self.thread.join(timeout=5.0) + if self.stderr_thread: + self.stderr_thread.join(timeout=2.0) + log_info(self.camera_id, "Stream stopped") + + def _start_ffmpeg_process(self): + """Start FFmpeg subprocess outputting BMP frames to stdout pipe.""" + cmd = [ + 'ffmpeg', + # DO NOT REMOVE + '-hwaccel', 'cuda', + '-hwaccel_device', '0', + '-rtsp_transport', 'tcp', + '-i', self.rtsp_url, + '-f', 'image2pipe', # Output images to pipe + '-vcodec', 'bmp', # BMP format with header containing dimensions + # Use native stream resolution and framerate + '-an', # No audio + '-' # Output to stdout + ] + + try: + # Start FFmpeg with stdout pipe to read frames directly + self.process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, # Capture stdout for frame data + stderr=subprocess.PIPE, # Capture stderr for error logging + bufsize=0 # Unbuffered for real-time processing + ) + + # Start stderr reading thread + if self.stderr_thread and self.stderr_thread.is_alive(): + # Stop previous stderr thread + try: + self.stderr_thread.join(timeout=1.0) + except: + pass + + self.stderr_thread = threading.Thread(target=self._read_stderr, daemon=True) + self.stderr_thread.start() + + # Set process start time for watchdog + self.process_start_time = time.time() + self.last_frame_time = None # Reset frame time + + # After successful restart, next timeout will be back to 30s + if self.is_restart: + log_info(self.camera_id, f"FFmpeg restarted successfully, next timeout: {self.first_start_timeout}s") + self.is_restart = False + + return True + except Exception as e: + log_error(self.camera_id, f"FFmpeg startup failed: {e}") + return False + + def _read_bmp_frame(self, pipe): + """Read BMP frame from pipe - BMP header contains dimensions.""" + try: + # Read BMP header (14 bytes file header + 40 bytes info header = 54 bytes minimum) + header_data = b'' + bytes_to_read = 54 + + while len(header_data) < bytes_to_read: + chunk = pipe.read(bytes_to_read - len(header_data)) + if not chunk: + return None # Silent end of stream + header_data += chunk + + # Parse BMP header + if header_data[:2] != b'BM': + return None # Invalid format, skip frame silently + + # Extract file size from header (bytes 2-5) + file_size = struct.unpack(' bool: + """Check if watchdog timeout has been exceeded.""" + if not self.process_start_time: + return False + + current_time = time.time() + time_since_start = current_time - self.process_start_time + + # Determine timeout based on whether this is a restart + timeout = self.restart_timeout if self.is_restart else self.first_start_timeout + + # If no frames received yet, check against process start time + if not self.last_frame_time: + if time_since_start > timeout: + log_warning(self.camera_id, f"Watchdog timeout: No frames for {time_since_start:.1f}s (limit: {timeout}s)") + return True + else: + # Check time since last frame + time_since_frame = current_time - self.last_frame_time + if time_since_frame > timeout: + log_warning(self.camera_id, f"Watchdog timeout: No frames for {time_since_frame:.1f}s (limit: {timeout}s)") + return True + + return False + + def _restart_ffmpeg_process(self): + """Restart FFmpeg process due to watchdog timeout.""" + log_warning(self.camera_id, "Watchdog triggered FFmpeg restart") + + # Terminate current process + if self.process: + try: + self.process.terminate() + self.process.wait(timeout=3) + except subprocess.TimeoutExpired: + self.process.kill() + except Exception: + pass + self.process = None + + # Mark as restart for shorter timeout + self.is_restart = True + + # Small delay before restart + time.sleep(1.0) + + def _read_frames(self): + """Read frames directly from FFmpeg stdout pipe.""" + frame_count = 0 + last_log_time = time.time() + + while not self.stop_event.is_set(): + try: + # Check watchdog timeout if process is running + if self.process and self.process.poll() is None: + if self._check_watchdog_timeout(): + self._restart_ffmpeg_process() + continue + + # Start FFmpeg if not running + if not self.process or self.process.poll() is not None: + if self.process and self.process.poll() is not None: + log_warning(self.camera_id, "Stream disconnected, reconnecting...") + + if not self._start_ffmpeg_process(): + time.sleep(5.0) + continue + + # Read frames directly from FFmpeg stdout + try: + if self.process and self.process.stdout: + # Read BMP frame data + frame = self._read_bmp_frame(self.process.stdout) + if frame is None: + continue + + # Update watchdog - we got a frame + self.last_frame_time = time.time() + + # Call frame callback + if self.frame_callback: + self.frame_callback(self.camera_id, frame) + + frame_count += 1 + + # Log progress every 60 seconds (quieter) + current_time = time.time() + if current_time - last_log_time >= 60: + log_success(self.camera_id, f"{frame_count} frames captured ({frame.shape[1]}x{frame.shape[0]})") + last_log_time = current_time + + except Exception: + # Process might have died, let it restart on next iteration + if self.process: + self.process.terminate() + self.process = None + time.sleep(1.0) + + except Exception: + time.sleep(1.0) + + # Cleanup + if self.process: + self.process.terminate() \ No newline at end of file diff --git a/core/streaming/readers/http_snapshot.py b/core/streaming/readers/http_snapshot.py new file mode 100644 index 0000000..5a479db --- /dev/null +++ b/core/streaming/readers/http_snapshot.py @@ -0,0 +1,249 @@ +""" +HTTP snapshot reader optimized for 2560x1440 (2K) high quality images. +""" +import cv2 +import logging +import time +import threading +import requests +import numpy as np +from typing import Optional, Callable + +from .base import VideoReader +from .utils import log_success, log_warning, log_error, log_info + +logger = logging.getLogger(__name__) + + +class HTTPSnapshotReader(VideoReader): + """HTTP snapshot reader optimized for 2560x1440 (2K) high quality images.""" + + def __init__(self, camera_id: str, snapshot_url: str, interval_ms: int = 5000, max_retries: int = 3): + super().__init__(camera_id, snapshot_url, max_retries) + self.snapshot_url = snapshot_url + self.interval_ms = interval_ms + self.stop_event = threading.Event() + self.thread = None + + # Expected snapshot specifications + self.expected_width = 2560 + self.expected_height = 1440 + self.max_file_size = 10 * 1024 * 1024 # 10MB max for 2K image + + @property + def is_running(self) -> bool: + """Check if the reader is currently running.""" + return self.thread is not None and self.thread.is_alive() + + @property + def reader_type(self) -> str: + """Get the type of reader.""" + return "http_snapshot" + + def set_frame_callback(self, callback: Callable[[str, np.ndarray], None]): + """Set callback function to handle captured frames.""" + self.frame_callback = callback + + def start(self): + """Start the snapshot reader thread.""" + if self.thread and self.thread.is_alive(): + logger.warning(f"Snapshot reader for {self.camera_id} already running") + return + + self.stop_event.clear() + self.thread = threading.Thread(target=self._read_snapshots, daemon=True) + self.thread.start() + logger.info(f"Started snapshot reader for camera {self.camera_id}") + + def stop(self): + """Stop the snapshot reader thread.""" + self.stop_event.set() + if self.thread: + self.thread.join(timeout=5.0) + logger.info(f"Stopped snapshot reader for camera {self.camera_id}") + + def _read_snapshots(self): + """Main snapshot reading loop for high quality 2K images.""" + retries = 0 + frame_count = 0 + last_log_time = time.time() + interval_seconds = self.interval_ms / 1000.0 + + logger.info(f"Snapshot interval for camera {self.camera_id}: {interval_seconds}s") + + while not self.stop_event.is_set(): + try: + start_time = time.time() + frame = self._fetch_snapshot() + + if frame is None: + retries += 1 + logger.warning(f"Failed to fetch snapshot for camera {self.camera_id}, retry {retries}/{self.max_retries}") + + if self.max_retries != -1 and retries > self.max_retries: + logger.error(f"Max retries reached for snapshot camera {self.camera_id}") + break + + time.sleep(min(2.0, interval_seconds)) + continue + + # Accept any valid image dimensions - don't force specific resolution + if frame.shape[1] <= 0 or frame.shape[0] <= 0: + logger.warning(f"Camera {self.camera_id}: Invalid frame dimensions {frame.shape[1]}x{frame.shape[0]}") + continue + + # Reset retry counter on successful fetch + retries = 0 + frame_count += 1 + + # Call frame callback + if self.frame_callback: + try: + self.frame_callback(self.camera_id, frame) + except Exception as e: + logger.error(f"Camera {self.camera_id}: Frame callback error: {e}") + + # Log progress every 30 seconds + current_time = time.time() + if current_time - last_log_time >= 30: + logger.info(f"Camera {self.camera_id}: {frame_count} snapshots processed") + last_log_time = current_time + + # Wait for next interval + elapsed = time.time() - start_time + sleep_time = max(0, interval_seconds - elapsed) + if sleep_time > 0: + self.stop_event.wait(sleep_time) + + except Exception as e: + logger.error(f"Error in snapshot loop for camera {self.camera_id}: {e}") + retries += 1 + if self.max_retries != -1 and retries > self.max_retries: + break + time.sleep(min(2.0, interval_seconds)) + + logger.info(f"Snapshot reader thread ended for camera {self.camera_id}") + + def _fetch_snapshot(self) -> Optional[np.ndarray]: + """Fetch a single high quality snapshot from HTTP URL.""" + try: + # Parse URL for authentication + from urllib.parse import urlparse + parsed_url = urlparse(self.snapshot_url) + + headers = { + 'User-Agent': 'Python-Detector-Worker/1.0', + 'Accept': 'image/jpeg, image/png, image/*' + } + auth = None + + if parsed_url.username and parsed_url.password: + from requests.auth import HTTPBasicAuth, HTTPDigestAuth + auth = HTTPBasicAuth(parsed_url.username, parsed_url.password) + + # Reconstruct URL without credentials + clean_url = f"{parsed_url.scheme}://{parsed_url.hostname}" + if parsed_url.port: + clean_url += f":{parsed_url.port}" + clean_url += parsed_url.path + if parsed_url.query: + clean_url += f"?{parsed_url.query}" + + # Try Basic Auth first + response = requests.get(clean_url, auth=auth, timeout=15, headers=headers, + stream=True, verify=False) + + # If Basic Auth fails, try Digest Auth + if response.status_code == 401: + auth = HTTPDigestAuth(parsed_url.username, parsed_url.password) + response = requests.get(clean_url, auth=auth, timeout=15, headers=headers, + stream=True, verify=False) + else: + response = requests.get(self.snapshot_url, timeout=15, headers=headers, + stream=True, verify=False) + + if response.status_code == 200: + # Check content size + content_length = int(response.headers.get('content-length', 0)) + if content_length > self.max_file_size: + logger.warning(f"Snapshot too large for camera {self.camera_id}: {content_length} bytes") + return None + + # Read content + content = response.content + + # Convert to numpy array + image_array = np.frombuffer(content, np.uint8) + + # Decode as high quality image + frame = cv2.imdecode(image_array, cv2.IMREAD_COLOR) + + if frame is None: + logger.error(f"Failed to decode snapshot for camera {self.camera_id}") + return None + + logger.debug(f"Fetched snapshot for camera {self.camera_id}: {frame.shape[1]}x{frame.shape[0]}") + return frame + else: + logger.warning(f"HTTP {response.status_code} from {self.camera_id}") + return None + + except requests.RequestException as e: + logger.error(f"Request error fetching snapshot for {self.camera_id}: {e}") + return None + except Exception as e: + logger.error(f"Error decoding snapshot for {self.camera_id}: {e}") + return None + + def fetch_single_snapshot(self) -> Optional[np.ndarray]: + """ + Fetch a single high-quality snapshot on demand for pipeline processing. + This method is for one-time fetch from HTTP URL, not continuous streaming. + + Returns: + High quality 2K snapshot frame or None if failed + """ + logger.info(f"[SNAPSHOT] Fetching snapshot for {self.camera_id} from {self.snapshot_url}") + + # Try to fetch snapshot with retries + for attempt in range(self.max_retries): + frame = self._fetch_snapshot() + + if frame is not None: + logger.info(f"[SNAPSHOT] Successfully fetched {frame.shape[1]}x{frame.shape[0]} snapshot for {self.camera_id}") + return frame + + if attempt < self.max_retries - 1: + logger.warning(f"[SNAPSHOT] Attempt {attempt + 1}/{self.max_retries} failed for {self.camera_id}, retrying...") + time.sleep(0.5) + + logger.error(f"[SNAPSHOT] Failed to fetch snapshot for {self.camera_id} after {self.max_retries} attempts") + return None + + def _resize_maintain_aspect(self, frame: np.ndarray, target_width: int, target_height: int) -> np.ndarray: + """Resize image while maintaining aspect ratio for high quality.""" + h, w = frame.shape[:2] + aspect = w / h + target_aspect = target_width / target_height + + if aspect > target_aspect: + # Image is wider + new_width = target_width + new_height = int(target_width / aspect) + else: + # Image is taller + new_height = target_height + new_width = int(target_height * aspect) + + # Use INTER_LANCZOS4 for high quality downsampling + resized = cv2.resize(frame, (new_width, new_height), interpolation=cv2.INTER_LANCZOS4) + + # Pad to target size if needed + if new_width < target_width or new_height < target_height: + top = (target_height - new_height) // 2 + bottom = target_height - new_height - top + left = (target_width - new_width) // 2 + right = target_width - new_width - left + resized = cv2.copyMakeBorder(resized, top, bottom, left, right, cv2.BORDER_CONSTANT, value=[0, 0, 0]) + + return resized \ No newline at end of file diff --git a/core/streaming/readers/utils.py b/core/streaming/readers/utils.py new file mode 100644 index 0000000..813f49f --- /dev/null +++ b/core/streaming/readers/utils.py @@ -0,0 +1,38 @@ +""" +Utility functions for stream readers. +""" +import logging +import os + +# Keep OpenCV errors visible but allow FFmpeg stderr logging +os.environ["OPENCV_LOG_LEVEL"] = "ERROR" + +logger = logging.getLogger(__name__) + +# Color codes for pretty logging +class Colors: + GREEN = '\033[92m' + YELLOW = '\033[93m' + RED = '\033[91m' + BLUE = '\033[94m' + PURPLE = '\033[95m' + CYAN = '\033[96m' + WHITE = '\033[97m' + BOLD = '\033[1m' + END = '\033[0m' + +def log_success(camera_id: str, message: str): + """Log success messages in green""" + logger.info(f"{Colors.GREEN}[{camera_id}] {message}{Colors.END}") + +def log_warning(camera_id: str, message: str): + """Log warnings in yellow""" + logger.warning(f"{Colors.YELLOW}[{camera_id}] {message}{Colors.END}") + +def log_error(camera_id: str, message: str): + """Log errors in red""" + logger.error(f"{Colors.RED}[{camera_id}] {message}{Colors.END}") + +def log_info(camera_id: str, message: str): + """Log info in cyan""" + logger.info(f"{Colors.CYAN}[{camera_id}] {message}{Colors.END}") \ No newline at end of file diff --git a/requirements.base.txt b/requirements.base.txt index 722962f..b8af923 100644 --- a/requirements.base.txt +++ b/requirements.base.txt @@ -9,5 +9,4 @@ lap>=0.5.12 pynvml PyTurboJPEG PyNvVideoCodec -pycuda cupy-cuda12x \ No newline at end of file From b08ce27de22a80e31f34cc5f3b89756d74eb2677 Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Sat, 27 Sep 2025 12:27:38 +0700 Subject: [PATCH 41/55] Implement comprehensive health monitoring for streams and threads - Added RecoveryManager for automatic handling of health issues, including circuit breaker patterns, automatic restarts, and graceful degradation. - Introduced StreamHealthTracker to monitor video stream metrics, including frame production, connection health, and error rates. - Developed ThreadHealthMonitor for detecting unresponsive and deadlocked threads, providing liveness detection and responsiveness testing. - Integrated health checks for streams and threads, reporting metrics and recovery actions to the health monitor. - Enhanced logging for recovery attempts, errors, and health checks to improve observability and debugging. --- .claude/settings.local.json | 3 +- app.py | 314 ++++++++++++++++ core/monitoring/__init__.py | 18 + core/monitoring/health.py | 456 ++++++++++++++++++++++++ core/monitoring/recovery.py | 385 ++++++++++++++++++++ core/monitoring/stream_health.py | 351 ++++++++++++++++++ core/monitoring/thread_health.py | 381 ++++++++++++++++++++ core/streaming/readers/ffmpeg_rtsp.py | 139 +++++++- core/streaming/readers/http_snapshot.py | 137 ++++++- 9 files changed, 2173 insertions(+), 11 deletions(-) create mode 100644 core/monitoring/__init__.py create mode 100644 core/monitoring/health.py create mode 100644 core/monitoring/recovery.py create mode 100644 core/monitoring/stream_health.py create mode 100644 core/monitoring/thread_health.py diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 97cf5c1..9e296ac 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -2,7 +2,8 @@ "permissions": { "allow": [ "Bash(dir:*)", - "WebSearch" + "WebSearch", + "Bash(mkdir:*)" ], "deny": [], "ask": [] diff --git a/app.py b/app.py index 605aa0b..eb1440f 100644 --- a/app.py +++ b/app.py @@ -8,6 +8,7 @@ import os import time import cv2 from contextlib import asynccontextmanager +from typing import Dict, Any from fastapi import FastAPI, WebSocket, HTTPException from fastapi.responses import Response @@ -31,21 +32,135 @@ logger.setLevel(logging.DEBUG) # Frames are now stored in the shared cache buffer from core.streaming.buffers # latest_frames = {} # Deprecated - using shared_cache_buffer instead + +# Health monitoring recovery handlers +def _handle_stream_restart_recovery(component: str, details: Dict[str, Any]) -> bool: + """Handle stream restart recovery at the application level.""" + try: + from core.streaming.manager import shared_stream_manager + + # Extract camera ID from component name (e.g., "stream_cam-001" -> "cam-001") + if component.startswith("stream_"): + camera_id = component[7:] # Remove "stream_" prefix + else: + camera_id = component + + logger.info(f"Attempting stream restart recovery for {camera_id}") + + # Find and restart the subscription + subscriptions = shared_stream_manager.get_all_subscriptions() + for sub_info in subscriptions: + if sub_info.camera_id == camera_id: + # Remove and re-add the subscription + shared_stream_manager.remove_subscription(sub_info.subscription_id) + time.sleep(1.0) # Brief delay + + # Re-add subscription + success = shared_stream_manager.add_subscription( + sub_info.subscription_id, + sub_info.stream_config, + sub_info.crop_coords, + sub_info.model_id, + sub_info.model_url, + sub_info.tracking_integration + ) + + if success: + logger.info(f"Stream restart recovery successful for {camera_id}") + return True + else: + logger.error(f"Stream restart recovery failed for {camera_id}") + return False + + logger.warning(f"No subscription found for camera {camera_id} during recovery") + return False + + except Exception as e: + logger.error(f"Error in stream restart recovery for {component}: {e}") + return False + + +def _handle_stream_reconnect_recovery(component: str, details: Dict[str, Any]) -> bool: + """Handle stream reconnect recovery at the application level.""" + try: + from core.streaming.manager import shared_stream_manager + + # Extract camera ID from component name + if component.startswith("stream_"): + camera_id = component[7:] + else: + camera_id = component + + logger.info(f"Attempting stream reconnect recovery for {camera_id}") + + # For reconnect, we just need to trigger the stream's internal reconnect + # The stream readers handle their own reconnection logic + active_cameras = shared_stream_manager.get_active_cameras() + + if camera_id in active_cameras: + logger.info(f"Stream reconnect recovery triggered for {camera_id}") + return True + else: + logger.warning(f"Camera {camera_id} not found in active cameras during reconnect recovery") + return False + + except Exception as e: + logger.error(f"Error in stream reconnect recovery for {component}: {e}") + return False + # Lifespan event handler (modern FastAPI approach) @asynccontextmanager async def lifespan(app: FastAPI): """Application lifespan management.""" # Startup logger.info("Detector Worker started successfully") + + # Initialize health monitoring system + try: + from core.monitoring.health import health_monitor + from core.monitoring.stream_health import stream_health_tracker + from core.monitoring.thread_health import thread_health_monitor + from core.monitoring.recovery import recovery_manager + + # Start health monitoring + health_monitor.start() + logger.info("Health monitoring system started") + + # Register recovery handlers for stream management + from core.streaming.manager import shared_stream_manager + recovery_manager.register_recovery_handler( + "restart_stream", + _handle_stream_restart_recovery + ) + recovery_manager.register_recovery_handler( + "reconnect", + _handle_stream_reconnect_recovery + ) + + logger.info("Recovery handlers registered") + + except Exception as e: + logger.error(f"Failed to initialize health monitoring: {e}") + logger.info("WebSocket endpoint available at: ws://0.0.0.0:8001/") logger.info("HTTP camera endpoint available at: http://0.0.0.0:8001/camera/{camera_id}/image") logger.info("Health check available at: http://0.0.0.0:8001/health") + logger.info("Detailed health monitoring available at: http://0.0.0.0:8001/health/detailed") logger.info("Ready and waiting for backend WebSocket connections") yield # Shutdown logger.info("Detector Worker shutting down...") + + # Stop health monitoring + try: + from core.monitoring.health import health_monitor + health_monitor.stop() + logger.info("Health monitoring system stopped") + except Exception as e: + logger.error(f"Error stopping health monitoring: {e}") + # Clear all state worker_state.set_subscriptions([]) worker_state.session_ids.clear() @@ -197,6 +312,205 @@ async def health_check(): } +@app.get("/health/detailed") +async def detailed_health_check(): + """Comprehensive health status with detailed monitoring data.""" + try: + from core.monitoring.health import health_monitor + from core.monitoring.stream_health import stream_health_tracker + from core.monitoring.thread_health import thread_health_monitor + from core.monitoring.recovery import recovery_manager + + # Get comprehensive health status + overall_health = health_monitor.get_health_status() + stream_metrics = stream_health_tracker.get_all_metrics() + thread_info = thread_health_monitor.get_all_thread_info() + recovery_stats = recovery_manager.get_recovery_stats() + + return { + "timestamp": time.time(), + "overall_health": overall_health, + "stream_metrics": stream_metrics, + "thread_health": thread_info, + "recovery_stats": recovery_stats, + "system_info": { + "active_subscriptions": len(worker_state.subscriptions), + "active_sessions": len(worker_state.session_ids), + "version": "2.0.0" + } + } + + except Exception as e: + logger.error(f"Error generating detailed health report: {e}") + raise HTTPException(status_code=500, detail=f"Health monitoring error: {str(e)}") + + +@app.get("/health/streams") +async def stream_health_status(): + """Stream-specific health monitoring.""" + try: + from core.monitoring.stream_health import stream_health_tracker + from core.streaming.buffers import shared_cache_buffer + + stream_metrics = stream_health_tracker.get_all_metrics() + buffer_stats = shared_cache_buffer.get_stats() + + return { + "timestamp": time.time(), + "stream_count": len(stream_metrics), + "stream_metrics": stream_metrics, + "buffer_stats": buffer_stats, + "frame_ages": { + camera_id: { + "age_seconds": time.time() - info["last_frame_time"] if info and info.get("last_frame_time") else None, + "total_frames": info.get("frame_count", 0) if info else 0 + } + for camera_id, info in stream_metrics.items() + } + } + + except Exception as e: + logger.error(f"Error generating stream health report: {e}") + raise HTTPException(status_code=500, detail=f"Stream health error: {str(e)}") + + +@app.get("/health/threads") +async def thread_health_status(): + """Thread-specific health monitoring.""" + try: + from core.monitoring.thread_health import thread_health_monitor + + thread_info = thread_health_monitor.get_all_thread_info() + deadlocks = thread_health_monitor.detect_deadlocks() + + return { + "timestamp": time.time(), + "thread_count": len(thread_info), + "thread_info": thread_info, + "potential_deadlocks": deadlocks, + "summary": { + "responsive_threads": sum(1 for info in thread_info.values() if info.get("is_responsive", False)), + "unresponsive_threads": sum(1 for info in thread_info.values() if not info.get("is_responsive", True)), + "deadlock_count": len(deadlocks) + } + } + + except Exception as e: + logger.error(f"Error generating thread health report: {e}") + raise HTTPException(status_code=500, detail=f"Thread health error: {str(e)}") + + +@app.get("/health/recovery") +async def recovery_status(): + """Recovery system status and history.""" + try: + from core.monitoring.recovery import recovery_manager + + recovery_stats = recovery_manager.get_recovery_stats() + + return { + "timestamp": time.time(), + "recovery_stats": recovery_stats, + "summary": { + "total_recoveries_last_hour": recovery_stats.get("total_recoveries_last_hour", 0), + "components_with_recovery_state": len(recovery_stats.get("recovery_states", {})), + "total_recovery_failures": sum( + state.get("failure_count", 0) + for state in recovery_stats.get("recovery_states", {}).values() + ), + "total_recovery_successes": sum( + state.get("success_count", 0) + for state in recovery_stats.get("recovery_states", {}).values() + ) + } + } + + except Exception as e: + logger.error(f"Error generating recovery status report: {e}") + raise HTTPException(status_code=500, detail=f"Recovery status error: {str(e)}") + + +@app.post("/health/recovery/force/{component}") +async def force_recovery(component: str, action: str = "restart_stream"): + """Force recovery action for a specific component.""" + try: + from core.monitoring.recovery import recovery_manager, RecoveryAction + + # Validate action + try: + recovery_action = RecoveryAction(action) + except ValueError: + raise HTTPException( + status_code=400, + detail=f"Invalid recovery action: {action}. Valid actions: {[a.value for a in RecoveryAction]}" + ) + + # Force recovery + success = recovery_manager.force_recovery(component, recovery_action, "manual_api_request") + + return { + "timestamp": time.time(), + "component": component, + "action": action, + "success": success, + "message": f"Recovery {'successful' if success else 'failed'} for component {component}" + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error forcing recovery for {component}: {e}") + raise HTTPException(status_code=500, detail=f"Recovery error: {str(e)}") + + +@app.get("/health/metrics") +async def health_metrics(): + """Performance and health metrics in a format suitable for monitoring systems.""" + try: + from core.monitoring.health import health_monitor + from core.monitoring.stream_health import stream_health_tracker + from core.streaming.buffers import shared_cache_buffer + + # Get basic metrics + overall_health = health_monitor.get_health_status() + stream_metrics = stream_health_tracker.get_all_metrics() + buffer_stats = shared_cache_buffer.get_stats() + + # Format for monitoring systems (Prometheus-style) + metrics = { + "detector_worker_up": 1, + "detector_worker_streams_total": len(stream_metrics), + "detector_worker_subscriptions_total": len(worker_state.subscriptions), + "detector_worker_sessions_total": len(worker_state.session_ids), + "detector_worker_memory_mb": buffer_stats.get("total_memory_mb", 0), + "detector_worker_health_status": { + "healthy": 1, + "warning": 2, + "critical": 3, + "unknown": 4 + }.get(overall_health.get("overall_status", "unknown"), 4) + } + + # Add per-stream metrics + for camera_id, stream_info in stream_metrics.items(): + safe_camera_id = camera_id.replace("-", "_").replace(".", "_") + metrics.update({ + f"detector_worker_stream_frames_total{{camera=\"{safe_camera_id}\"}}": stream_info.get("frame_count", 0), + f"detector_worker_stream_errors_total{{camera=\"{safe_camera_id}\"}}": stream_info.get("error_count", 0), + f"detector_worker_stream_fps{{camera=\"{safe_camera_id}\"}}": stream_info.get("frames_per_second", 0), + f"detector_worker_stream_frame_age_seconds{{camera=\"{safe_camera_id}\"}}": stream_info.get("last_frame_age_seconds") or 0 + }) + + return { + "timestamp": time.time(), + "metrics": metrics + } + + except Exception as e: + logger.error(f"Error generating health metrics: {e}") + raise HTTPException(status_code=500, detail=f"Metrics error: {str(e)}") + + if __name__ == "__main__": diff --git a/core/monitoring/__init__.py b/core/monitoring/__init__.py new file mode 100644 index 0000000..2ad32ed --- /dev/null +++ b/core/monitoring/__init__.py @@ -0,0 +1,18 @@ +""" +Comprehensive health monitoring system for detector worker. +Tracks stream health, thread responsiveness, and system performance. +""" + +from .health import HealthMonitor, HealthStatus, HealthCheck +from .stream_health import StreamHealthTracker +from .thread_health import ThreadHealthMonitor +from .recovery import RecoveryManager + +__all__ = [ + 'HealthMonitor', + 'HealthStatus', + 'HealthCheck', + 'StreamHealthTracker', + 'ThreadHealthMonitor', + 'RecoveryManager' +] \ No newline at end of file diff --git a/core/monitoring/health.py b/core/monitoring/health.py new file mode 100644 index 0000000..be094f3 --- /dev/null +++ b/core/monitoring/health.py @@ -0,0 +1,456 @@ +""" +Core health monitoring system for comprehensive stream and system health tracking. +Provides centralized health status, alerting, and recovery coordination. +""" +import time +import threading +import logging +import psutil +from typing import Dict, List, Optional, Any, Callable +from dataclasses import dataclass, field +from enum import Enum +from collections import defaultdict, deque + + +logger = logging.getLogger(__name__) + + +class HealthStatus(Enum): + """Health status levels.""" + HEALTHY = "healthy" + WARNING = "warning" + CRITICAL = "critical" + UNKNOWN = "unknown" + + +@dataclass +class HealthCheck: + """Individual health check result.""" + name: str + status: HealthStatus + message: str + timestamp: float = field(default_factory=time.time) + details: Dict[str, Any] = field(default_factory=dict) + recovery_action: Optional[str] = None + + +@dataclass +class HealthMetrics: + """Health metrics for a component.""" + component_id: str + last_update: float + frame_count: int = 0 + error_count: int = 0 + warning_count: int = 0 + restart_count: int = 0 + avg_frame_interval: float = 0.0 + last_frame_time: Optional[float] = None + thread_alive: bool = True + connection_healthy: bool = True + memory_usage_mb: float = 0.0 + cpu_usage_percent: float = 0.0 + + +class HealthMonitor: + """Comprehensive health monitoring system.""" + + def __init__(self, check_interval: float = 30.0): + """ + Initialize health monitor. + + Args: + check_interval: Interval between health checks in seconds + """ + self.check_interval = check_interval + self.running = False + self.monitor_thread = None + self._lock = threading.RLock() + + # Health data storage + self.health_checks: Dict[str, HealthCheck] = {} + self.metrics: Dict[str, HealthMetrics] = {} + self.alert_history: deque = deque(maxlen=1000) + self.recovery_actions: deque = deque(maxlen=500) + + # Thresholds (configurable) + self.thresholds = { + 'frame_stale_warning_seconds': 120, # 2 minutes + 'frame_stale_critical_seconds': 300, # 5 minutes + 'thread_unresponsive_seconds': 60, # 1 minute + 'memory_warning_mb': 500, # 500MB per stream + 'memory_critical_mb': 1000, # 1GB per stream + 'cpu_warning_percent': 80, # 80% CPU + 'cpu_critical_percent': 95, # 95% CPU + 'error_rate_warning': 0.1, # 10% error rate + 'error_rate_critical': 0.3, # 30% error rate + 'restart_threshold': 3 # Max restarts per hour + } + + # Health check functions + self.health_checkers: List[Callable[[], List[HealthCheck]]] = [] + self.recovery_callbacks: Dict[str, Callable[[str, HealthCheck], bool]] = {} + + # System monitoring + self.process = psutil.Process() + self.system_start_time = time.time() + + def start(self): + """Start health monitoring.""" + if self.running: + logger.warning("Health monitor already running") + return + + self.running = True + self.monitor_thread = threading.Thread(target=self._monitor_loop, daemon=True) + self.monitor_thread.start() + logger.info(f"Health monitor started (check interval: {self.check_interval}s)") + + def stop(self): + """Stop health monitoring.""" + self.running = False + if self.monitor_thread: + self.monitor_thread.join(timeout=5.0) + logger.info("Health monitor stopped") + + def register_health_checker(self, checker: Callable[[], List[HealthCheck]]): + """Register a health check function.""" + self.health_checkers.append(checker) + logger.debug(f"Registered health checker: {checker.__name__}") + + def register_recovery_callback(self, component: str, callback: Callable[[str, HealthCheck], bool]): + """Register a recovery callback for a component.""" + self.recovery_callbacks[component] = callback + logger.debug(f"Registered recovery callback for {component}") + + def update_metrics(self, component_id: str, **kwargs): + """Update metrics for a component.""" + with self._lock: + if component_id not in self.metrics: + self.metrics[component_id] = HealthMetrics( + component_id=component_id, + last_update=time.time() + ) + + metrics = self.metrics[component_id] + metrics.last_update = time.time() + + # Update provided metrics + for key, value in kwargs.items(): + if hasattr(metrics, key): + setattr(metrics, key, value) + + def report_frame_received(self, component_id: str): + """Report that a frame was received for a component.""" + current_time = time.time() + with self._lock: + if component_id not in self.metrics: + self.metrics[component_id] = HealthMetrics( + component_id=component_id, + last_update=current_time + ) + + metrics = self.metrics[component_id] + + # Update frame metrics + if metrics.last_frame_time: + interval = current_time - metrics.last_frame_time + # Moving average of frame intervals + if metrics.avg_frame_interval == 0: + metrics.avg_frame_interval = interval + else: + metrics.avg_frame_interval = (metrics.avg_frame_interval * 0.9) + (interval * 0.1) + + metrics.last_frame_time = current_time + metrics.frame_count += 1 + metrics.last_update = current_time + + def report_error(self, component_id: str, error_type: str = "general"): + """Report an error for a component.""" + with self._lock: + if component_id not in self.metrics: + self.metrics[component_id] = HealthMetrics( + component_id=component_id, + last_update=time.time() + ) + + self.metrics[component_id].error_count += 1 + self.metrics[component_id].last_update = time.time() + + logger.debug(f"Error reported for {component_id}: {error_type}") + + def report_warning(self, component_id: str, warning_type: str = "general"): + """Report a warning for a component.""" + with self._lock: + if component_id not in self.metrics: + self.metrics[component_id] = HealthMetrics( + component_id=component_id, + last_update=time.time() + ) + + self.metrics[component_id].warning_count += 1 + self.metrics[component_id].last_update = time.time() + + logger.debug(f"Warning reported for {component_id}: {warning_type}") + + def report_restart(self, component_id: str): + """Report that a component was restarted.""" + with self._lock: + if component_id not in self.metrics: + self.metrics[component_id] = HealthMetrics( + component_id=component_id, + last_update=time.time() + ) + + self.metrics[component_id].restart_count += 1 + self.metrics[component_id].last_update = time.time() + + # Log recovery action + recovery_action = { + 'timestamp': time.time(), + 'component': component_id, + 'action': 'restart', + 'reason': 'manual_restart' + } + + with self._lock: + self.recovery_actions.append(recovery_action) + + logger.info(f"Restart reported for {component_id}") + + def get_health_status(self, component_id: Optional[str] = None) -> Dict[str, Any]: + """Get comprehensive health status.""" + with self._lock: + if component_id: + # Get health for specific component + return self._get_component_health(component_id) + else: + # Get overall health status + return self._get_overall_health() + + def _get_component_health(self, component_id: str) -> Dict[str, Any]: + """Get health status for a specific component.""" + if component_id not in self.metrics: + return { + 'component_id': component_id, + 'status': HealthStatus.UNKNOWN.value, + 'message': 'No metrics available', + 'metrics': {} + } + + metrics = self.metrics[component_id] + current_time = time.time() + + # Determine health status + status = HealthStatus.HEALTHY + issues = [] + + # Check frame freshness + if metrics.last_frame_time: + frame_age = current_time - metrics.last_frame_time + if frame_age > self.thresholds['frame_stale_critical_seconds']: + status = HealthStatus.CRITICAL + issues.append(f"Frames stale for {frame_age:.1f}s") + elif frame_age > self.thresholds['frame_stale_warning_seconds']: + if status == HealthStatus.HEALTHY: + status = HealthStatus.WARNING + issues.append(f"Frames aging ({frame_age:.1f}s)") + + # Check error rates + if metrics.frame_count > 0: + error_rate = metrics.error_count / metrics.frame_count + if error_rate > self.thresholds['error_rate_critical']: + status = HealthStatus.CRITICAL + issues.append(f"High error rate ({error_rate:.1%})") + elif error_rate > self.thresholds['error_rate_warning']: + if status == HealthStatus.HEALTHY: + status = HealthStatus.WARNING + issues.append(f"Elevated error rate ({error_rate:.1%})") + + # Check restart frequency + restart_rate = metrics.restart_count / max(1, (current_time - self.system_start_time) / 3600) + if restart_rate > self.thresholds['restart_threshold']: + status = HealthStatus.CRITICAL + issues.append(f"Frequent restarts ({restart_rate:.1f}/hour)") + + # Check thread health + if not metrics.thread_alive: + status = HealthStatus.CRITICAL + issues.append("Thread not alive") + + # Check connection health + if not metrics.connection_healthy: + if status == HealthStatus.HEALTHY: + status = HealthStatus.WARNING + issues.append("Connection unhealthy") + + return { + 'component_id': component_id, + 'status': status.value, + 'message': '; '.join(issues) if issues else 'All checks passing', + 'metrics': { + 'frame_count': metrics.frame_count, + 'error_count': metrics.error_count, + 'warning_count': metrics.warning_count, + 'restart_count': metrics.restart_count, + 'avg_frame_interval': metrics.avg_frame_interval, + 'last_frame_age': current_time - metrics.last_frame_time if metrics.last_frame_time else None, + 'thread_alive': metrics.thread_alive, + 'connection_healthy': metrics.connection_healthy, + 'memory_usage_mb': metrics.memory_usage_mb, + 'cpu_usage_percent': metrics.cpu_usage_percent, + 'uptime_seconds': current_time - self.system_start_time + }, + 'last_update': metrics.last_update + } + + def _get_overall_health(self) -> Dict[str, Any]: + """Get overall system health status.""" + current_time = time.time() + components = {} + overall_status = HealthStatus.HEALTHY + + # Get health for all components + for component_id in self.metrics.keys(): + component_health = self._get_component_health(component_id) + components[component_id] = component_health + + # Determine overall status + component_status = HealthStatus(component_health['status']) + if component_status == HealthStatus.CRITICAL: + overall_status = HealthStatus.CRITICAL + elif component_status == HealthStatus.WARNING and overall_status == HealthStatus.HEALTHY: + overall_status = HealthStatus.WARNING + + # System metrics + try: + system_memory = self.process.memory_info() + system_cpu = self.process.cpu_percent() + except Exception: + system_memory = None + system_cpu = 0.0 + + return { + 'overall_status': overall_status.value, + 'timestamp': current_time, + 'uptime_seconds': current_time - self.system_start_time, + 'total_components': len(self.metrics), + 'components': components, + 'system_metrics': { + 'memory_mb': system_memory.rss / (1024 * 1024) if system_memory else 0, + 'cpu_percent': system_cpu, + 'process_id': self.process.pid + }, + 'recent_alerts': list(self.alert_history)[-10:], # Last 10 alerts + 'recent_recoveries': list(self.recovery_actions)[-10:] # Last 10 recovery actions + } + + def _monitor_loop(self): + """Main health monitoring loop.""" + logger.info("Health monitor loop started") + + while self.running: + try: + start_time = time.time() + + # Run all registered health checks + all_checks = [] + for checker in self.health_checkers: + try: + checks = checker() + all_checks.extend(checks) + except Exception as e: + logger.error(f"Error in health checker {checker.__name__}: {e}") + + # Process health checks and trigger recovery if needed + for check in all_checks: + self._process_health_check(check) + + # Update system metrics + self._update_system_metrics() + + # Sleep until next check + elapsed = time.time() - start_time + sleep_time = max(0, self.check_interval - elapsed) + if sleep_time > 0: + time.sleep(sleep_time) + + except Exception as e: + logger.error(f"Error in health monitor loop: {e}") + time.sleep(5.0) # Fallback sleep + + logger.info("Health monitor loop ended") + + def _process_health_check(self, check: HealthCheck): + """Process a health check result and trigger recovery if needed.""" + with self._lock: + # Store health check + self.health_checks[check.name] = check + + # Log alerts for non-healthy status + if check.status != HealthStatus.HEALTHY: + alert = { + 'timestamp': check.timestamp, + 'component': check.name, + 'status': check.status.value, + 'message': check.message, + 'details': check.details + } + self.alert_history.append(alert) + + logger.warning(f"Health alert [{check.status.value.upper()}] {check.name}: {check.message}") + + # Trigger recovery if critical and recovery action available + if check.status == HealthStatus.CRITICAL and check.recovery_action: + self._trigger_recovery(check.name, check) + + def _trigger_recovery(self, component: str, check: HealthCheck): + """Trigger recovery action for a component.""" + if component in self.recovery_callbacks: + try: + logger.info(f"Triggering recovery for {component}: {check.recovery_action}") + + success = self.recovery_callbacks[component](component, check) + + recovery_action = { + 'timestamp': time.time(), + 'component': component, + 'action': check.recovery_action, + 'reason': check.message, + 'success': success + } + + with self._lock: + self.recovery_actions.append(recovery_action) + + if success: + logger.info(f"Recovery successful for {component}") + else: + logger.error(f"Recovery failed for {component}") + + except Exception as e: + logger.error(f"Error in recovery callback for {component}: {e}") + + def _update_system_metrics(self): + """Update system-level metrics.""" + try: + # Update process metrics for all components + current_time = time.time() + + with self._lock: + for component_id, metrics in self.metrics.items(): + # Update CPU and memory if available + try: + # This is a simplified approach - in practice you'd want + # per-thread or per-component resource tracking + metrics.cpu_usage_percent = self.process.cpu_percent() / len(self.metrics) + memory_info = self.process.memory_info() + metrics.memory_usage_mb = memory_info.rss / (1024 * 1024) / len(self.metrics) + except Exception: + pass + + except Exception as e: + logger.error(f"Error updating system metrics: {e}") + + +# Global health monitor instance +health_monitor = HealthMonitor() \ No newline at end of file diff --git a/core/monitoring/recovery.py b/core/monitoring/recovery.py new file mode 100644 index 0000000..4ea16dc --- /dev/null +++ b/core/monitoring/recovery.py @@ -0,0 +1,385 @@ +""" +Recovery manager for automatic handling of health issues. +Provides circuit breaker patterns, automatic restarts, and graceful degradation. +""" +import time +import logging +import threading +from typing import Dict, List, Optional, Any, Callable +from dataclasses import dataclass +from enum import Enum +from collections import defaultdict, deque + +from .health import HealthCheck, HealthStatus, health_monitor + + +logger = logging.getLogger(__name__) + + +class RecoveryAction(Enum): + """Types of recovery actions.""" + RESTART_STREAM = "restart_stream" + RESTART_THREAD = "restart_thread" + CLEAR_BUFFER = "clear_buffer" + RECONNECT = "reconnect" + THROTTLE = "throttle" + DISABLE = "disable" + + +@dataclass +class RecoveryAttempt: + """Record of a recovery attempt.""" + timestamp: float + component: str + action: RecoveryAction + reason: str + success: bool + details: Dict[str, Any] = None + + +@dataclass +class RecoveryState: + """Recovery state for a component - simplified without circuit breaker.""" + failure_count: int = 0 + success_count: int = 0 + last_failure_time: Optional[float] = None + last_success_time: Optional[float] = None + + +class RecoveryManager: + """Manages automatic recovery actions for health issues.""" + + def __init__(self): + self.recovery_handlers: Dict[str, Callable[[str, HealthCheck], bool]] = {} + self.recovery_states: Dict[str, RecoveryState] = {} + self.recovery_history: deque = deque(maxlen=1000) + self._lock = threading.RLock() + + # Configuration - simplified without circuit breaker + self.recovery_cooldown = 30 # 30 seconds between recovery attempts + self.max_attempts_per_hour = 20 # Still limit to prevent spam, but much higher + + # Track recovery attempts per component + self.recovery_attempts: Dict[str, deque] = defaultdict(lambda: deque(maxlen=50)) + + # Register with health monitor + health_monitor.register_recovery_callback("stream", self._handle_stream_recovery) + health_monitor.register_recovery_callback("thread", self._handle_thread_recovery) + health_monitor.register_recovery_callback("buffer", self._handle_buffer_recovery) + + def register_recovery_handler(self, action: RecoveryAction, handler: Callable[[str, Dict[str, Any]], bool]): + """ + Register a recovery handler for a specific action. + + Args: + action: Type of recovery action + handler: Function that performs the recovery + """ + self.recovery_handlers[action.value] = handler + logger.info(f"Registered recovery handler for {action.value}") + + def can_attempt_recovery(self, component: str) -> bool: + """ + Check if recovery can be attempted for a component. + + Args: + component: Component identifier + + Returns: + True if recovery can be attempted (always allow with minimal throttling) + """ + with self._lock: + current_time = time.time() + + # Check recovery attempt rate limiting (much more permissive) + recent_attempts = [ + attempt for attempt in self.recovery_attempts[component] + if current_time - attempt <= 3600 # Last hour + ] + + # Only block if truly excessive attempts + if len(recent_attempts) >= self.max_attempts_per_hour: + logger.warning(f"Recovery rate limit exceeded for {component} " + f"({len(recent_attempts)} attempts in last hour)") + return False + + # Check cooldown period (shorter cooldown) + if recent_attempts: + last_attempt = max(recent_attempts) + if current_time - last_attempt < self.recovery_cooldown: + logger.debug(f"Recovery cooldown active for {component} " + f"(last attempt {current_time - last_attempt:.1f}s ago)") + return False + + return True + + def attempt_recovery(self, component: str, action: RecoveryAction, reason: str, + details: Optional[Dict[str, Any]] = None) -> bool: + """ + Attempt recovery for a component. + + Args: + component: Component identifier + action: Recovery action to perform + reason: Reason for recovery + details: Additional details + + Returns: + True if recovery was successful + """ + if not self.can_attempt_recovery(component): + return False + + current_time = time.time() + + logger.info(f"Attempting recovery for {component}: {action.value} ({reason})") + + try: + # Record recovery attempt + with self._lock: + self.recovery_attempts[component].append(current_time) + + # Perform recovery action + success = self._execute_recovery_action(component, action, details or {}) + + # Record recovery result + attempt = RecoveryAttempt( + timestamp=current_time, + component=component, + action=action, + reason=reason, + success=success, + details=details + ) + + with self._lock: + self.recovery_history.append(attempt) + + # Update recovery state + self._update_recovery_state(component, success) + + if success: + logger.info(f"Recovery successful for {component}: {action.value}") + else: + logger.error(f"Recovery failed for {component}: {action.value}") + + return success + + except Exception as e: + logger.error(f"Error during recovery for {component}: {e}") + self._update_recovery_state(component, False) + return False + + def _execute_recovery_action(self, component: str, action: RecoveryAction, + details: Dict[str, Any]) -> bool: + """Execute a specific recovery action.""" + handler_key = action.value + + if handler_key not in self.recovery_handlers: + logger.error(f"No recovery handler registered for action: {handler_key}") + return False + + try: + handler = self.recovery_handlers[handler_key] + return handler(component, details) + + except Exception as e: + logger.error(f"Error executing recovery action {handler_key} for {component}: {e}") + return False + + def _update_recovery_state(self, component: str, success: bool): + """Update recovery state based on recovery result.""" + current_time = time.time() + + with self._lock: + if component not in self.recovery_states: + self.recovery_states[component] = RecoveryState() + + state = self.recovery_states[component] + + if success: + state.success_count += 1 + state.last_success_time = current_time + # Reset failure count on success + state.failure_count = max(0, state.failure_count - 1) + logger.debug(f"Recovery success for {component} (total successes: {state.success_count})") + else: + state.failure_count += 1 + state.last_failure_time = current_time + logger.debug(f"Recovery failure for {component} (total failures: {state.failure_count})") + + def _handle_stream_recovery(self, component: str, health_check: HealthCheck) -> bool: + """Handle recovery for stream-related issues.""" + if "frames" in health_check.name: + # Frame-related issue - restart stream + return self.attempt_recovery( + component, + RecoveryAction.RESTART_STREAM, + health_check.message, + health_check.details + ) + elif "connection" in health_check.name: + # Connection issue - reconnect + return self.attempt_recovery( + component, + RecoveryAction.RECONNECT, + health_check.message, + health_check.details + ) + elif "errors" in health_check.name: + # High error rate - throttle or restart + return self.attempt_recovery( + component, + RecoveryAction.THROTTLE, + health_check.message, + health_check.details + ) + else: + # Generic stream issue - restart + return self.attempt_recovery( + component, + RecoveryAction.RESTART_STREAM, + health_check.message, + health_check.details + ) + + def _handle_thread_recovery(self, component: str, health_check: HealthCheck) -> bool: + """Handle recovery for thread-related issues.""" + if "deadlock" in health_check.name: + # Deadlock detected - restart thread + return self.attempt_recovery( + component, + RecoveryAction.RESTART_THREAD, + health_check.message, + health_check.details + ) + elif "responsive" in health_check.name: + # Thread unresponsive - restart + return self.attempt_recovery( + component, + RecoveryAction.RESTART_THREAD, + health_check.message, + health_check.details + ) + else: + # Generic thread issue - restart + return self.attempt_recovery( + component, + RecoveryAction.RESTART_THREAD, + health_check.message, + health_check.details + ) + + def _handle_buffer_recovery(self, component: str, health_check: HealthCheck) -> bool: + """Handle recovery for buffer-related issues.""" + # Buffer issues - clear buffer + return self.attempt_recovery( + component, + RecoveryAction.CLEAR_BUFFER, + health_check.message, + health_check.details + ) + + def get_recovery_stats(self) -> Dict[str, Any]: + """Get recovery statistics.""" + current_time = time.time() + + with self._lock: + # Calculate stats from history + recent_recoveries = [ + attempt for attempt in self.recovery_history + if current_time - attempt.timestamp <= 3600 # Last hour + ] + + stats_by_component = defaultdict(lambda: { + 'attempts': 0, + 'successes': 0, + 'failures': 0, + 'last_attempt': None, + 'last_success': None + }) + + for attempt in recent_recoveries: + stats = stats_by_component[attempt.component] + stats['attempts'] += 1 + + if attempt.success: + stats['successes'] += 1 + if not stats['last_success'] or attempt.timestamp > stats['last_success']: + stats['last_success'] = attempt.timestamp + else: + stats['failures'] += 1 + + if not stats['last_attempt'] or attempt.timestamp > stats['last_attempt']: + stats['last_attempt'] = attempt.timestamp + + return { + 'total_recoveries_last_hour': len(recent_recoveries), + 'recovery_by_component': dict(stats_by_component), + 'recovery_states': { + component: { + 'failure_count': state.failure_count, + 'success_count': state.success_count, + 'last_failure_time': state.last_failure_time, + 'last_success_time': state.last_success_time + } + for component, state in self.recovery_states.items() + }, + 'recent_history': [ + { + 'timestamp': attempt.timestamp, + 'component': attempt.component, + 'action': attempt.action.value, + 'reason': attempt.reason, + 'success': attempt.success + } + for attempt in list(self.recovery_history)[-10:] # Last 10 attempts + ] + } + + def force_recovery(self, component: str, action: RecoveryAction, reason: str = "manual") -> bool: + """ + Force recovery for a component, bypassing rate limiting. + + Args: + component: Component identifier + action: Recovery action to perform + reason: Reason for forced recovery + + Returns: + True if recovery was successful + """ + logger.info(f"Forcing recovery for {component}: {action.value} ({reason})") + + current_time = time.time() + + try: + # Execute recovery action directly + success = self._execute_recovery_action(component, action, {}) + + # Record forced recovery + attempt = RecoveryAttempt( + timestamp=current_time, + component=component, + action=action, + reason=f"forced: {reason}", + success=success, + details={'forced': True} + ) + + with self._lock: + self.recovery_history.append(attempt) + self.recovery_attempts[component].append(current_time) + + # Update recovery state + self._update_recovery_state(component, success) + + return success + + except Exception as e: + logger.error(f"Error during forced recovery for {component}: {e}") + return False + + +# Global recovery manager instance +recovery_manager = RecoveryManager() \ No newline at end of file diff --git a/core/monitoring/stream_health.py b/core/monitoring/stream_health.py new file mode 100644 index 0000000..770dfe4 --- /dev/null +++ b/core/monitoring/stream_health.py @@ -0,0 +1,351 @@ +""" +Stream-specific health monitoring for video streams. +Tracks frame production, connection health, and stream-specific metrics. +""" +import time +import logging +import threading +import requests +from typing import Dict, Optional, List, Any +from collections import deque +from dataclasses import dataclass + +from .health import HealthCheck, HealthStatus, health_monitor + + +logger = logging.getLogger(__name__) + + +@dataclass +class StreamMetrics: + """Metrics for an individual stream.""" + camera_id: str + stream_type: str # 'rtsp', 'http_snapshot' + start_time: float + last_frame_time: Optional[float] = None + frame_count: int = 0 + error_count: int = 0 + reconnect_count: int = 0 + bytes_received: int = 0 + frames_per_second: float = 0.0 + connection_attempts: int = 0 + last_connection_test: Optional[float] = None + connection_healthy: bool = True + last_error: Optional[str] = None + last_error_time: Optional[float] = None + + +class StreamHealthTracker: + """Tracks health for individual video streams.""" + + def __init__(self): + self.streams: Dict[str, StreamMetrics] = {} + self._lock = threading.RLock() + + # Configuration + self.connection_test_interval = 300 # Test connection every 5 minutes + self.frame_timeout_warning = 120 # Warn if no frames for 2 minutes + self.frame_timeout_critical = 300 # Critical if no frames for 5 minutes + self.error_rate_threshold = 0.1 # 10% error rate threshold + + # Register with health monitor + health_monitor.register_health_checker(self._perform_health_checks) + + def register_stream(self, camera_id: str, stream_type: str, source_url: Optional[str] = None): + """Register a new stream for monitoring.""" + with self._lock: + if camera_id not in self.streams: + self.streams[camera_id] = StreamMetrics( + camera_id=camera_id, + stream_type=stream_type, + start_time=time.time() + ) + logger.info(f"Registered stream for monitoring: {camera_id} ({stream_type})") + + # Update health monitor metrics + health_monitor.update_metrics( + camera_id, + thread_alive=True, + connection_healthy=True + ) + + def unregister_stream(self, camera_id: str): + """Unregister a stream from monitoring.""" + with self._lock: + if camera_id in self.streams: + del self.streams[camera_id] + logger.info(f"Unregistered stream from monitoring: {camera_id}") + + def report_frame_received(self, camera_id: str, frame_size_bytes: int = 0): + """Report that a frame was received.""" + current_time = time.time() + + with self._lock: + if camera_id not in self.streams: + logger.warning(f"Frame received for unregistered stream: {camera_id}") + return + + stream = self.streams[camera_id] + + # Update frame metrics + if stream.last_frame_time: + interval = current_time - stream.last_frame_time + # Calculate FPS as moving average + if stream.frames_per_second == 0: + stream.frames_per_second = 1.0 / interval if interval > 0 else 0 + else: + new_fps = 1.0 / interval if interval > 0 else 0 + stream.frames_per_second = (stream.frames_per_second * 0.9) + (new_fps * 0.1) + + stream.last_frame_time = current_time + stream.frame_count += 1 + stream.bytes_received += frame_size_bytes + + # Report to health monitor + health_monitor.report_frame_received(camera_id) + health_monitor.update_metrics( + camera_id, + frame_count=stream.frame_count, + avg_frame_interval=1.0 / stream.frames_per_second if stream.frames_per_second > 0 else 0, + last_frame_time=current_time + ) + + def report_error(self, camera_id: str, error_message: str): + """Report an error for a stream.""" + current_time = time.time() + + with self._lock: + if camera_id not in self.streams: + logger.warning(f"Error reported for unregistered stream: {camera_id}") + return + + stream = self.streams[camera_id] + stream.error_count += 1 + stream.last_error = error_message + stream.last_error_time = current_time + + # Report to health monitor + health_monitor.report_error(camera_id, "stream_error") + health_monitor.update_metrics( + camera_id, + error_count=stream.error_count + ) + + logger.debug(f"Error reported for stream {camera_id}: {error_message}") + + def report_reconnect(self, camera_id: str, reason: str = "unknown"): + """Report that a stream reconnected.""" + current_time = time.time() + + with self._lock: + if camera_id not in self.streams: + logger.warning(f"Reconnect reported for unregistered stream: {camera_id}") + return + + stream = self.streams[camera_id] + stream.reconnect_count += 1 + + # Report to health monitor + health_monitor.report_restart(camera_id) + health_monitor.update_metrics( + camera_id, + restart_count=stream.reconnect_count + ) + + logger.info(f"Reconnect reported for stream {camera_id}: {reason}") + + def report_connection_attempt(self, camera_id: str, success: bool): + """Report a connection attempt.""" + with self._lock: + if camera_id not in self.streams: + return + + stream = self.streams[camera_id] + stream.connection_attempts += 1 + stream.connection_healthy = success + + # Report to health monitor + health_monitor.update_metrics( + camera_id, + connection_healthy=success + ) + + def test_http_connection(self, camera_id: str, url: str) -> bool: + """Test HTTP connection health for snapshot streams.""" + try: + # Quick HEAD request to test connectivity + response = requests.head(url, timeout=5, verify=False) + success = response.status_code in [200, 404] # 404 might be normal for some cameras + + self.report_connection_attempt(camera_id, success) + + if success: + logger.debug(f"Connection test passed for {camera_id}") + else: + logger.warning(f"Connection test failed for {camera_id}: HTTP {response.status_code}") + + return success + + except Exception as e: + logger.warning(f"Connection test failed for {camera_id}: {e}") + self.report_connection_attempt(camera_id, False) + return False + + def get_stream_metrics(self, camera_id: str) -> Optional[Dict[str, Any]]: + """Get metrics for a specific stream.""" + with self._lock: + if camera_id not in self.streams: + return None + + stream = self.streams[camera_id] + current_time = time.time() + + # Calculate derived metrics + uptime = current_time - stream.start_time + frame_age = current_time - stream.last_frame_time if stream.last_frame_time else None + error_rate = stream.error_count / max(1, stream.frame_count) + + return { + 'camera_id': camera_id, + 'stream_type': stream.stream_type, + 'uptime_seconds': uptime, + 'frame_count': stream.frame_count, + 'frames_per_second': stream.frames_per_second, + 'bytes_received': stream.bytes_received, + 'error_count': stream.error_count, + 'error_rate': error_rate, + 'reconnect_count': stream.reconnect_count, + 'connection_attempts': stream.connection_attempts, + 'connection_healthy': stream.connection_healthy, + 'last_frame_age_seconds': frame_age, + 'last_error': stream.last_error, + 'last_error_time': stream.last_error_time + } + + def get_all_metrics(self) -> Dict[str, Dict[str, Any]]: + """Get metrics for all streams.""" + with self._lock: + return { + camera_id: self.get_stream_metrics(camera_id) + for camera_id in self.streams.keys() + } + + def _perform_health_checks(self) -> List[HealthCheck]: + """Perform health checks for all streams.""" + checks = [] + current_time = time.time() + + with self._lock: + for camera_id, stream in self.streams.items(): + checks.extend(self._check_stream_health(camera_id, stream, current_time)) + + return checks + + def _check_stream_health(self, camera_id: str, stream: StreamMetrics, current_time: float) -> List[HealthCheck]: + """Perform health checks for a single stream.""" + checks = [] + + # Check frame freshness + if stream.last_frame_time: + frame_age = current_time - stream.last_frame_time + + if frame_age > self.frame_timeout_critical: + checks.append(HealthCheck( + name=f"stream_{camera_id}_frames", + status=HealthStatus.CRITICAL, + message=f"No frames for {frame_age:.1f}s (critical threshold: {self.frame_timeout_critical}s)", + details={ + 'frame_age': frame_age, + 'threshold': self.frame_timeout_critical, + 'last_frame_time': stream.last_frame_time + }, + recovery_action="restart_stream" + )) + elif frame_age > self.frame_timeout_warning: + checks.append(HealthCheck( + name=f"stream_{camera_id}_frames", + status=HealthStatus.WARNING, + message=f"Frames aging: {frame_age:.1f}s (warning threshold: {self.frame_timeout_warning}s)", + details={ + 'frame_age': frame_age, + 'threshold': self.frame_timeout_warning, + 'last_frame_time': stream.last_frame_time + } + )) + else: + # No frames received yet + startup_time = current_time - stream.start_time + if startup_time > 60: # Allow 1 minute for initial connection + checks.append(HealthCheck( + name=f"stream_{camera_id}_startup", + status=HealthStatus.CRITICAL, + message=f"No frames received since startup {startup_time:.1f}s ago", + details={ + 'startup_time': startup_time, + 'start_time': stream.start_time + }, + recovery_action="restart_stream" + )) + + # Check error rate + if stream.frame_count > 10: # Need sufficient samples + error_rate = stream.error_count / stream.frame_count + if error_rate > self.error_rate_threshold: + checks.append(HealthCheck( + name=f"stream_{camera_id}_errors", + status=HealthStatus.WARNING, + message=f"High error rate: {error_rate:.1%} ({stream.error_count}/{stream.frame_count})", + details={ + 'error_rate': error_rate, + 'error_count': stream.error_count, + 'frame_count': stream.frame_count, + 'last_error': stream.last_error + } + )) + + # Check connection health + if not stream.connection_healthy: + checks.append(HealthCheck( + name=f"stream_{camera_id}_connection", + status=HealthStatus.WARNING, + message="Connection unhealthy (last test failed)", + details={ + 'connection_attempts': stream.connection_attempts, + 'last_connection_test': stream.last_connection_test + } + )) + + # Check excessive reconnects + uptime_hours = (current_time - stream.start_time) / 3600 + if uptime_hours > 1 and stream.reconnect_count > 5: # More than 5 reconnects per hour + reconnect_rate = stream.reconnect_count / uptime_hours + checks.append(HealthCheck( + name=f"stream_{camera_id}_stability", + status=HealthStatus.WARNING, + message=f"Frequent reconnects: {reconnect_rate:.1f}/hour ({stream.reconnect_count} total)", + details={ + 'reconnect_rate': reconnect_rate, + 'reconnect_count': stream.reconnect_count, + 'uptime_hours': uptime_hours + } + )) + + # Check frame rate health + if stream.last_frame_time and stream.frames_per_second > 0: + expected_fps = 6.0 # Expected FPS for streams + if stream.frames_per_second < expected_fps * 0.5: # Less than 50% of expected + checks.append(HealthCheck( + name=f"stream_{camera_id}_framerate", + status=HealthStatus.WARNING, + message=f"Low frame rate: {stream.frames_per_second:.1f} fps (expected: ~{expected_fps} fps)", + details={ + 'current_fps': stream.frames_per_second, + 'expected_fps': expected_fps + } + )) + + return checks + + +# Global stream health tracker instance +stream_health_tracker = StreamHealthTracker() \ No newline at end of file diff --git a/core/monitoring/thread_health.py b/core/monitoring/thread_health.py new file mode 100644 index 0000000..a29625b --- /dev/null +++ b/core/monitoring/thread_health.py @@ -0,0 +1,381 @@ +""" +Thread health monitoring for detecting unresponsive and deadlocked threads. +Provides thread liveness detection and responsiveness testing. +""" +import time +import threading +import logging +import signal +import traceback +from typing import Dict, List, Optional, Any, Callable +from dataclasses import dataclass +from collections import defaultdict + +from .health import HealthCheck, HealthStatus, health_monitor + + +logger = logging.getLogger(__name__) + + +@dataclass +class ThreadInfo: + """Information about a monitored thread.""" + thread_id: int + thread_name: str + start_time: float + last_heartbeat: float + heartbeat_count: int = 0 + is_responsive: bool = True + last_activity: Optional[str] = None + stack_traces: List[str] = None + + +class ThreadHealthMonitor: + """Monitors thread health and responsiveness.""" + + def __init__(self): + self.monitored_threads: Dict[int, ThreadInfo] = {} + self.heartbeat_callbacks: Dict[int, Callable[[], bool]] = {} + self._lock = threading.RLock() + + # Configuration + self.heartbeat_timeout = 60.0 # 1 minute without heartbeat = unresponsive + self.responsiveness_test_interval = 30.0 # Test responsiveness every 30 seconds + self.stack_trace_count = 5 # Keep last 5 stack traces for analysis + + # Register with health monitor + health_monitor.register_health_checker(self._perform_health_checks) + + # Enable periodic responsiveness testing + self.test_thread = threading.Thread(target=self._responsiveness_test_loop, daemon=True) + self.test_thread.start() + + def register_thread(self, thread: threading.Thread, heartbeat_callback: Optional[Callable[[], bool]] = None): + """ + Register a thread for monitoring. + + Args: + thread: Thread to monitor + heartbeat_callback: Optional callback to test thread responsiveness + """ + with self._lock: + thread_info = ThreadInfo( + thread_id=thread.ident, + thread_name=thread.name, + start_time=time.time(), + last_heartbeat=time.time() + ) + + self.monitored_threads[thread.ident] = thread_info + + if heartbeat_callback: + self.heartbeat_callbacks[thread.ident] = heartbeat_callback + + logger.info(f"Registered thread for monitoring: {thread.name} (ID: {thread.ident})") + + def unregister_thread(self, thread_id: int): + """Unregister a thread from monitoring.""" + with self._lock: + if thread_id in self.monitored_threads: + thread_name = self.monitored_threads[thread_id].thread_name + del self.monitored_threads[thread_id] + + if thread_id in self.heartbeat_callbacks: + del self.heartbeat_callbacks[thread_id] + + logger.info(f"Unregistered thread from monitoring: {thread_name} (ID: {thread_id})") + + def heartbeat(self, thread_id: Optional[int] = None, activity: Optional[str] = None): + """ + Report thread heartbeat. + + Args: + thread_id: Thread ID (uses current thread if None) + activity: Description of current activity + """ + if thread_id is None: + thread_id = threading.current_thread().ident + + current_time = time.time() + + with self._lock: + if thread_id in self.monitored_threads: + thread_info = self.monitored_threads[thread_id] + thread_info.last_heartbeat = current_time + thread_info.heartbeat_count += 1 + thread_info.is_responsive = True + + if activity: + thread_info.last_activity = activity + + # Report to health monitor + health_monitor.update_metrics( + f"thread_{thread_info.thread_name}", + thread_alive=True, + last_frame_time=current_time + ) + + def get_thread_info(self, thread_id: int) -> Optional[Dict[str, Any]]: + """Get information about a monitored thread.""" + with self._lock: + if thread_id not in self.monitored_threads: + return None + + thread_info = self.monitored_threads[thread_id] + current_time = time.time() + + return { + 'thread_id': thread_id, + 'thread_name': thread_info.thread_name, + 'uptime_seconds': current_time - thread_info.start_time, + 'last_heartbeat_age': current_time - thread_info.last_heartbeat, + 'heartbeat_count': thread_info.heartbeat_count, + 'is_responsive': thread_info.is_responsive, + 'last_activity': thread_info.last_activity, + 'stack_traces': thread_info.stack_traces or [] + } + + def get_all_thread_info(self) -> Dict[int, Dict[str, Any]]: + """Get information about all monitored threads.""" + with self._lock: + return { + thread_id: self.get_thread_info(thread_id) + for thread_id in self.monitored_threads.keys() + } + + def test_thread_responsiveness(self, thread_id: int) -> bool: + """ + Test if a thread is responsive by calling its heartbeat callback. + + Args: + thread_id: ID of thread to test + + Returns: + True if thread responds within timeout + """ + if thread_id not in self.heartbeat_callbacks: + return True # Can't test if no callback provided + + try: + # Call the heartbeat callback with a timeout + callback = self.heartbeat_callbacks[thread_id] + + # This is a simple approach - in practice you might want to use + # threading.Timer or asyncio for more sophisticated timeout handling + start_time = time.time() + result = callback() + response_time = time.time() - start_time + + with self._lock: + if thread_id in self.monitored_threads: + self.monitored_threads[thread_id].is_responsive = result + + if response_time > 5.0: # Slow response + logger.warning(f"Thread {thread_id} slow response: {response_time:.1f}s") + + return result + + except Exception as e: + logger.error(f"Error testing thread {thread_id} responsiveness: {e}") + with self._lock: + if thread_id in self.monitored_threads: + self.monitored_threads[thread_id].is_responsive = False + return False + + def capture_stack_trace(self, thread_id: int) -> Optional[str]: + """ + Capture stack trace for a thread. + + Args: + thread_id: ID of thread to capture + + Returns: + Stack trace string or None if not available + """ + try: + # Get all frames for all threads + frames = dict(threading._current_frames()) + + if thread_id not in frames: + return None + + # Format stack trace + frame = frames[thread_id] + stack_trace = ''.join(traceback.format_stack(frame)) + + # Store in thread info + with self._lock: + if thread_id in self.monitored_threads: + thread_info = self.monitored_threads[thread_id] + if thread_info.stack_traces is None: + thread_info.stack_traces = [] + + thread_info.stack_traces.append(f"{time.time()}: {stack_trace}") + + # Keep only last N stack traces + if len(thread_info.stack_traces) > self.stack_trace_count: + thread_info.stack_traces = thread_info.stack_traces[-self.stack_trace_count:] + + return stack_trace + + except Exception as e: + logger.error(f"Error capturing stack trace for thread {thread_id}: {e}") + return None + + def detect_deadlocks(self) -> List[Dict[str, Any]]: + """ + Attempt to detect potential deadlocks by analyzing thread states. + + Returns: + List of potential deadlock scenarios + """ + deadlocks = [] + current_time = time.time() + + with self._lock: + # Look for threads that haven't had heartbeats for a long time + # and are supposedly alive + for thread_id, thread_info in self.monitored_threads.items(): + heartbeat_age = current_time - thread_info.last_heartbeat + + if heartbeat_age > self.heartbeat_timeout * 2: # Double the timeout + # Check if thread still exists + thread_exists = any( + t.ident == thread_id and t.is_alive() + for t in threading.enumerate() + ) + + if thread_exists: + # Thread exists but not responding - potential deadlock + stack_trace = self.capture_stack_trace(thread_id) + + deadlock_info = { + 'thread_id': thread_id, + 'thread_name': thread_info.thread_name, + 'heartbeat_age': heartbeat_age, + 'last_activity': thread_info.last_activity, + 'stack_trace': stack_trace, + 'detection_time': current_time + } + + deadlocks.append(deadlock_info) + logger.warning(f"Potential deadlock detected in thread {thread_info.thread_name}") + + return deadlocks + + def _responsiveness_test_loop(self): + """Background loop to test thread responsiveness.""" + logger.info("Thread responsiveness testing started") + + while True: + try: + time.sleep(self.responsiveness_test_interval) + + with self._lock: + thread_ids = list(self.monitored_threads.keys()) + + for thread_id in thread_ids: + try: + self.test_thread_responsiveness(thread_id) + except Exception as e: + logger.error(f"Error testing thread {thread_id}: {e}") + + except Exception as e: + logger.error(f"Error in responsiveness test loop: {e}") + time.sleep(10.0) # Fallback sleep + + def _perform_health_checks(self) -> List[HealthCheck]: + """Perform health checks for all monitored threads.""" + checks = [] + current_time = time.time() + + with self._lock: + for thread_id, thread_info in self.monitored_threads.items(): + checks.extend(self._check_thread_health(thread_id, thread_info, current_time)) + + # Check for deadlocks + deadlocks = self.detect_deadlocks() + for deadlock in deadlocks: + checks.append(HealthCheck( + name=f"deadlock_detection_{deadlock['thread_id']}", + status=HealthStatus.CRITICAL, + message=f"Potential deadlock in thread {deadlock['thread_name']} " + f"(unresponsive for {deadlock['heartbeat_age']:.1f}s)", + details=deadlock, + recovery_action="restart_thread" + )) + + return checks + + def _check_thread_health(self, thread_id: int, thread_info: ThreadInfo, current_time: float) -> List[HealthCheck]: + """Perform health checks for a single thread.""" + checks = [] + + # Check if thread still exists + thread_exists = any( + t.ident == thread_id and t.is_alive() + for t in threading.enumerate() + ) + + if not thread_exists: + checks.append(HealthCheck( + name=f"thread_{thread_info.thread_name}_alive", + status=HealthStatus.CRITICAL, + message=f"Thread {thread_info.thread_name} is no longer alive", + details={ + 'thread_id': thread_id, + 'uptime': current_time - thread_info.start_time, + 'last_heartbeat': thread_info.last_heartbeat + }, + recovery_action="restart_thread" + )) + return checks + + # Check heartbeat freshness + heartbeat_age = current_time - thread_info.last_heartbeat + + if heartbeat_age > self.heartbeat_timeout: + checks.append(HealthCheck( + name=f"thread_{thread_info.thread_name}_responsive", + status=HealthStatus.CRITICAL, + message=f"Thread {thread_info.thread_name} unresponsive for {heartbeat_age:.1f}s", + details={ + 'thread_id': thread_id, + 'heartbeat_age': heartbeat_age, + 'heartbeat_count': thread_info.heartbeat_count, + 'last_activity': thread_info.last_activity, + 'is_responsive': thread_info.is_responsive + }, + recovery_action="restart_thread" + )) + elif heartbeat_age > self.heartbeat_timeout * 0.5: # Warning at 50% of timeout + checks.append(HealthCheck( + name=f"thread_{thread_info.thread_name}_responsive", + status=HealthStatus.WARNING, + message=f"Thread {thread_info.thread_name} slow heartbeat: {heartbeat_age:.1f}s", + details={ + 'thread_id': thread_id, + 'heartbeat_age': heartbeat_age, + 'heartbeat_count': thread_info.heartbeat_count, + 'last_activity': thread_info.last_activity, + 'is_responsive': thread_info.is_responsive + } + )) + + # Check responsiveness test results + if not thread_info.is_responsive: + checks.append(HealthCheck( + name=f"thread_{thread_info.thread_name}_callback", + status=HealthStatus.WARNING, + message=f"Thread {thread_info.thread_name} failed responsiveness test", + details={ + 'thread_id': thread_id, + 'last_activity': thread_info.last_activity + } + )) + + return checks + + +# Global thread health monitor instance +thread_health_monitor = ThreadHealthMonitor() \ No newline at end of file diff --git a/core/streaming/readers/ffmpeg_rtsp.py b/core/streaming/readers/ffmpeg_rtsp.py index 8641495..f2fb8d1 100644 --- a/core/streaming/readers/ffmpeg_rtsp.py +++ b/core/streaming/readers/ffmpeg_rtsp.py @@ -1,5 +1,6 @@ """ FFmpeg RTSP stream reader using subprocess piping frames directly to buffer. +Enhanced with comprehensive health monitoring and automatic recovery. """ import cv2 import time @@ -7,10 +8,13 @@ import threading import numpy as np import subprocess import struct -from typing import Optional, Callable +from typing import Optional, Callable, Dict, Any from .base import VideoReader from .utils import log_success, log_warning, log_error, log_info +from ..monitoring.stream_health import stream_health_tracker +from ..monitoring.thread_health import thread_health_monitor +from ..monitoring.recovery import recovery_manager, RecoveryAction class FFmpegRTSPReader(VideoReader): @@ -35,6 +39,21 @@ class FFmpegRTSPReader(VideoReader): self.first_start_timeout = 30.0 # 30s timeout on first start self.restart_timeout = 15.0 # 15s timeout after restart + # Health monitoring setup + self.last_heartbeat = time.time() + self.consecutive_errors = 0 + self.ffmpeg_restart_count = 0 + + # Register recovery handlers + recovery_manager.register_recovery_handler( + RecoveryAction.RESTART_STREAM, + self._handle_restart_recovery + ) + recovery_manager.register_recovery_handler( + RecoveryAction.RECONNECT, + self._handle_reconnect_recovery + ) + @property def is_running(self) -> bool: """Check if the reader is currently running.""" @@ -58,21 +77,35 @@ class FFmpegRTSPReader(VideoReader): self.stop_event.clear() self.thread = threading.Thread(target=self._read_frames, daemon=True) self.thread.start() - log_success(self.camera_id, "Stream started") + + # Register with health monitoring + stream_health_tracker.register_stream(self.camera_id, "rtsp_ffmpeg", self.rtsp_url) + thread_health_monitor.register_thread(self.thread, self._heartbeat_callback) + + log_success(self.camera_id, "Stream started with health monitoring") def stop(self): """Stop the FFmpeg subprocess reader.""" self.stop_event.set() + + # Unregister from health monitoring + if self.thread: + thread_health_monitor.unregister_thread(self.thread.ident) + if self.process: self.process.terminate() try: self.process.wait(timeout=5) except subprocess.TimeoutExpired: self.process.kill() + if self.thread: self.thread.join(timeout=5.0) if self.stderr_thread: self.stderr_thread.join(timeout=2.0) + + stream_health_tracker.unregister_stream(self.camera_id) + log_info(self.camera_id, "Stream stopped") def _start_ffmpeg_process(self): @@ -249,6 +282,9 @@ class FFmpegRTSPReader(VideoReader): while not self.stop_event.is_set(): try: + # Send heartbeat for thread health monitoring + self._send_heartbeat("reading_frames") + # Check watchdog timeout if process is running if self.process and self.process.poll() is None: if self._check_watchdog_timeout(): @@ -259,8 +295,17 @@ class FFmpegRTSPReader(VideoReader): if not self.process or self.process.poll() is not None: if self.process and self.process.poll() is not None: log_warning(self.camera_id, "Stream disconnected, reconnecting...") + stream_health_tracker.report_error( + self.camera_id, + "FFmpeg process disconnected" + ) if not self._start_ffmpeg_process(): + self.consecutive_errors += 1 + stream_health_tracker.report_error( + self.camera_id, + "Failed to start FFmpeg process" + ) time.sleep(5.0) continue @@ -275,9 +320,22 @@ class FFmpegRTSPReader(VideoReader): # Update watchdog - we got a frame self.last_frame_time = time.time() + # Reset error counter on successful frame + self.consecutive_errors = 0 + + # Report successful frame to health monitoring + frame_size = frame.nbytes + stream_health_tracker.report_frame_received(self.camera_id, frame_size) + # Call frame callback if self.frame_callback: - self.frame_callback(self.camera_id, frame) + try: + self.frame_callback(self.camera_id, frame) + except Exception as e: + stream_health_tracker.report_error( + self.camera_id, + f"Frame callback error: {e}" + ) frame_count += 1 @@ -287,16 +345,85 @@ class FFmpegRTSPReader(VideoReader): log_success(self.camera_id, f"{frame_count} frames captured ({frame.shape[1]}x{frame.shape[0]})") last_log_time = current_time - except Exception: + except Exception as e: # Process might have died, let it restart on next iteration + stream_health_tracker.report_error( + self.camera_id, + f"Frame reading error: {e}" + ) if self.process: self.process.terminate() self.process = None time.sleep(1.0) - except Exception: + except Exception as e: + stream_health_tracker.report_error( + self.camera_id, + f"Main loop error: {e}" + ) time.sleep(1.0) # Cleanup if self.process: - self.process.terminate() \ No newline at end of file + self.process.terminate() + + # Health monitoring methods + def _send_heartbeat(self, activity: str = "running"): + """Send heartbeat to thread health monitor.""" + self.last_heartbeat = time.time() + thread_health_monitor.heartbeat(activity=activity) + + def _heartbeat_callback(self) -> bool: + """Heartbeat callback for thread responsiveness testing.""" + try: + # Check if thread is responsive by checking recent heartbeat + current_time = time.time() + age = current_time - self.last_heartbeat + + # Thread is responsive if heartbeat is recent + return age < 30.0 # 30 second responsiveness threshold + + except Exception: + return False + + def _handle_restart_recovery(self, component: str, details: Dict[str, Any]) -> bool: + """Handle restart recovery action.""" + try: + log_info(self.camera_id, "Restarting FFmpeg RTSP reader for health recovery") + + # Stop current instance + self.stop() + + # Small delay + time.sleep(2.0) + + # Restart + self.start() + + # Report successful restart + stream_health_tracker.report_reconnect(self.camera_id, "health_recovery_restart") + self.ffmpeg_restart_count += 1 + + return True + + except Exception as e: + log_error(self.camera_id, f"Failed to restart FFmpeg RTSP reader: {e}") + return False + + def _handle_reconnect_recovery(self, component: str, details: Dict[str, Any]) -> bool: + """Handle reconnect recovery action.""" + try: + log_info(self.camera_id, "Reconnecting FFmpeg RTSP reader for health recovery") + + # Force restart FFmpeg process + self._restart_ffmpeg_process() + + # Reset error counters + self.consecutive_errors = 0 + stream_health_tracker.report_reconnect(self.camera_id, "health_recovery_reconnect") + + return True + + except Exception as e: + log_error(self.camera_id, f"Failed to reconnect FFmpeg RTSP reader: {e}") + return False \ No newline at end of file diff --git a/core/streaming/readers/http_snapshot.py b/core/streaming/readers/http_snapshot.py index 5a479db..1aab967 100644 --- a/core/streaming/readers/http_snapshot.py +++ b/core/streaming/readers/http_snapshot.py @@ -1,5 +1,6 @@ """ HTTP snapshot reader optimized for 2560x1440 (2K) high quality images. +Enhanced with comprehensive health monitoring and automatic recovery. """ import cv2 import logging @@ -7,10 +8,13 @@ import time import threading import requests import numpy as np -from typing import Optional, Callable +from typing import Optional, Callable, Dict, Any from .base import VideoReader from .utils import log_success, log_warning, log_error, log_info +from ..monitoring.stream_health import stream_health_tracker +from ..monitoring.thread_health import thread_health_monitor +from ..monitoring.recovery import recovery_manager, RecoveryAction logger = logging.getLogger(__name__) @@ -30,6 +34,22 @@ class HTTPSnapshotReader(VideoReader): self.expected_height = 1440 self.max_file_size = 10 * 1024 * 1024 # 10MB max for 2K image + # Health monitoring setup + self.last_heartbeat = time.time() + self.consecutive_errors = 0 + self.connection_test_interval = 300 # Test connection every 5 minutes + self.last_connection_test = None + + # Register recovery handlers + recovery_manager.register_recovery_handler( + RecoveryAction.RESTART_STREAM, + self._handle_restart_recovery + ) + recovery_manager.register_recovery_handler( + RecoveryAction.RECONNECT, + self._handle_reconnect_recovery + ) + @property def is_running(self) -> bool: """Check if the reader is currently running.""" @@ -53,13 +73,24 @@ class HTTPSnapshotReader(VideoReader): self.stop_event.clear() self.thread = threading.Thread(target=self._read_snapshots, daemon=True) self.thread.start() - logger.info(f"Started snapshot reader for camera {self.camera_id}") + + # Register with health monitoring + stream_health_tracker.register_stream(self.camera_id, "http_snapshot", self.snapshot_url) + thread_health_monitor.register_thread(self.thread, self._heartbeat_callback) + + logger.info(f"Started snapshot reader for camera {self.camera_id} with health monitoring") def stop(self): """Stop the snapshot reader thread.""" self.stop_event.set() + + # Unregister from health monitoring if self.thread: + thread_health_monitor.unregister_thread(self.thread.ident) self.thread.join(timeout=5.0) + + stream_health_tracker.unregister_stream(self.camera_id) + logger.info(f"Stopped snapshot reader for camera {self.camera_id}") def _read_snapshots(self): @@ -67,17 +98,29 @@ class HTTPSnapshotReader(VideoReader): retries = 0 frame_count = 0 last_log_time = time.time() + last_connection_test = time.time() interval_seconds = self.interval_ms / 1000.0 logger.info(f"Snapshot interval for camera {self.camera_id}: {interval_seconds}s") while not self.stop_event.is_set(): try: + # Send heartbeat for thread health monitoring + self._send_heartbeat("fetching_snapshot") + start_time = time.time() frame = self._fetch_snapshot() if frame is None: retries += 1 + self.consecutive_errors += 1 + + # Report error to health monitoring + stream_health_tracker.report_error( + self.camera_id, + f"Failed to fetch snapshot (retry {retries}/{self.max_retries})" + ) + logger.warning(f"Failed to fetch snapshot for camera {self.camera_id}, retry {retries}/{self.max_retries}") if self.max_retries != -1 and retries > self.max_retries: @@ -90,21 +133,36 @@ class HTTPSnapshotReader(VideoReader): # Accept any valid image dimensions - don't force specific resolution if frame.shape[1] <= 0 or frame.shape[0] <= 0: logger.warning(f"Camera {self.camera_id}: Invalid frame dimensions {frame.shape[1]}x{frame.shape[0]}") + stream_health_tracker.report_error( + self.camera_id, + f"Invalid frame dimensions: {frame.shape[1]}x{frame.shape[0]}" + ) continue # Reset retry counter on successful fetch retries = 0 + self.consecutive_errors = 0 frame_count += 1 + # Report successful frame to health monitoring + frame_size = frame.nbytes + stream_health_tracker.report_frame_received(self.camera_id, frame_size) + # Call frame callback if self.frame_callback: try: self.frame_callback(self.camera_id, frame) except Exception as e: logger.error(f"Camera {self.camera_id}: Frame callback error: {e}") + stream_health_tracker.report_error(self.camera_id, f"Frame callback error: {e}") + + # Periodic connection health test + current_time = time.time() + if current_time - last_connection_test >= self.connection_test_interval: + self._test_connection_health() + last_connection_test = current_time # Log progress every 30 seconds - current_time = time.time() if current_time - last_log_time >= 30: logger.info(f"Camera {self.camera_id}: {frame_count} snapshots processed") last_log_time = current_time @@ -117,6 +175,7 @@ class HTTPSnapshotReader(VideoReader): except Exception as e: logger.error(f"Error in snapshot loop for camera {self.camera_id}: {e}") + stream_health_tracker.report_error(self.camera_id, f"Snapshot loop error: {e}") retries += 1 if self.max_retries != -1 and retries > self.max_retries: break @@ -246,4 +305,74 @@ class HTTPSnapshotReader(VideoReader): right = target_width - new_width - left resized = cv2.copyMakeBorder(resized, top, bottom, left, right, cv2.BORDER_CONSTANT, value=[0, 0, 0]) - return resized \ No newline at end of file + return resized + + # Health monitoring methods + def _send_heartbeat(self, activity: str = "running"): + """Send heartbeat to thread health monitor.""" + self.last_heartbeat = time.time() + thread_health_monitor.heartbeat(activity=activity) + + def _heartbeat_callback(self) -> bool: + """Heartbeat callback for thread responsiveness testing.""" + try: + # Check if thread is responsive by checking recent heartbeat + current_time = time.time() + age = current_time - self.last_heartbeat + + # Thread is responsive if heartbeat is recent + return age < 30.0 # 30 second responsiveness threshold + + except Exception: + return False + + def _test_connection_health(self): + """Test HTTP connection health.""" + try: + stream_health_tracker.test_http_connection(self.camera_id, self.snapshot_url) + except Exception as e: + logger.error(f"Error testing connection health for {self.camera_id}: {e}") + + def _handle_restart_recovery(self, component: str, details: Dict[str, Any]) -> bool: + """Handle restart recovery action.""" + try: + logger.info(f"Restarting HTTP snapshot reader for {self.camera_id}") + + # Stop current instance + self.stop() + + # Small delay + time.sleep(2.0) + + # Restart + self.start() + + # Report successful restart + stream_health_tracker.report_reconnect(self.camera_id, "health_recovery_restart") + + return True + + except Exception as e: + logger.error(f"Failed to restart HTTP snapshot reader for {self.camera_id}: {e}") + return False + + def _handle_reconnect_recovery(self, component: str, details: Dict[str, Any]) -> bool: + """Handle reconnect recovery action.""" + try: + logger.info(f"Reconnecting HTTP snapshot reader for {self.camera_id}") + + # Test connection first + success = stream_health_tracker.test_http_connection(self.camera_id, self.snapshot_url) + + if success: + # Reset error counters + self.consecutive_errors = 0 + stream_health_tracker.report_reconnect(self.camera_id, "health_recovery_reconnect") + return True + else: + logger.warning(f"Connection test failed during recovery for {self.camera_id}") + return False + + except Exception as e: + logger.error(f"Failed to reconnect HTTP snapshot reader for {self.camera_id}: {e}") + return False \ No newline at end of file From eb57de02c37300d57100924596eaf42c794e5a08 Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Sat, 27 Sep 2025 14:57:20 +0700 Subject: [PATCH 42/55] fix: update import paths for monitoring modules in FFmpegRTSPReader and HTTPSnapshotReader --- core/streaming/readers/ffmpeg_rtsp.py | 6 +++--- core/streaming/readers/http_snapshot.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/core/streaming/readers/ffmpeg_rtsp.py b/core/streaming/readers/ffmpeg_rtsp.py index f2fb8d1..7c453f3 100644 --- a/core/streaming/readers/ffmpeg_rtsp.py +++ b/core/streaming/readers/ffmpeg_rtsp.py @@ -12,9 +12,9 @@ from typing import Optional, Callable, Dict, Any from .base import VideoReader from .utils import log_success, log_warning, log_error, log_info -from ..monitoring.stream_health import stream_health_tracker -from ..monitoring.thread_health import thread_health_monitor -from ..monitoring.recovery import recovery_manager, RecoveryAction +from ...monitoring.stream_health import stream_health_tracker +from ...monitoring.thread_health import thread_health_monitor +from ...monitoring.recovery import recovery_manager, RecoveryAction class FFmpegRTSPReader(VideoReader): diff --git a/core/streaming/readers/http_snapshot.py b/core/streaming/readers/http_snapshot.py index 1aab967..bbbf943 100644 --- a/core/streaming/readers/http_snapshot.py +++ b/core/streaming/readers/http_snapshot.py @@ -12,9 +12,9 @@ from typing import Optional, Callable, Dict, Any from .base import VideoReader from .utils import log_success, log_warning, log_error, log_info -from ..monitoring.stream_health import stream_health_tracker -from ..monitoring.thread_health import thread_health_monitor -from ..monitoring.recovery import recovery_manager, RecoveryAction +from ...monitoring.stream_health import stream_health_tracker +from ...monitoring.thread_health import thread_health_monitor +from ...monitoring.recovery import recovery_manager, RecoveryAction logger = logging.getLogger(__name__) From 52ba1ff316fb784102fd0937629f1d704823491d Mon Sep 17 00:00:00 2001 From: ziesorx Date: Mon, 29 Sep 2025 17:43:30 +0700 Subject: [PATCH 43/55] fix: sessionId type mismatch --- core/communication/websocket.py | 2 +- core/streaming/manager.py | 2 ++ core/tracking/integration.py | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/core/communication/websocket.py b/core/communication/websocket.py index 4e40d2a..e53096a 100644 --- a/core/communication/websocket.py +++ b/core/communication/websocket.py @@ -539,7 +539,7 @@ class WebSocketHandler: async def _handle_set_session_id(self, message: SetSessionIdMessage) -> None: """Handle setSessionId message.""" display_identifier = message.payload.displayIdentifier - session_id = message.payload.sessionId + session_id = str(message.payload.sessionId) if message.payload.sessionId is not None else None logger.info(f"[RX Processing] setSessionId for display {display_identifier}: {session_id}") diff --git a/core/streaming/manager.py b/core/streaming/manager.py index 5b4637c..e2f02d9 100644 --- a/core/streaming/manager.py +++ b/core/streaming/manager.py @@ -380,6 +380,8 @@ class StreamManager: def set_session_id(self, display_id: str, session_id: str): """Set session ID for tracking integration.""" + # Ensure session_id is always a string for consistent type handling + session_id = str(session_id) if session_id is not None else None with self._lock: for subscription_info in self._subscriptions.values(): # Check if this subscription matches the display_id diff --git a/core/tracking/integration.py b/core/tracking/integration.py index 3f1ebe0..8c96750 100644 --- a/core/tracking/integration.py +++ b/core/tracking/integration.py @@ -474,6 +474,8 @@ class TrackingPipelineIntegration: display_id: Display identifier session_id: Session identifier """ + # Ensure session_id is always a string for consistent type handling + session_id = str(session_id) if session_id is not None else None self.active_sessions[display_id] = session_id logger.info(f"Set session {session_id} for display {display_id}") From ee484b4655c0d5e89fa7a351187d4331ff647973 Mon Sep 17 00:00:00 2001 From: ziesorx Date: Mon, 29 Sep 2025 23:45:20 +0700 Subject: [PATCH 44/55] feat: add min bbox for frontal tracking --- core/tracking/integration.py | 60 +++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/core/tracking/integration.py b/core/tracking/integration.py index 8c96750..d1401ef 100644 --- a/core/tracking/integration.py +++ b/core/tracking/integration.py @@ -71,12 +71,17 @@ class TrackingPipelineIntegration: # Thread pool for pipeline execution self.executor = ThreadPoolExecutor(max_workers=2) + # Min bbox filtering configuration + # TODO: Make this configurable via pipeline.json in the future + self.min_bbox_area_percentage = 4.5 # 4.5% of frame area minimum + # Statistics self.stats = { 'frames_processed': 0, 'vehicles_detected': 0, 'vehicles_validated': 0, - 'pipelines_executed': 0 + 'pipelines_executed': 0, + 'frontals_filtered_small': 0 # Track filtered detections } @@ -202,6 +207,10 @@ class TrackingPipelineIntegration: else: logger.debug(f"No tracking results or detections attribute") + # Filter out small frontal detections (neighboring pumps/distant cars) + if tracking_results and hasattr(tracking_results, 'detections'): + tracking_results = self._filter_small_frontals(tracking_results, frame) + # Process tracking results tracked_vehicles = self.tracker.process_detections( tracking_results, @@ -667,6 +676,55 @@ class TrackingPipelineIntegration: if stage == "car_wait_staff": logger.info(f"Started monitoring session {session_id} for car abandonment") + def _filter_small_frontals(self, tracking_results, frame): + """ + Filter out frontal detections that are smaller than minimum bbox area percentage. + This prevents processing of cars from neighboring pumps that appear in camera view. + + Args: + tracking_results: YOLO tracking results with detections + frame: Input frame for calculating frame area + + Returns: + Modified tracking_results with small frontals removed + """ + if not hasattr(tracking_results, 'detections') or not tracking_results.detections: + return tracking_results + + # Calculate frame area and minimum bbox area threshold + frame_area = frame.shape[0] * frame.shape[1] # height * width + min_bbox_area = frame_area * (self.min_bbox_area_percentage / 100.0) + + # Filter detections + filtered_detections = [] + filtered_count = 0 + + for detection in tracking_results.detections: + # Calculate detection bbox area + bbox = detection.bbox # Assuming bbox is [x1, y1, x2, y2] + bbox_area = (bbox[2] - bbox[0]) * (bbox[3] - bbox[1]) + + if bbox_area >= min_bbox_area: + # Keep detection - bbox is large enough + filtered_detections.append(detection) + else: + # Filter out small detection + filtered_count += 1 + area_percentage = (bbox_area / frame_area) * 100 + logger.debug(f"Filtered small frontal: area={bbox_area:.0f}px² ({area_percentage:.1f}% of frame, " + f"min required: {self.min_bbox_area_percentage}%)") + + # Update tracking results with filtered detections + tracking_results.detections = filtered_detections + + # Update statistics + if filtered_count > 0: + self.stats['frontals_filtered_small'] += filtered_count + logger.info(f"Filtered {filtered_count} small frontal detections, " + f"{len(filtered_detections)} remaining (total filtered: {self.stats['frontals_filtered_small']})") + + return tracking_results + def cleanup(self): """Cleanup resources.""" self.executor.shutdown(wait=False) From fa0f865319753d30c499899450117d4094293009 Mon Sep 17 00:00:00 2001 From: ziesorx Date: Tue, 30 Sep 2025 00:53:27 +0700 Subject: [PATCH 45/55] feat: add fallback when cant initially detect but backend start session --- core/tracking/integration.py | 136 +++++++++++++++++++++++++++++------ 1 file changed, 116 insertions(+), 20 deletions(-) diff --git a/core/tracking/integration.py b/core/tracking/integration.py index d1401ef..7d5f3f8 100644 --- a/core/tracking/integration.py +++ b/core/tracking/integration.py @@ -411,27 +411,12 @@ class TrackingPipelineIntegration: logger.info(f"Executing processing phase for session {session_id}, vehicle {vehicle.track_id}") # Capture high-quality snapshot for pipeline processing - frame = None - if self.subscription_info and self.subscription_info.stream_config.snapshot_url: - from ..streaming.readers import HTTPSnapshotReader + logger.info(f"[PROCESSING PHASE] Fetching 2K snapshot for session {session_id}") + frame = self._fetch_snapshot() - logger.info(f"[PROCESSING PHASE] Fetching 2K snapshot for session {session_id}") - snapshot_reader = HTTPSnapshotReader( - camera_id=self.subscription_info.camera_id, - snapshot_url=self.subscription_info.stream_config.snapshot_url, - max_retries=3 - ) - - frame = snapshot_reader.fetch_single_snapshot() - - if frame is not None: - logger.info(f"[PROCESSING PHASE] Successfully fetched {frame.shape[1]}x{frame.shape[0]} snapshot for pipeline") - else: - logger.warning(f"[PROCESSING PHASE] Failed to capture snapshot, falling back to RTSP frame") - # Fall back to RTSP frame if snapshot fails - frame = processing_data['frame'] - else: - logger.warning(f"[PROCESSING PHASE] No snapshot URL available, using RTSP frame") + if frame is None: + logger.warning(f"[PROCESSING PHASE] Failed to capture snapshot, falling back to RTSP frame") + # Fall back to RTSP frame if snapshot fails frame = processing_data['frame'] # Extract detected regions from detection phase result if available @@ -527,6 +512,19 @@ class TrackingPipelineIntegration: else: logger.warning(f"No pending processing data found for display {display_id} when setting session {session_id}") + # FALLBACK: Execute pipeline for POS-initiated sessions + logger.info(f"[FALLBACK] Triggering fallback pipeline for session {session_id} on display {display_id}") + + # Create subscription_id for fallback (needed for pipeline execution) + fallback_subscription_id = f"{display_id};fallback" + + # Trigger the fallback pipeline asynchronously + asyncio.create_task(self._execute_fallback_pipeline( + display_id=display_id, + session_id=session_id, + subscription_id=fallback_subscription_id + )) + def clear_session_id(self, session_id: str): """ Clear session ID (post-fueling). @@ -676,6 +674,104 @@ class TrackingPipelineIntegration: if stage == "car_wait_staff": logger.info(f"Started monitoring session {session_id} for car abandonment") + def _fetch_snapshot(self) -> Optional[np.ndarray]: + """ + Fetch high-quality snapshot from camera's snapshot URL. + Reusable method for both processing phase and fallback pipeline. + + Returns: + Snapshot frame or None if unavailable + """ + if not (self.subscription_info and self.subscription_info.stream_config.snapshot_url): + logger.warning("[SNAPSHOT] No subscription info or snapshot URL available") + return None + + try: + from ..streaming.readers import HTTPSnapshotReader + + logger.info(f"[SNAPSHOT] Fetching snapshot for {self.subscription_info.camera_id}") + snapshot_reader = HTTPSnapshotReader( + camera_id=self.subscription_info.camera_id, + snapshot_url=self.subscription_info.stream_config.snapshot_url, + max_retries=3 + ) + + frame = snapshot_reader.fetch_single_snapshot() + + if frame is not None: + logger.info(f"[SNAPSHOT] Successfully fetched {frame.shape[1]}x{frame.shape[0]} snapshot") + return frame + else: + logger.warning("[SNAPSHOT] Failed to fetch snapshot") + return None + + except Exception as e: + logger.error(f"[SNAPSHOT] Error fetching snapshot: {e}", exc_info=True) + return None + + async def _execute_fallback_pipeline(self, display_id: str, session_id: str, subscription_id: str): + """ + Execute fallback pipeline when sessionId is received without prior detection. + This handles POS-initiated sessions where backend starts transaction before car detection. + + Args: + display_id: Display identifier + session_id: Session ID from backend + subscription_id: Subscription identifier for pipeline execution + """ + try: + logger.info(f"[FALLBACK PIPELINE] Executing for session {session_id}, display {display_id}") + + # Fetch fresh snapshot from camera + frame = self._fetch_snapshot() + + if frame is None: + logger.error(f"[FALLBACK] Failed to fetch snapshot for session {session_id}, cannot execute pipeline") + return + + logger.info(f"[FALLBACK] Using snapshot frame {frame.shape[1]}x{frame.shape[0]} for session {session_id}") + + # Check if detection pipeline is available + if not self.detection_pipeline: + logger.error(f"[FALLBACK] Detection pipeline not available for session {session_id}") + return + + # Execute detection phase to get detected regions + detection_result = await self.detection_pipeline.execute_detection_phase( + frame=frame, + display_id=display_id, + subscription_id=subscription_id + ) + + logger.info(f"[FALLBACK] Detection phase completed for session {session_id}: " + f"status={detection_result.get('status', 'unknown')}, " + f"regions={list(detection_result.get('detected_regions', {}).keys())}") + + # If detection found regions, execute processing phase + detected_regions = detection_result.get('detected_regions', {}) + if detected_regions: + processing_result = await self.detection_pipeline.execute_processing_phase( + frame=frame, + display_id=display_id, + session_id=session_id, + subscription_id=subscription_id, + detected_regions=detected_regions + ) + + logger.info(f"[FALLBACK] Processing phase completed for session {session_id}: " + f"status={processing_result.get('status', 'unknown')}, " + f"branches={len(processing_result.get('branch_results', {}))}, " + f"actions={len(processing_result.get('actions_executed', []))}") + + # Update statistics + self.stats['pipelines_executed'] += 1 + + else: + logger.warning(f"[FALLBACK] No detections found in snapshot for session {session_id}") + + except Exception as e: + logger.error(f"[FALLBACK] Error executing fallback pipeline for session {session_id}: {e}", exc_info=True) + def _filter_small_frontals(self, tracking_results, frame): """ Filter out frontal detections that are smaller than minimum bbox area percentage. From 31bc91d57ba03d0cd2e4d6f8b936ad18d9adfaae Mon Sep 17 00:00:00 2001 From: ziesorx Date: Tue, 30 Sep 2025 12:06:03 +0700 Subject: [PATCH 46/55] fix: add ffmpeg flags fix frame delay --- core/streaming/readers/ffmpeg_rtsp.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/streaming/readers/ffmpeg_rtsp.py b/core/streaming/readers/ffmpeg_rtsp.py index 7c453f3..352c28e 100644 --- a/core/streaming/readers/ffmpeg_rtsp.py +++ b/core/streaming/readers/ffmpeg_rtsp.py @@ -115,10 +115,17 @@ class FFmpegRTSPReader(VideoReader): # DO NOT REMOVE '-hwaccel', 'cuda', '-hwaccel_device', '0', + # Real-time input flags + '-fflags', 'nobuffer+genpts+discardcorrupt', + '-flags', 'low_delay', + '-max_delay', '0', # No reordering delay + # RTSP configuration '-rtsp_transport', 'tcp', '-i', self.rtsp_url, + # Output configuration (keeping BMP) '-f', 'image2pipe', # Output images to pipe '-vcodec', 'bmp', # BMP format with header containing dimensions + '-vsync', 'passthrough', # Pass frames as-is # Use native stream resolution and framerate '-an', # No audio '-' # Output to stdout From fed71046a9437be76cc80c2ce6705e4f273405a6 Mon Sep 17 00:00:00 2001 From: ziesorx Date: Tue, 30 Sep 2025 12:20:52 +0700 Subject: [PATCH 47/55] fix: update ffmpeg flags to improve frame handling --- core/streaming/readers/ffmpeg_rtsp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/streaming/readers/ffmpeg_rtsp.py b/core/streaming/readers/ffmpeg_rtsp.py index 352c28e..88f45ae 100644 --- a/core/streaming/readers/ffmpeg_rtsp.py +++ b/core/streaming/readers/ffmpeg_rtsp.py @@ -116,7 +116,7 @@ class FFmpegRTSPReader(VideoReader): '-hwaccel', 'cuda', '-hwaccel_device', '0', # Real-time input flags - '-fflags', 'nobuffer+genpts+discardcorrupt', + '-fflags', 'nobuffer+genpts', '-flags', 'low_delay', '-max_delay', '0', # No reordering delay # RTSP configuration From 8d2a71fcd73daa8f6ddc156f72e20eb09b0bf3de Mon Sep 17 00:00:00 2001 From: ziesorx Date: Tue, 30 Sep 2025 14:21:29 +0700 Subject: [PATCH 48/55] fix: inference in reader thread --- core/streaming/manager.py | 223 +++++++++++++++++++++++++- core/streaming/readers/ffmpeg_rtsp.py | 4 +- 2 files changed, 223 insertions(+), 4 deletions(-) diff --git a/core/streaming/manager.py b/core/streaming/manager.py index e2f02d9..c082e70 100644 --- a/core/streaming/manager.py +++ b/core/streaming/manager.py @@ -5,6 +5,8 @@ Optimized for 1280x720@6fps RTSP and 2560x1440 HTTP snapshots. import logging import threading import time +import queue +import asyncio from typing import Dict, Set, Optional, List, Any from dataclasses import dataclass from collections import defaultdict @@ -50,6 +52,64 @@ class StreamManager: self._camera_subscribers: Dict[str, Set[str]] = defaultdict(set) # camera_id -> set of subscription_ids self._lock = threading.RLock() + # Fair tracking queue system - per camera queues + self._tracking_queues: Dict[str, queue.Queue] = {} # camera_id -> queue + self._tracking_workers = [] + self._stop_workers = threading.Event() + self._dropped_frame_counts: Dict[str, int] = {} # per-camera drop counts + + # Round-robin scheduling state + self._camera_list = [] # Ordered list of active cameras + self._camera_round_robin_index = 0 + self._round_robin_lock = threading.Lock() + + # Start worker threads for tracking processing + num_workers = min(4, max_streams // 2 + 1) # Scale with streams + for i in range(num_workers): + worker = threading.Thread( + target=self._tracking_worker_loop, + name=f"TrackingWorker-{i}", + daemon=True + ) + worker.start() + self._tracking_workers.append(worker) + + logger.info(f"Started {num_workers} tracking worker threads") + + def _ensure_camera_queue(self, camera_id: str): + """Ensure a tracking queue exists for the camera.""" + if camera_id not in self._tracking_queues: + self._tracking_queues[camera_id] = queue.Queue(maxsize=10) # 10 frames per camera + self._dropped_frame_counts[camera_id] = 0 + + with self._round_robin_lock: + if camera_id not in self._camera_list: + self._camera_list.append(camera_id) + + logger.info(f"Created tracking queue for camera {camera_id}") + + def _remove_camera_queue(self, camera_id: str): + """Remove tracking queue for a camera that's no longer active.""" + if camera_id in self._tracking_queues: + # Clear any remaining items + while not self._tracking_queues[camera_id].empty(): + try: + self._tracking_queues[camera_id].get_nowait() + except queue.Empty: + break + + del self._tracking_queues[camera_id] + del self._dropped_frame_counts[camera_id] + + with self._round_robin_lock: + if camera_id in self._camera_list: + self._camera_list.remove(camera_id) + # Reset index if needed + if self._camera_round_robin_index >= len(self._camera_list): + self._camera_round_robin_index = 0 + + logger.info(f"Removed tracking queue for camera {camera_id}") + def add_subscription(self, subscription_id: str, stream_config: StreamConfig, crop_coords: Optional[tuple] = None, model_id: Optional[str] = None, @@ -139,6 +199,7 @@ class StreamManager: reader.set_frame_callback(self._frame_callback) reader.start() self._streams[camera_id] = reader + self._ensure_camera_queue(camera_id) # Create tracking queue logger.info(f"\033[92m[RTSP] {camera_id} connected\033[0m") elif stream_config.snapshot_url: @@ -153,6 +214,7 @@ class StreamManager: reader.set_frame_callback(self._frame_callback) reader.start() self._streams[camera_id] = reader + self._ensure_camera_queue(camera_id) # Create tracking queue logger.info(f"\033[92m[HTTP] {camera_id} connected\033[0m") else: @@ -171,6 +233,7 @@ class StreamManager: try: self._streams[camera_id].stop() del self._streams[camera_id] + self._remove_camera_queue(camera_id) # Remove tracking queue # DON'T clear frames - they should persist until replaced # shared_cache_buffer.clear_camera(camera_id) # REMOVED - frames should persist logger.info(f"Stopped stream for camera {camera_id} (frames preserved in buffer)") @@ -193,8 +256,19 @@ class StreamManager: available_cameras = shared_cache_buffer.frame_buffer.get_camera_list() logger.info(f"\033[96m[BUFFER] {len(available_cameras)} active cameras: {', '.join(available_cameras)}\033[0m") - # Process tracking for subscriptions with tracking integration - self._process_tracking_for_camera(camera_id, frame) + # Queue for tracking processing (non-blocking) - route to camera-specific queue + if camera_id in self._tracking_queues: + try: + self._tracking_queues[camera_id].put_nowait({ + 'frame': frame, + 'timestamp': time.time() + }) + except queue.Full: + # Drop frame if camera queue is full (maintain real-time) + self._dropped_frame_counts[camera_id] += 1 + + if self._dropped_frame_counts[camera_id] % 50 == 0: + logger.warning(f"Dropped {self._dropped_frame_counts[camera_id]} frames for camera {camera_id} due to full queue") except Exception as e: logger.error(f"Error in frame callback for camera {camera_id}: {e}") @@ -251,6 +325,127 @@ class StreamManager: except Exception as e: logger.error(f"Error processing tracking for camera {camera_id}: {e}") + def _tracking_worker_loop(self): + """Worker thread loop for round-robin processing of camera queues.""" + logger.info(f"Tracking worker {threading.current_thread().name} started") + + consecutive_empty = 0 + max_consecutive_empty = 10 # Sleep if all cameras empty this many times + + while not self._stop_workers.is_set(): + try: + # Get next camera in round-robin fashion + camera_id, item = self._get_next_camera_item() + + if camera_id is None: + # No cameras have items, sleep briefly + consecutive_empty += 1 + if consecutive_empty >= max_consecutive_empty: + time.sleep(0.1) # Sleep 100ms if nothing to process + consecutive_empty = 0 + continue + + consecutive_empty = 0 # Reset counter when we find work + + frame = item['frame'] + timestamp = item['timestamp'] + + # Check if frame is too old (drop if > 1 second old) + age = time.time() - timestamp + if age > 1.0: + logger.debug(f"Dropping old frame for {camera_id} (age: {age:.2f}s)") + continue + + # Process tracking for this camera's frame + self._process_tracking_for_camera_sync(camera_id, frame) + + except Exception as e: + logger.error(f"Error in tracking worker: {e}", exc_info=True) + + logger.info(f"Tracking worker {threading.current_thread().name} stopped") + + def _get_next_camera_item(self): + """Get next item from camera queues using round-robin scheduling.""" + with self._round_robin_lock: + if not self._camera_list: + return None, None + + attempts = 0 + max_attempts = len(self._camera_list) + + while attempts < max_attempts: + # Get current camera + if self._camera_round_robin_index >= len(self._camera_list): + self._camera_round_robin_index = 0 + + camera_id = self._camera_list[self._camera_round_robin_index] + + # Move to next camera for next call + self._camera_round_robin_index = (self._camera_round_robin_index + 1) % len(self._camera_list) + + # Try to get item from this camera's queue + if camera_id in self._tracking_queues: + try: + item = self._tracking_queues[camera_id].get_nowait() + return camera_id, item + except queue.Empty: + pass # Try next camera + + attempts += 1 + + return None, None # All cameras empty + + def _process_tracking_for_camera_sync(self, camera_id: str, frame): + """Synchronous version of tracking processing for worker threads.""" + try: + with self._lock: + subscription_ids = list(self._camera_subscribers.get(camera_id, [])) + + for subscription_id in subscription_ids: + subscription_info = self._subscriptions.get(subscription_id) + + if not subscription_info or not subscription_info.tracking_integration: + continue + + display_id = subscription_id.split(';')[0] if ';' in subscription_id else subscription_id + + try: + # Run async tracking in thread's event loop + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + result = loop.run_until_complete( + subscription_info.tracking_integration.process_frame( + frame, display_id, subscription_id + ) + ) + + # Log tracking results + if result: + tracked_count = len(result.get('tracked_vehicles', [])) + validated_vehicle = result.get('validated_vehicle') + pipeline_result = result.get('pipeline_result') + + if tracked_count > 0: + logger.info(f"[Tracking] {camera_id}: {tracked_count} vehicles tracked") + + if validated_vehicle: + logger.info(f"[Tracking] {camera_id}: Vehicle {validated_vehicle['track_id']} " + f"validated as {validated_vehicle['state']} " + f"(confidence: {validated_vehicle['confidence']:.2f})") + + if pipeline_result: + logger.info(f"[Pipeline] {camera_id}: {pipeline_result.get('status', 'unknown')} - " + f"{pipeline_result.get('message', 'no message')}") + finally: + loop.close() + + except Exception as track_e: + logger.error(f"Error in tracking for {subscription_id}: {track_e}") + + except Exception as e: + logger.error(f"Error processing tracking for camera {camera_id}: {e}") + def get_frame(self, camera_id: str, crop_coords: Optional[tuple] = None): """Get the latest frame for a camera with optional cropping.""" return shared_cache_buffer.get_frame(camera_id, crop_coords) @@ -366,6 +561,30 @@ class StreamManager: def stop_all(self): """Stop all streams and clear all subscriptions.""" + # Signal workers to stop + self._stop_workers.set() + + # Clear all camera queues + for camera_id, camera_queue in list(self._tracking_queues.items()): + while not camera_queue.empty(): + try: + camera_queue.get_nowait() + except queue.Empty: + break + + # Wait for workers to finish + for worker in self._tracking_workers: + worker.join(timeout=2.0) + + # Clear queue management structures + self._tracking_queues.clear() + self._dropped_frame_counts.clear() + with self._round_robin_lock: + self._camera_list.clear() + self._camera_round_robin_index = 0 + + logger.info("Stopped all tracking worker threads") + with self._lock: # Stop all streams for camera_id in list(self._streams.keys()): diff --git a/core/streaming/readers/ffmpeg_rtsp.py b/core/streaming/readers/ffmpeg_rtsp.py index 88f45ae..e469c9e 100644 --- a/core/streaming/readers/ffmpeg_rtsp.py +++ b/core/streaming/readers/ffmpeg_rtsp.py @@ -113,8 +113,8 @@ class FFmpegRTSPReader(VideoReader): cmd = [ 'ffmpeg', # DO NOT REMOVE - '-hwaccel', 'cuda', - '-hwaccel_device', '0', + # '-hwaccel', 'cuda', + # '-hwaccel_device', '0', # Real-time input flags '-fflags', 'nobuffer+genpts', '-flags', 'low_delay', From e92efdbe11e6fe9254d2f44581fab2fc92546eb1 Mon Sep 17 00:00:00 2001 From: ziesorx Date: Tue, 30 Sep 2025 15:14:28 +0700 Subject: [PATCH 49/55] fix: custom subscriptionIdentifier --- core/streaming/manager.py | 9 +++++++-- core/tracking/integration.py | 35 +++++++++++++++++++++++------------ 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/core/streaming/manager.py b/core/streaming/manager.py index c082e70..497f1b8 100644 --- a/core/streaming/manager.py +++ b/core/streaming/manager.py @@ -606,8 +606,13 @@ class StreamManager: # Check if this subscription matches the display_id subscription_display_id = subscription_info.subscription_id.split(';')[0] if subscription_display_id == display_id and subscription_info.tracking_integration: - subscription_info.tracking_integration.set_session_id(display_id, session_id) - logger.debug(f"Set session {session_id} for display {display_id}") + # Pass the full subscription_id (displayId;cameraId) to the tracking integration + subscription_info.tracking_integration.set_session_id( + display_id, + session_id, + subscription_id=subscription_info.subscription_id + ) + logger.debug(f"Set session {session_id} for display {display_id} with subscription {subscription_info.subscription_id}") def clear_session_id(self, session_id: str): """Clear session ID from the specific tracking integration handling this session.""" diff --git a/core/tracking/integration.py b/core/tracking/integration.py index 7d5f3f8..58afcec 100644 --- a/core/tracking/integration.py +++ b/core/tracking/integration.py @@ -61,6 +61,7 @@ class TrackingPipelineIntegration: self.cleared_sessions: Dict[str, float] = {} # session_id -> clear_time self.pending_vehicles: Dict[str, int] = {} # display_id -> track_id (waiting for session ID) self.pending_processing_data: Dict[str, Dict] = {} # display_id -> processing data (waiting for session ID) + self.display_to_subscription: Dict[str, str] = {} # display_id -> subscription_id (for fallback) # Additional validators for enhanced flow control self.permanently_processed: Dict[str, float] = {} # "camera_id:track_id" -> process_time (never process again) @@ -459,7 +460,7 @@ class TrackingPipelineIntegration: self.subscription_info = subscription_info logger.debug(f"Set subscription info with snapshot_url: {subscription_info.stream_config.snapshot_url if subscription_info else None}") - def set_session_id(self, display_id: str, session_id: str): + def set_session_id(self, display_id: str, session_id: str, subscription_id: str = None): """ Set session ID for a display (from backend). This is called when backend sends setSessionId after receiving imageDetection. @@ -467,11 +468,18 @@ class TrackingPipelineIntegration: Args: display_id: Display identifier session_id: Session identifier + subscription_id: Subscription identifier (displayId;cameraId) - needed for fallback """ # Ensure session_id is always a string for consistent type handling session_id = str(session_id) if session_id is not None else None self.active_sessions[display_id] = session_id - logger.info(f"Set session {session_id} for display {display_id}") + + # Store subscription_id for fallback usage + if subscription_id: + self.display_to_subscription[display_id] = subscription_id + logger.info(f"Set session {session_id} for display {display_id} with subscription {subscription_id}") + else: + logger.info(f"Set session {session_id} for display {display_id}") # Check if we have a pending vehicle for this display if display_id in self.pending_vehicles: @@ -513,17 +521,19 @@ class TrackingPipelineIntegration: logger.warning(f"No pending processing data found for display {display_id} when setting session {session_id}") # FALLBACK: Execute pipeline for POS-initiated sessions - logger.info(f"[FALLBACK] Triggering fallback pipeline for session {session_id} on display {display_id}") + # Use stored subscription_id instead of creating fake one + stored_subscription_id = self.display_to_subscription.get(display_id) + if stored_subscription_id: + logger.info(f"[FALLBACK] Triggering fallback pipeline for session {session_id} on display {display_id} with subscription {stored_subscription_id}") - # Create subscription_id for fallback (needed for pipeline execution) - fallback_subscription_id = f"{display_id};fallback" - - # Trigger the fallback pipeline asynchronously - asyncio.create_task(self._execute_fallback_pipeline( - display_id=display_id, - session_id=session_id, - subscription_id=fallback_subscription_id - )) + # Trigger the fallback pipeline asynchronously with real subscription_id + asyncio.create_task(self._execute_fallback_pipeline( + display_id=display_id, + session_id=session_id, + subscription_id=stored_subscription_id + )) + else: + logger.error(f"[FALLBACK] No subscription_id stored for display {display_id}, cannot execute fallback pipeline") def clear_session_id(self, session_id: str): """ @@ -574,6 +584,7 @@ class TrackingPipelineIntegration: self.cleared_sessions.clear() self.pending_vehicles.clear() self.pending_processing_data.clear() + self.display_to_subscription.clear() self.permanently_processed.clear() self.progression_stages.clear() self.last_detection_time.clear() From 354ed9ce3cfae296450b2e747ac77e963d3080a4 Mon Sep 17 00:00:00 2001 From: ziesorx Date: Tue, 30 Sep 2025 15:46:32 +0700 Subject: [PATCH 50/55] fix: fallback when there is sessionId --- core/detection/pipeline.py | 92 ++++++++++++++++++++++++++++-------- core/tracking/integration.py | 26 +++++----- 2 files changed, 88 insertions(+), 30 deletions(-) diff --git a/core/detection/pipeline.py b/core/detection/pipeline.py index 076cdc9..d395f3a 100644 --- a/core/detection/pipeline.py +++ b/core/detection/pipeline.py @@ -64,6 +64,10 @@ class DetectionPipeline: # SessionId to processing results mapping (for combining with license plate results) self.session_processing_results = {} + # Field mappings from parallelActions (e.g., {"car_brand": "{car_brand_cls_v3.brand}"}) + self.field_mappings = {} + self._parse_field_mappings() + # Statistics self.stats = { 'detections_processed': 0, @@ -74,6 +78,25 @@ class DetectionPipeline: logger.info("DetectionPipeline initialized") + def _parse_field_mappings(self): + """ + Parse field mappings from parallelActions.postgresql_update_combined.fields. + Extracts mappings like {"car_brand": "{car_brand_cls_v3.brand}"} for dynamic field resolution. + """ + try: + if not self.pipeline_config or not hasattr(self.pipeline_config, 'parallel_actions'): + return + + for action in self.pipeline_config.parallel_actions: + if action.type.value == 'postgresql_update_combined': + fields = action.params.get('fields', {}) + self.field_mappings = fields + logger.info(f"[FIELD MAPPINGS] Parsed from pipeline config: {self.field_mappings}") + break + + except Exception as e: + logger.error(f"Error parsing field mappings: {e}", exc_info=True) + async def initialize(self) -> bool: """ Initialize all pipeline components including models, Redis, and database. @@ -165,6 +188,44 @@ class DetectionPipeline: logger.error(f"Error initializing detection model: {e}", exc_info=True) return False + def _extract_fields_from_branches(self, branch_results: Dict[str, Any]) -> Dict[str, Any]: + """ + Extract fields dynamically from branch results using field mappings. + + Args: + branch_results: Dictionary of branch execution results + + Returns: + Dictionary with extracted field values (e.g., {"car_brand": "Honda", "body_type": "Sedan"}) + """ + extracted = {} + + try: + for db_field_name, template in self.field_mappings.items(): + # Parse template like "{car_brand_cls_v3.brand}" -> branch_id="car_brand_cls_v3", field="brand" + if template.startswith('{') and template.endswith('}'): + var_name = template[1:-1] + if '.' in var_name: + branch_id, field_name = var_name.split('.', 1) + + # Look up value in branch_results + if branch_id in branch_results: + branch_data = branch_results[branch_id] + if isinstance(branch_data, dict) and 'result' in branch_data: + result_data = branch_data['result'] + if isinstance(result_data, dict) and field_name in result_data: + extracted[field_name] = result_data[field_name] + logger.debug(f"[DYNAMIC EXTRACT] {field_name}={result_data[field_name]} from branch {branch_id}") + else: + logger.debug(f"[DYNAMIC EXTRACT] Field '{field_name}' not found in branch {branch_id}") + else: + logger.debug(f"[DYNAMIC EXTRACT] Branch '{branch_id}' not in results") + + except Exception as e: + logger.error(f"Error extracting fields from branches: {e}", exc_info=True) + + return extracted + async def _on_license_plate_result(self, session_id: str, license_data: Dict[str, Any]): """ Callback for handling license plate results from LPR service. @@ -272,12 +333,12 @@ class DetectionPipeline: branch_results = self.session_processing_results[session_id_for_lookup] logger.info(f"[LICENSE PLATE] Retrieved processing results for session {session_id_for_lookup}") - if 'car_brand_cls_v2' in branch_results: - brand_result = branch_results['car_brand_cls_v2'].get('result', {}) - car_brand = brand_result.get('brand') - if 'car_bodytype_cls_v1' in branch_results: - bodytype_result = branch_results['car_bodytype_cls_v1'].get('result', {}) - body_type = bodytype_result.get('body_type') + # Extract fields dynamically using field mappings from pipeline config + extracted_fields = self._extract_fields_from_branches(branch_results) + car_brand = extracted_fields.get('brand') + body_type = extracted_fields.get('body_type') + + logger.info(f"[LICENSE PLATE] Extracted fields: brand={car_brand}, body_type={body_type}") # Clean up stored results after use del self.session_processing_results[session_id_for_lookup] @@ -1003,7 +1064,7 @@ class DetectionPipeline: Resolve field template using branch results and context. Args: - template: Template string like "{car_brand_cls_v2.brand}" + template: Template string like "{car_brand_cls_v3.brand}" branch_results: Dictionary of branch execution results context: Detection context @@ -1015,7 +1076,7 @@ class DetectionPipeline: if template.startswith('{') and template.endswith('}'): var_name = template[1:-1] - # Check for branch result reference (e.g., "car_brand_cls_v2.brand") + # Check for branch result reference (e.g., "car_brand_cls_v3.brand") if '.' in var_name: branch_id, field_name = var_name.split('.', 1) if branch_id in branch_results: @@ -1061,17 +1122,10 @@ class DetectionPipeline: logger.warning("No session_id in context for processing results") return - # Extract car brand from car_brand_cls_v2 results - car_brand = None - if 'car_brand_cls_v2' in branch_results: - brand_result = branch_results['car_brand_cls_v2'].get('result', {}) - car_brand = brand_result.get('brand') - - # Extract body type from car_bodytype_cls_v1 results - body_type = None - if 'car_bodytype_cls_v1' in branch_results: - bodytype_result = branch_results['car_bodytype_cls_v1'].get('result', {}) - body_type = bodytype_result.get('body_type') + # Extract fields dynamically using field mappings from pipeline config + extracted_fields = self._extract_fields_from_branches(branch_results) + car_brand = extracted_fields.get('brand') + body_type = extracted_fields.get('body_type') logger.info(f"[PROCESSING RESULTS] Completed for session {session_id}: " f"brand={car_brand}, bodyType={body_type}") diff --git a/core/tracking/integration.py b/core/tracking/integration.py index 58afcec..8e0d8fa 100644 --- a/core/tracking/integration.py +++ b/core/tracking/integration.py @@ -521,19 +521,23 @@ class TrackingPipelineIntegration: logger.warning(f"No pending processing data found for display {display_id} when setting session {session_id}") # FALLBACK: Execute pipeline for POS-initiated sessions - # Use stored subscription_id instead of creating fake one - stored_subscription_id = self.display_to_subscription.get(display_id) - if stored_subscription_id: - logger.info(f"[FALLBACK] Triggering fallback pipeline for session {session_id} on display {display_id} with subscription {stored_subscription_id}") + # Skip if session_id is None (no car present or car has left) + if session_id is not None: + # Use stored subscription_id instead of creating fake one + stored_subscription_id = self.display_to_subscription.get(display_id) + if stored_subscription_id: + logger.info(f"[FALLBACK] Triggering fallback pipeline for session {session_id} on display {display_id} with subscription {stored_subscription_id}") - # Trigger the fallback pipeline asynchronously with real subscription_id - asyncio.create_task(self._execute_fallback_pipeline( - display_id=display_id, - session_id=session_id, - subscription_id=stored_subscription_id - )) + # Trigger the fallback pipeline asynchronously with real subscription_id + asyncio.create_task(self._execute_fallback_pipeline( + display_id=display_id, + session_id=session_id, + subscription_id=stored_subscription_id + )) + else: + logger.error(f"[FALLBACK] No subscription_id stored for display {display_id}, cannot execute fallback pipeline") else: - logger.error(f"[FALLBACK] No subscription_id stored for display {display_id}, cannot execute fallback pipeline") + logger.debug(f"[FALLBACK] Skipping pipeline execution for session_id=None on display {display_id}") def clear_session_id(self, session_id: str): """ From 793beb15710cb46605a754a83b08abb0e4fe1d92 Mon Sep 17 00:00:00 2001 From: ziesorx Date: Tue, 30 Sep 2025 16:04:24 +0700 Subject: [PATCH 51/55] fix: tracking works but absent not work --- app.py | 9 +++-- core/communication/websocket.py | 10 ++++- core/streaming/manager.py | 71 +++++++++++++++++++++++++-------- 3 files changed, 68 insertions(+), 22 deletions(-) diff --git a/app.py b/app.py index eb1440f..7b82d23 100644 --- a/app.py +++ b/app.py @@ -201,10 +201,11 @@ else: os.makedirs("models", exist_ok=True) logger.info("Ensured models directory exists") -# Initialize stream manager with config value -from core.streaming import initialize_stream_manager -initialize_stream_manager(max_streams=config.get('max_streams', 10)) -logger.info(f"Initialized stream manager with max_streams={config.get('max_streams', 10)}") +# Stream manager already initialized at module level with max_streams=20 +# Calling initialize_stream_manager() creates a NEW instance, breaking references +# from core.streaming import initialize_stream_manager +# initialize_stream_manager(max_streams=config.get('max_streams', 10)) +logger.info(f"Using stream manager with max_streams=20 (module-level initialization)") # Frames are now stored in the shared cache buffer from core.streaming.buffers # latest_frames = {} # Deprecated - using shared_cache_buffer instead diff --git a/core/communication/websocket.py b/core/communication/websocket.py index e53096a..d20ee32 100644 --- a/core/communication/websocket.py +++ b/core/communication/websocket.py @@ -197,18 +197,24 @@ class WebSocketHandler: async def _handle_set_subscription_list(self, message: SetSubscriptionListMessage) -> None: """Handle setSubscriptionList message for declarative subscription management.""" - logger.info(f"[RX Processing] setSubscriptionList with {len(message.subscriptions)} subscriptions") + logger.info(f"🎯 [RX Processing] setSubscriptionList with {len(message.subscriptions)} subscriptions") + for i, sub in enumerate(message.subscriptions): + logger.info(f" 📋 Sub {i+1}: {sub.subscriptionIdentifier} (model: {sub.modelId})") # Update worker state with new subscriptions worker_state.set_subscriptions(message.subscriptions) # Phase 2: Download and manage models + logger.info("📦 Starting model download phase...") await self._ensure_models(message.subscriptions) + logger.info("✅ Model download phase complete") # Phase 3 & 4: Integrate with streaming management and tracking + logger.info("🎬 Starting stream subscription update...") await self._update_stream_subscriptions(message.subscriptions) + logger.info("✅ Stream subscription update complete") - logger.info("Subscription list updated successfully") + logger.info("🏁 Subscription list updated successfully") async def _ensure_models(self, subscriptions) -> None: """Ensure all required models are downloaded and available.""" diff --git a/core/streaming/manager.py b/core/streaming/manager.py index 497f1b8..2de86e4 100644 --- a/core/streaming/manager.py +++ b/core/streaming/manager.py @@ -85,8 +85,11 @@ class StreamManager: with self._round_robin_lock: if camera_id not in self._camera_list: self._camera_list.append(camera_id) - - logger.info(f"Created tracking queue for camera {camera_id}") + logger.info(f"✅ Created tracking queue for camera {camera_id}, camera_list now has {len(self._camera_list)} cameras: {self._camera_list}") + else: + logger.warning(f"Camera {camera_id} already in camera_list") + else: + logger.debug(f"Camera {camera_id} already has tracking queue") def _remove_camera_queue(self, camera_id: str): """Remove tracking queue for a camera that's no longer active.""" @@ -153,6 +156,10 @@ class StreamManager: if not success: self._remove_subscription_internal(subscription_id) return False + else: + # Stream already exists, but ensure queue exists too + logger.info(f"Stream already exists for {camera_id}, ensuring queue exists") + self._ensure_camera_queue(camera_id) logger.info(f"Added subscription {subscription_id} for camera {camera_id} " f"({len(self._camera_subscribers[camera_id])} total subscribers)") @@ -188,6 +195,7 @@ class StreamManager: def _start_stream(self, camera_id: str, stream_config: StreamConfig) -> bool: """Start a stream for the given camera.""" try: + logger.info(f"🚀 _start_stream called for {camera_id}") if stream_config.rtsp_url: # RTSP stream using FFmpeg subprocess with CUDA acceleration logger.info(f"\033[94m[RTSP] Starting {camera_id}\033[0m") @@ -199,7 +207,9 @@ class StreamManager: reader.set_frame_callback(self._frame_callback) reader.start() self._streams[camera_id] = reader + logger.info(f"🎬 About to call _ensure_camera_queue for {camera_id}") self._ensure_camera_queue(camera_id) # Create tracking queue + logger.info(f"✅ _ensure_camera_queue completed for {camera_id}") logger.info(f"\033[92m[RTSP] {camera_id} connected\033[0m") elif stream_config.snapshot_url: @@ -214,7 +224,9 @@ class StreamManager: reader.set_frame_callback(self._frame_callback) reader.start() self._streams[camera_id] = reader + logger.info(f"🎬 About to call _ensure_camera_queue for {camera_id}") self._ensure_camera_queue(camera_id) # Create tracking queue + logger.info(f"✅ _ensure_camera_queue completed for {camera_id}") logger.info(f"\033[92m[HTTP] {camera_id} connected\033[0m") else: @@ -334,18 +346,22 @@ class StreamManager: while not self._stop_workers.is_set(): try: + logger.debug(f"Worker {threading.current_thread().name} loop iteration, stop_event={self._stop_workers.is_set()}") + # Get next camera in round-robin fashion camera_id, item = self._get_next_camera_item() if camera_id is None: # No cameras have items, sleep briefly consecutive_empty += 1 + logger.debug(f"Worker {threading.current_thread().name}: All queues empty ({consecutive_empty}/{max_consecutive_empty})") if consecutive_empty >= max_consecutive_empty: time.sleep(0.1) # Sleep 100ms if nothing to process consecutive_empty = 0 continue consecutive_empty = 0 # Reset counter when we find work + logger.info(f"Worker {threading.current_thread().name}: Processing frame from {camera_id}") frame = item['frame'] timestamp = item['timestamp'] @@ -353,11 +369,13 @@ class StreamManager: # Check if frame is too old (drop if > 1 second old) age = time.time() - timestamp if age > 1.0: - logger.debug(f"Dropping old frame for {camera_id} (age: {age:.2f}s)") + logger.warning(f"Dropping old frame for {camera_id} (age: {age:.2f}s)") continue + logger.info(f"Worker {threading.current_thread().name}: Calling tracking sync for {camera_id}") # Process tracking for this camera's frame self._process_tracking_for_camera_sync(camera_id, frame) + logger.info(f"Worker {threading.current_thread().name}: Finished tracking sync for {camera_id}") except Exception as e: logger.error(f"Error in tracking worker: {e}", exc_info=True) @@ -367,32 +385,48 @@ class StreamManager: def _get_next_camera_item(self): """Get next item from camera queues using round-robin scheduling.""" with self._round_robin_lock: - if not self._camera_list: + # Get current list of cameras from actual tracking queues (central state) + camera_list = list(self._tracking_queues.keys()) + + # Debug: show ALL state + logger.info(f"🔍 _tracking_queues keys: {list(self._tracking_queues.keys())}") + logger.info(f"🔍 _streams keys: {list(self._streams.keys())}") + logger.info(f"🔍 _subscriptions keys: {list(self._subscriptions.keys())}") + + if not camera_list: + logger.warning("⚠️ _get_next_camera_item: No cameras have tracking queues yet, but streams/subscriptions exist!") return None, None + logger.debug(f"_get_next_camera_item: {len(camera_list)} cameras with queues: {camera_list}") + attempts = 0 - max_attempts = len(self._camera_list) + max_attempts = len(camera_list) while attempts < max_attempts: - # Get current camera - if self._camera_round_robin_index >= len(self._camera_list): + # Get current camera using round-robin index + if self._camera_round_robin_index >= len(camera_list): self._camera_round_robin_index = 0 - camera_id = self._camera_list[self._camera_round_robin_index] + camera_id = camera_list[self._camera_round_robin_index] + logger.debug(f"_get_next_camera_item: Trying camera {camera_id} (attempt {attempts + 1}/{max_attempts})") # Move to next camera for next call - self._camera_round_robin_index = (self._camera_round_robin_index + 1) % len(self._camera_list) + self._camera_round_robin_index = (self._camera_round_robin_index + 1) % len(camera_list) # Try to get item from this camera's queue - if camera_id in self._tracking_queues: - try: - item = self._tracking_queues[camera_id].get_nowait() - return camera_id, item - except queue.Empty: - pass # Try next camera + queue_size = self._tracking_queues[camera_id].qsize() + logger.debug(f"_get_next_camera_item: Camera {camera_id} queue has {queue_size} items") + try: + item = self._tracking_queues[camera_id].get_nowait() + logger.info(f"_get_next_camera_item: Got item from {camera_id}") + return camera_id, item + except queue.Empty: + logger.debug(f"_get_next_camera_item: Camera {camera_id} queue empty") + pass # Try next camera attempts += 1 + logger.debug("_get_next_camera_item: All cameras empty") return None, None # All cameras empty def _process_tracking_for_camera_sync(self, camera_id: str, frame): @@ -404,7 +438,12 @@ class StreamManager: for subscription_id in subscription_ids: subscription_info = self._subscriptions.get(subscription_id) - if not subscription_info or not subscription_info.tracking_integration: + if not subscription_info: + logger.warning(f"No subscription info found for {subscription_id}") + continue + + if not subscription_info.tracking_integration: + logger.debug(f"No tracking integration for {subscription_id} (camera {camera_id}), skipping inference") continue display_id = subscription_id.split(';')[0] if ';' in subscription_id else subscription_id From 3ed7a2cd53dbf3fd06055fc189f3b3f1368770d7 Mon Sep 17 00:00:00 2001 From: ziesorx Date: Tue, 30 Sep 2025 16:20:39 +0700 Subject: [PATCH 52/55] fix: abandonment works --- core/communication/websocket.py | 10 ++-------- core/streaming/manager.py | 31 ++----------------------------- core/tracking/integration.py | 11 ++++++++++- 3 files changed, 14 insertions(+), 38 deletions(-) diff --git a/core/communication/websocket.py b/core/communication/websocket.py index d20ee32..e53096a 100644 --- a/core/communication/websocket.py +++ b/core/communication/websocket.py @@ -197,24 +197,18 @@ class WebSocketHandler: async def _handle_set_subscription_list(self, message: SetSubscriptionListMessage) -> None: """Handle setSubscriptionList message for declarative subscription management.""" - logger.info(f"🎯 [RX Processing] setSubscriptionList with {len(message.subscriptions)} subscriptions") - for i, sub in enumerate(message.subscriptions): - logger.info(f" 📋 Sub {i+1}: {sub.subscriptionIdentifier} (model: {sub.modelId})") + logger.info(f"[RX Processing] setSubscriptionList with {len(message.subscriptions)} subscriptions") # Update worker state with new subscriptions worker_state.set_subscriptions(message.subscriptions) # Phase 2: Download and manage models - logger.info("📦 Starting model download phase...") await self._ensure_models(message.subscriptions) - logger.info("✅ Model download phase complete") # Phase 3 & 4: Integrate with streaming management and tracking - logger.info("🎬 Starting stream subscription update...") await self._update_stream_subscriptions(message.subscriptions) - logger.info("✅ Stream subscription update complete") - logger.info("🏁 Subscription list updated successfully") + logger.info("Subscription list updated successfully") async def _ensure_models(self, subscriptions) -> None: """Ensure all required models are downloaded and available.""" diff --git a/core/streaming/manager.py b/core/streaming/manager.py index 2de86e4..c4ebd77 100644 --- a/core/streaming/manager.py +++ b/core/streaming/manager.py @@ -85,9 +85,7 @@ class StreamManager: with self._round_robin_lock: if camera_id not in self._camera_list: self._camera_list.append(camera_id) - logger.info(f"✅ Created tracking queue for camera {camera_id}, camera_list now has {len(self._camera_list)} cameras: {self._camera_list}") - else: - logger.warning(f"Camera {camera_id} already in camera_list") + logger.info(f"Created tracking queue for camera {camera_id}") else: logger.debug(f"Camera {camera_id} already has tracking queue") @@ -195,7 +193,6 @@ class StreamManager: def _start_stream(self, camera_id: str, stream_config: StreamConfig) -> bool: """Start a stream for the given camera.""" try: - logger.info(f"🚀 _start_stream called for {camera_id}") if stream_config.rtsp_url: # RTSP stream using FFmpeg subprocess with CUDA acceleration logger.info(f"\033[94m[RTSP] Starting {camera_id}\033[0m") @@ -207,9 +204,7 @@ class StreamManager: reader.set_frame_callback(self._frame_callback) reader.start() self._streams[camera_id] = reader - logger.info(f"🎬 About to call _ensure_camera_queue for {camera_id}") self._ensure_camera_queue(camera_id) # Create tracking queue - logger.info(f"✅ _ensure_camera_queue completed for {camera_id}") logger.info(f"\033[92m[RTSP] {camera_id} connected\033[0m") elif stream_config.snapshot_url: @@ -224,9 +219,7 @@ class StreamManager: reader.set_frame_callback(self._frame_callback) reader.start() self._streams[camera_id] = reader - logger.info(f"🎬 About to call _ensure_camera_queue for {camera_id}") self._ensure_camera_queue(camera_id) # Create tracking queue - logger.info(f"✅ _ensure_camera_queue completed for {camera_id}") logger.info(f"\033[92m[HTTP] {camera_id} connected\033[0m") else: @@ -346,22 +339,18 @@ class StreamManager: while not self._stop_workers.is_set(): try: - logger.debug(f"Worker {threading.current_thread().name} loop iteration, stop_event={self._stop_workers.is_set()}") - # Get next camera in round-robin fashion camera_id, item = self._get_next_camera_item() if camera_id is None: # No cameras have items, sleep briefly consecutive_empty += 1 - logger.debug(f"Worker {threading.current_thread().name}: All queues empty ({consecutive_empty}/{max_consecutive_empty})") if consecutive_empty >= max_consecutive_empty: time.sleep(0.1) # Sleep 100ms if nothing to process consecutive_empty = 0 continue consecutive_empty = 0 # Reset counter when we find work - logger.info(f"Worker {threading.current_thread().name}: Processing frame from {camera_id}") frame = item['frame'] timestamp = item['timestamp'] @@ -369,13 +358,11 @@ class StreamManager: # Check if frame is too old (drop if > 1 second old) age = time.time() - timestamp if age > 1.0: - logger.warning(f"Dropping old frame for {camera_id} (age: {age:.2f}s)") + logger.debug(f"Dropping old frame for {camera_id} (age: {age:.2f}s)") continue - logger.info(f"Worker {threading.current_thread().name}: Calling tracking sync for {camera_id}") # Process tracking for this camera's frame self._process_tracking_for_camera_sync(camera_id, frame) - logger.info(f"Worker {threading.current_thread().name}: Finished tracking sync for {camera_id}") except Exception as e: logger.error(f"Error in tracking worker: {e}", exc_info=True) @@ -388,17 +375,9 @@ class StreamManager: # Get current list of cameras from actual tracking queues (central state) camera_list = list(self._tracking_queues.keys()) - # Debug: show ALL state - logger.info(f"🔍 _tracking_queues keys: {list(self._tracking_queues.keys())}") - logger.info(f"🔍 _streams keys: {list(self._streams.keys())}") - logger.info(f"🔍 _subscriptions keys: {list(self._subscriptions.keys())}") - if not camera_list: - logger.warning("⚠️ _get_next_camera_item: No cameras have tracking queues yet, but streams/subscriptions exist!") return None, None - logger.debug(f"_get_next_camera_item: {len(camera_list)} cameras with queues: {camera_list}") - attempts = 0 max_attempts = len(camera_list) @@ -408,25 +387,19 @@ class StreamManager: self._camera_round_robin_index = 0 camera_id = camera_list[self._camera_round_robin_index] - logger.debug(f"_get_next_camera_item: Trying camera {camera_id} (attempt {attempts + 1}/{max_attempts})") # Move to next camera for next call self._camera_round_robin_index = (self._camera_round_robin_index + 1) % len(camera_list) # Try to get item from this camera's queue - queue_size = self._tracking_queues[camera_id].qsize() - logger.debug(f"_get_next_camera_item: Camera {camera_id} queue has {queue_size} items") try: item = self._tracking_queues[camera_id].get_nowait() - logger.info(f"_get_next_camera_item: Got item from {camera_id}") return camera_id, item except queue.Empty: - logger.debug(f"_get_next_camera_item: Camera {camera_id} queue empty") pass # Try next camera attempts += 1 - logger.debug("_get_next_camera_item: All cameras empty") return None, None # All cameras empty def _process_tracking_for_camera_sync(self, camera_id: str, frame): diff --git a/core/tracking/integration.py b/core/tracking/integration.py index 8e0d8fa..28e7d3a 100644 --- a/core/tracking/integration.py +++ b/core/tracking/integration.py @@ -220,8 +220,10 @@ class TrackingPipelineIntegration: ) # Update last detection time for abandonment detection + # Update when vehicles ARE detected, so when they leave, timestamp ages if tracked_vehicles: self.last_detection_time[display_id] = time.time() + logger.debug(f"Updated last_detection_time for {display_id}: {len(tracked_vehicles)} vehicles") # Check for car abandonment (vehicle left after getting car_wait_staff stage) await self._check_car_abandonment(display_id, subscription_id) @@ -632,10 +634,16 @@ class TrackingPipelineIntegration: last_detection = self.last_detection_time.get(session_display, 0) time_since_detection = current_time - last_detection + logger.info(f"[ABANDON CHECK] Session {session_id} (display: {session_display}): " + f"time_since_detection={time_since_detection:.1f}s, " + f"timeout={self.abandonment_timeout}s") + if time_since_detection > self.abandonment_timeout: - logger.info(f"Car abandonment detected: session {session_id}, " + logger.warning(f"🚨 Car abandonment detected: session {session_id}, " f"no detection for {time_since_detection:.1f}s") abandoned_sessions.append(session_id) + else: + logger.debug(f"[ABANDON CHECK] Session {session_id} has no associated display") # Send abandonment detection for each abandoned session for session_id in abandoned_sessions: @@ -643,6 +651,7 @@ class TrackingPipelineIntegration: # Remove from progression stages to avoid repeated detection if session_id in self.progression_stages: del self.progression_stages[session_id] + logger.info(f"[ABANDON] Removed session {session_id} from progression_stages after notification") async def _send_abandonment_detection(self, subscription_id: str, session_id: str): """ From 9e5b5a32adf02658b6f699fcdbba1aa98f172bcc Mon Sep 17 00:00:00 2001 From: ziesorx Date: Tue, 30 Sep 2025 16:23:07 +0700 Subject: [PATCH 53/55] fix: bring back gpu usage --- core/streaming/readers/ffmpeg_rtsp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/streaming/readers/ffmpeg_rtsp.py b/core/streaming/readers/ffmpeg_rtsp.py index e469c9e..88f45ae 100644 --- a/core/streaming/readers/ffmpeg_rtsp.py +++ b/core/streaming/readers/ffmpeg_rtsp.py @@ -113,8 +113,8 @@ class FFmpegRTSPReader(VideoReader): cmd = [ 'ffmpeg', # DO NOT REMOVE - # '-hwaccel', 'cuda', - # '-hwaccel_device', '0', + '-hwaccel', 'cuda', + '-hwaccel_device', '0', # Real-time input flags '-fflags', 'nobuffer+genpts', '-flags', 'low_delay', From 402f7732a8aeaa12c3916637798bab2f0d9243a2 Mon Sep 17 00:00:00 2001 From: ziesorx Date: Tue, 30 Sep 2025 17:24:33 +0700 Subject: [PATCH 54/55] fix: change min bbox size for frontal --- core/tracking/integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/tracking/integration.py b/core/tracking/integration.py index 28e7d3a..2fba002 100644 --- a/core/tracking/integration.py +++ b/core/tracking/integration.py @@ -74,7 +74,7 @@ class TrackingPipelineIntegration: # Min bbox filtering configuration # TODO: Make this configurable via pipeline.json in the future - self.min_bbox_area_percentage = 4.5 # 4.5% of frame area minimum + self.min_bbox_area_percentage = 3.5 # 3.5% of frame area minimum # Statistics self.stats = { From b2e7bc499d5edbaab724fc0e596ef8824671b9ac Mon Sep 17 00:00:00 2001 From: Siwat Sirichai Date: Wed, 1 Oct 2025 01:27:12 +0700 Subject: [PATCH 55/55] feat: add session image retrieval endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add HTTP endpoint to retrieve saved session images by session ID. Images are saved during car_fueling progression stage. - Add GET /session-image/{session_id} endpoint - Search images directory for files matching session ID pattern - Return most recent image if multiple exist - Proper error handling (404 for not found, 500 for errors) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- app.py | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/app.py b/app.py index 7b82d23..21d89db 100644 --- a/app.py +++ b/app.py @@ -302,6 +302,63 @@ async def get_camera_image(camera_id: str): raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}") +@app.get("/session-image/{session_id}") +async def get_session_image(session_id: int): + """ + HTTP endpoint to retrieve the saved session image by session ID. + + Args: + session_id: The session ID to retrieve the image for + + Returns: + JPEG image as binary response + + Raises: + HTTPException: 404 if no image found for the session + HTTPException: 500 if reading image fails + """ + try: + from pathlib import Path + import glob + + # Images directory + images_dir = Path("images") + + if not images_dir.exists(): + logger.warning(f"Images directory does not exist") + raise HTTPException( + status_code=404, + detail=f"No images directory found" + ) + + # Search for files matching session ID pattern: {session_id}_* + pattern = str(images_dir / f"{session_id}_*.jpg") + matching_files = glob.glob(pattern) + + if not matching_files: + logger.warning(f"No image found for session {session_id}") + raise HTTPException( + status_code=404, + detail=f"No image found for session {session_id}" + ) + + # Get the most recent file if multiple exist + most_recent_file = max(matching_files, key=os.path.getmtime) + logger.info(f"Found session image for session {session_id}: {most_recent_file}") + + # Read the image file + image_data = open(most_recent_file, 'rb').read() + + # Return image as binary response + return Response(content=image_data, media_type="image/jpeg") + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error retrieving session image for session {session_id}: {str(e)}", exc_info=True) + raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}") + + @app.get("/health") async def health_check(): """Health check endpoint for monitoring."""