diff --git a/.claude/settings.local.json b/.claude/settings.local.json deleted file mode 100644 index 9e296ac..0000000 --- a/.claude/settings.local.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(dir:*)", - "WebSearch", - "Bash(mkdir:*)" - ], - "deny": [], - "ask": [] - } -} \ No newline at end of file diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml index 316c4dc..585009f 100644 --- a/.gitea/workflows/build.yml +++ b/.gitea/workflows/build.yml @@ -51,7 +51,7 @@ jobs: registry: git.siwatsystem.com username: ${{ github.actor }} password: ${{ secrets.RUNNER_TOKEN }} - + - name: Build and push base Docker image uses: docker/build-push-action@v4 with: @@ -79,7 +79,7 @@ jobs: registry: git.siwatsystem.com username: ${{ github.actor }} password: ${{ secrets.RUNNER_TOKEN }} - + - name: Build and push Docker image uses: docker/build-push-action@v4 with: @@ -109,4 +109,4 @@ jobs: else echo "Deploying staging stack..." ssh -i ~/.ssh/id_rsa ${{ vars.DEPLOY_USER_CMS }}@${{ vars.DEPLOY_HOST_CMS }} "cd ~/cms-system-k8s && docker compose -f docker-compose.staging.yml pull && docker compose -f docker-compose.staging.yml up -d" - fi + fi \ No newline at end of file diff --git a/.gitignore b/.gitignore index 2da89cb..8e805c5 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,6 @@ app.log *.pt -images - # All pycache directories __pycache__/ .mptacache @@ -12,3 +10,17 @@ mptas detector_worker.log .gitignore no_frame_debug.log + +feeder/ +.venv/ +.vscode/ +dist/ +websocket_comm.log +temp_debug/ +.claude + +# Video Test +video_rtsp/ +multi_stream_viewer.py +multi_camera_simulator.py +start_4_cameras.bat \ No newline at end of file diff --git a/Dockerfile.base b/Dockerfile.base index 9684325..60999b1 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -1,130 +1,24 @@ -# Base image with complete ML and hardware acceleration stack +# Base image with all ML dependencies FROM pytorch/pytorch:2.8.0-cuda12.6-cudnn9-runtime -# Install build dependencies and system libraries -RUN apt-get update && apt-get install -y \ - # Build tools - build-essential \ - cmake \ - git \ - pkg-config \ - wget \ - unzip \ - yasm \ - nasm \ - # Additional dependencies for FFmpeg/NVIDIA build - libtool \ - libc6 \ - libc6-dev \ - libnuma1 \ - libnuma-dev \ - # Essential compilation libraries - gcc \ - g++ \ - libc6-dev \ - linux-libc-dev \ - # System libraries - libgl1-mesa-glx \ +# Install system dependencies +RUN apt update && apt install -y \ + libgl1 \ libglib2.0-0 \ + libgstreamer1.0-0 \ + libgtk-3-0 \ + libavcodec58 \ + libavformat58 \ + libswscale5 \ libgomp1 \ - # Core media libraries (essential ones only) - libjpeg-dev \ - libpng-dev \ - libx264-dev \ - libx265-dev \ - libvpx-dev \ - libmp3lame-dev \ - libv4l-dev \ - # TurboJPEG for fast JPEG encoding - libturbojpeg0-dev \ - # Python development - python3-dev \ - python3-numpy \ && rm -rf /var/lib/apt/lists/* -# Add NVIDIA CUDA repository and install minimal development tools -RUN apt-get update && apt-get install -y wget gnupg && \ - wget -O - https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub | apt-key add - && \ - echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 /" > /etc/apt/sources.list.d/cuda.list && \ - apt-get update && \ - apt-get install -y \ - cuda-nvcc-12-6 \ - cuda-cudart-dev-12-6 \ - libnpp-dev-12-6 \ - && apt-get remove -y wget gnupg && \ - apt-get autoremove -y && \ - rm -rf /var/lib/apt/lists/* - -# Ensure CUDA paths are available -ENV PATH="/usr/local/cuda/bin:${PATH}" -ENV LD_LIBRARY_PATH="/usr/local/cuda/lib64:${LD_LIBRARY_PATH}" - -# Install NVIDIA Video Codec SDK headers (official method) -RUN cd /tmp && \ - git clone https://git.videolan.org/git/ffmpeg/nv-codec-headers.git && \ - cd nv-codec-headers && \ - make install && \ - cd / && rm -rf /tmp/* - -# Build FFmpeg from source with NVIDIA CUDA support -RUN cd /tmp && \ - echo "Building FFmpeg with NVIDIA CUDA support..." && \ - # Download FFmpeg source (official method) - git clone https://git.ffmpeg.org/ffmpeg.git ffmpeg/ && \ - cd ffmpeg && \ - # Configure with NVIDIA support (simplified to avoid configure issues) - ./configure \ - --prefix=/usr/local \ - --enable-shared \ - --disable-static \ - --enable-nonfree \ - --enable-gpl \ - --enable-cuda-nvcc \ - --enable-cuvid \ - --enable-nvdec \ - --enable-nvenc \ - --enable-libnpp \ - --extra-cflags=-I/usr/local/cuda/include \ - --extra-ldflags=-L/usr/local/cuda/lib64 \ - --enable-libx264 \ - --enable-libx265 \ - --enable-libvpx \ - --enable-libmp3lame && \ - # Build and install - make -j$(nproc) && \ - make install && \ - ldconfig && \ - # Verify CUVID decoders are available - echo "=== Verifying FFmpeg CUVID Support ===" && \ - (ffmpeg -hide_banner -decoders 2>/dev/null | grep cuvid || echo "No CUVID decoders found") && \ - echo "=== Verifying FFmpeg NVENC Support ===" && \ - (ffmpeg -hide_banner -encoders 2>/dev/null | grep nvenc || echo "No NVENC encoders found") && \ - cd / && rm -rf /tmp/* - -# Set environment variables for maximum hardware acceleration -ENV LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/local/lib:${LD_LIBRARY_PATH}" -ENV PKG_CONFIG_PATH="/usr/local/lib/pkgconfig:${PKG_CONFIG_PATH}" -ENV PYTHONPATH="/usr/local/lib/python3.10/dist-packages:${PYTHONPATH}" - -# Optimized environment variables for hardware acceleration -ENV OPENCV_FFMPEG_CAPTURE_OPTIONS="rtsp_transport;tcp|hwaccel;cuda|hwaccel_device;0|video_codec;h264_cuvid|hwaccel_output_format;cuda" -ENV OPENCV_FFMPEG_WRITER_OPTIONS="video_codec;h264_nvenc|preset;fast|tune;zerolatency|gpu;0" -ENV CUDA_VISIBLE_DEVICES=0 -ENV NVIDIA_VISIBLE_DEVICES=all -ENV NVIDIA_DRIVER_CAPABILITIES=compute,video,utility - -# Copy and install base requirements (exclude opencv-python since we built from source) +# Copy and install base requirements (ML dependencies that rarely change) COPY requirements.base.txt . -RUN grep -v opencv-python requirements.base.txt > requirements.tmp && \ - mv requirements.tmp requirements.base.txt && \ - pip install --no-cache-dir -r requirements.base.txt +RUN pip install --no-cache-dir -r requirements.base.txt # Set working directory WORKDIR /app -# Create images directory for bind mount -RUN mkdir -p /app/images && \ - chmod 755 /app/images - # This base image will be reused for all worker builds CMD ["python3", "-m", "fastapi", "run", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file diff --git a/REFACTOR_PLAN.md b/REFACTOR_PLAN.md deleted file mode 100644 index e940ffd..0000000 --- a/REFACTOR_PLAN.md +++ /dev/null @@ -1,545 +0,0 @@ -# Detector Worker Refactoring Plan - -## Project Overview - -Transform the current monolithic structure (~4000 lines across `app.py` and `siwatsystem/pympta.py`) into a modular, maintainable system with clear separation of concerns. The goal is to make the sophisticated computer vision pipeline easily understandable for other engineers while maintaining all existing functionality. - -## Current System Flow Understanding - -### Validated System Flow -1. **WebSocket Connection** → Backend connects and sends `setSubscriptionList` -2. **Model Management** → Download unique `.mpta` files to `models/` and extract -3. **Tracking Phase** → Continuous tracking with `front_rear_detection_v1.pt` -4. **Validation Phase** → Validate stable car (not just passing by) -5. **Pipeline Execution** → - - Detect car with `yolo11m.pt` - - **Branch 1**: Front/rear detection → crop frontal → save to Redis + brand classification - - **Branch 2**: Body type classification from car crop -6. **Communication** → Send `imageDetection` → Backend generates `sessionId` → Fueling starts -7. **Post-Fueling** → Backend clears `sessionId` → Continue tracking same car to avoid re-pipeline - -### Core Responsibilities Identified -1. **WebSocket Communication** - Message handling and protocol compliance -2. **Stream Management** - RTSP/HTTP frame processing and buffering -3. **Model Management** - MPTA download, extraction, and loading -4. **Pipeline Configuration** - Parse `pipeline.json` and setup execution flow -5. **Vehicle Tracking** - Continuous tracking and car identification -6. **Validation Logic** - Stable car detection vs. passing-by cars -7. **Detection Pipeline** - Main ML pipeline with parallel branches -8. **Data Persistence** - Redis/PostgreSQL operations -9. **Session Management** - Handle session IDs and lifecycle - -## Proposed Directory Structure - -``` -core/ -├── communication/ -│ ├── __init__.py -│ ├── websocket.py # WebSocket message handling & protocol -│ ├── messages.py # Message types and validation -│ ├── models.py # Message data structures -│ └── state.py # Worker state management -├── streaming/ -│ ├── __init__.py -│ ├── manager.py # Stream coordination and lifecycle -│ ├── readers.py # RTSP/HTTP frame readers -│ └── buffers.py # Frame buffering and caching -├── models/ -│ ├── __init__.py -│ ├── manager.py # MPTA download and model loading -│ ├── pipeline.py # Pipeline.json parser and config -│ └── inference.py # YOLO model wrapper and optimization -├── tracking/ -│ ├── __init__.py -│ ├── tracker.py # Vehicle tracking with front_rear_detection_v1 -│ ├── validator.py # Stable car validation logic -│ └── integration.py # Tracking-pipeline integration -├── detection/ -│ ├── __init__.py -│ ├── pipeline.py # Main detection pipeline orchestration -│ └── branches.py # Parallel branch processing (brand/bodytype) -└── storage/ - ├── __init__.py - ├── redis.py # Redis operations and image storage - └── database.py # PostgreSQL operations (existing - will be moved) -``` - -## Implementation Strategy (Feature-by-Feature Testing) - -### Phase 1: Communication Layer -- WebSocket message handling (setSubscriptionList, sessionId management) -- HTTP API endpoints (camera image retrieval) -- Worker state reporting - -### Phase 2: Pipeline Configuration Reader -- Parse `pipeline.json` -- Model dependency resolution -- Branch configuration setup - -### Phase 3: Tracking System -- Continuous vehicle tracking -- Car identification and persistence - -### Phase 4: Tracking Validator -- Stable car detection logic -- Passing-by vs. fueling car differentiation - -### Phase 5: Model Pipeline Execution -- Main detection pipeline -- Parallel branch processing -- Redis/DB integration - -### Phase 6: Post-Session Tracking Validation -- Same car validation after sessionId cleared -- Prevent duplicate pipeline execution - -## Key Preservation Requirements -- **HTTP Endpoint**: `/camera/{camera_id}/image` must remain unchanged -- **WebSocket Protocol**: Full compliance with `worker.md` specification -- **MPTA Format**: Maintain compatibility with existing model archives -- **Database Schema**: Keep existing PostgreSQL structure -- **Redis Integration**: Preserve image storage and pub/sub functionality -- **Configuration**: Maintain `config.json` compatibility -- **Logging**: Preserve structured logging format - -## Expected Benefits -- **Maintainability**: Single responsibility modules (~200-400 lines each) -- **Testability**: Independent testing of each component -- **Readability**: Clear separation of concerns -- **Scalability**: Easy to extend and modify individual components -- **Documentation**: Self-documenting code structure - ---- - -# Comprehensive TODO List - -## ✅ Phase 1: Project Setup & Communication Layer - COMPLETED - -### 1.1 Project Structure Setup -- ✅ Create `core/` directory structure -- ✅ Create all module directories and `__init__.py` files -- ✅ Set up logging configuration for new modules -- ✅ Update imports in existing files to prepare for migration - -### 1.2 Communication Module (`core/communication/`) -- ✅ **Create `models.py`** - Message data structures - - ✅ Define WebSocket message models (SubscriptionList, StateReport, etc.) - - ✅ Add validation schemas for incoming messages - - ✅ Create response models for outgoing messages - -- ✅ **Create `messages.py`** - Message types and validation - - ✅ Implement message type constants - - ✅ Add message validation functions - - ✅ Create message builders for common responses - -- ✅ **Create `websocket.py`** - WebSocket message handling - - ✅ Extract WebSocket connection management from `app.py` - - ✅ Implement message routing and dispatching - - ✅ Add connection lifecycle management (connect, disconnect, reconnect) - - ✅ Handle `setSubscriptionList` message processing - - ✅ Handle `setSessionId` and `setProgressionStage` messages - - ✅ Handle `requestState` and `patchSessionResult` messages - -- ✅ **Create `state.py`** - Worker state management - - ✅ Extract state reporting logic from `app.py` - - ✅ Implement system metrics collection (CPU, memory, GPU) - - ✅ Manage active subscriptions state - - ✅ Handle session ID mapping and storage - -### 1.3 HTTP API Preservation -- ✅ **Preserve `/camera/{camera_id}/image` endpoint** - - ✅ Extract REST API logic from `app.py` - - ✅ Ensure frame caching mechanism works with new structure - - ✅ Maintain exact same response format and error handling - -### 1.4 Testing Phase 1 -- ✅ Test WebSocket connection and message handling -- ✅ Test HTTP API endpoint functionality -- ✅ Verify state reporting works correctly -- ✅ Test session management functionality - -### 1.5 Phase 1 Results -- ✅ **Modular Architecture**: Transformed ~900 lines into 4 focused modules (~200 lines each) -- ✅ **WebSocket Protocol**: Full compliance with worker.md specification -- ✅ **System Metrics**: Real-time CPU, memory, GPU monitoring -- ✅ **State Management**: Thread-safe subscription and session tracking -- ✅ **Backward Compatibility**: All existing endpoints preserved -- ✅ **Modern FastAPI**: Lifespan events, Pydantic v2 compatibility - -## ✅ Phase 2: Pipeline Configuration & Model Management - COMPLETED - -### 2.1 Models Module (`core/models/`) -- ✅ **Create `pipeline.py`** - Pipeline.json parser - - ✅ Extract pipeline configuration parsing from `pympta.py` - - ✅ Implement pipeline validation - - ✅ Add configuration schema validation - - ✅ Handle Redis and PostgreSQL configuration parsing - -- ✅ **Create `manager.py`** - MPTA download and model loading - - ✅ Extract MPTA download logic from `pympta.py` - - ✅ Implement ZIP extraction and validation - - ✅ Add model file management and caching - - ✅ Handle model loading with GPU optimization - - ✅ Implement model dependency resolution - -- ✅ **Create `inference.py`** - YOLO model wrapper - - ✅ Create unified YOLO model interface - - ✅ Add inference optimization and caching - - ✅ Implement batch processing capabilities - - ✅ Handle model switching and memory management - -### 2.2 Testing Phase 2 -- ✅ Test MPTA file download and extraction -- ✅ Test pipeline.json parsing and validation -- ✅ Test model loading with different configurations -- ✅ Verify GPU optimization works correctly - -### 2.3 Phase 2 Results -- ✅ **ModelManager**: Downloads, extracts, and manages MPTA files with model ID-based directory structure -- ✅ **PipelineParser**: Parses and validates pipeline.json with full support for Redis, PostgreSQL, tracking, and branches -- ✅ **YOLOWrapper**: Unified interface for YOLO models with caching, tracking, and classification support -- ✅ **Model Caching**: Shared model cache across instances to optimize memory usage -- ✅ **Dependency Resolution**: Automatically identifies and tracks all model file dependencies - -## ✅ Phase 3: Streaming System - COMPLETED - -### 3.1 Streaming Module (`core/streaming/`) -- ✅ **Create `readers.py`** - RTSP/HTTP frame readers - - ✅ Extract `frame_reader` function from `app.py` - - ✅ Extract `snapshot_reader` function from `app.py` - - ✅ Add connection management and retry logic - - ✅ Implement frame rate control and optimization - -- ✅ **Create `buffers.py`** - Frame buffering and caching - - ✅ Extract frame buffer management from `app.py` - - ✅ Implement efficient frame caching for REST API - - ✅ Add buffer size management and memory optimization - -- ✅ **Create `manager.py`** - Stream coordination - - ✅ Extract stream lifecycle management from `app.py` - - ✅ Implement shared stream optimization - - ✅ Add subscription reconciliation logic - - ✅ Handle stream sharing across multiple subscriptions - -### 3.2 Testing Phase 3 -- ✅ Test RTSP stream reading and buffering -- ✅ Test HTTP snapshot capture functionality -- ✅ Test shared stream optimization -- ✅ Verify frame caching for REST API access - -### 3.3 Phase 3 Results -- ✅ **RTSPReader**: OpenCV-based RTSP stream reader with automatic reconnection and frame callbacks -- ✅ **HTTPSnapshotReader**: Periodic HTTP snapshot capture with HTTPBasicAuth and HTTPDigestAuth support -- ✅ **FrameBuffer**: Thread-safe frame storage with automatic aging and cleanup -- ✅ **CacheBuffer**: Enhanced frame cache with cropping support and highest quality JPEG encoding (default quality=100) -- ✅ **StreamManager**: Complete stream lifecycle management with shared optimization and subscription reconciliation -- ✅ **Authentication Support**: Proper handling of credentials in URLs with automatic auth type detection -- ✅ **Real Camera Testing**: Verified with authenticated RTSP (1280x720) and HTTP snapshot (2688x1520) cameras -- ✅ **Production Ready**: Stable concurrent streaming from multiple camera sources -- ✅ **Dependencies**: Added opencv-python, numpy, and requests to requirements.txt - -### 3.4 Recent Streaming Enhancements (Post-Phase 3) -- ✅ **Format-Specific Optimization**: Tailored for 1280x720@6fps RTSP streams and 2560x1440 HTTP snapshots -- ✅ **H.264 Error Recovery**: Enhanced error handling for corrupted frames with automatic stream recovery -- ✅ **Frame Validation**: Implemented corruption detection using edge density analysis -- ✅ **Buffer Size Optimization**: Adjusted buffer limits to 3MB for RTSP frames (1280x720x3 bytes) -- ✅ **FFMPEG Integration**: Added environment variables to suppress verbose H.264 decoder errors -- ✅ **URL Preservation**: Maintained clean RTSP URLs without parameter injection -- ✅ **Type Detection**: Automatic stream type detection based on frame dimensions -- ✅ **Quality Settings**: Format-specific JPEG quality (90% for RTSP, 95% for HTTP) - -## ✅ Phase 4: Vehicle Tracking System - COMPLETED - -### 4.1 Tracking Module (`core/tracking/`) -- ✅ **Create `tracker.py`** - Vehicle tracking implementation (305 lines) - - ✅ Implement continuous tracking with configurable model (front_rear_detection_v1.pt) - - ✅ Add vehicle identification and persistence with TrackedVehicle dataclass - - ✅ Implement tracking state management with thread-safe operations - - ✅ Add bounding box tracking and motion analysis with position history - - ✅ Multi-class tracking support for complex detection scenarios - -- ✅ **Create `validator.py`** - Stable car validation (417 lines) - - ✅ Implement stable car detection algorithm with multiple validation criteria - - ✅ Add passing-by vs. fueling car differentiation using velocity and position analysis - - ✅ Implement validation thresholds and timing with configurable parameters - - ✅ Add confidence scoring for validation decisions with state history - - ✅ Advanced motion analysis with velocity smoothing and position variance - -- ✅ **Create `integration.py`** - Tracking-pipeline integration (547 lines) - - ✅ Connect tracking system with main pipeline through TrackingPipelineIntegration - - ✅ Handle tracking state transitions and session management - - ✅ Implement post-session tracking validation with cooldown periods - - ✅ Add same-car validation after sessionId cleared with 30-second cooldown - - ✅ Car abandonment detection with automatic timeout monitoring - - ✅ Mock detection system for backend communication - - ✅ Async pipeline execution with proper error handling - -### 4.2 Testing Phase 4 -- ✅ Test continuous vehicle tracking functionality -- ✅ Test stable car validation logic -- ✅ Test integration with existing pipeline -- ✅ Verify tracking performance and accuracy -- ✅ Test car abandonment detection with null detection messages -- ✅ Verify session management and progression stage handling - -### 4.3 Phase 4 Results -- ✅ **VehicleTracker**: Complete tracking implementation with YOLO tracking integration, position history, and stability calculations -- ✅ **StableCarValidator**: Sophisticated validation logic using velocity, position variance, and state consistency -- ✅ **TrackingPipelineIntegration**: Full integration with pipeline system including session management and async processing -- ✅ **StreamManager Integration**: Updated streaming manager to process tracking on every frame with proper threading -- ✅ **Thread-Safe Operations**: All tracking operations are thread-safe with proper locking mechanisms -- ✅ **Configurable Parameters**: All tracking parameters are configurable through pipeline.json -- ✅ **Session Management**: Complete session lifecycle management with post-fueling validation -- ✅ **Statistics and Monitoring**: Comprehensive statistics collection for tracking performance -- ✅ **Car Abandonment Detection**: Automatic detection when cars leave without fueling, sends `detection: null` to backend -- ✅ **Message Protocol**: Fixed JSON serialization to include `detection: null` for abandonment notifications -- ✅ **Streaming Optimization**: Enhanced RTSP/HTTP readers for 1280x720@6fps RTSP and 2560x1440 HTTP snapshots -- ✅ **Error Recovery**: Improved H.264 error handling and corrupted frame detection - -## ✅ Phase 5: Detection Pipeline System - COMPLETED - -### 5.1 Detection Module (`core/detection/`) ✅ -- ✅ **Create `pipeline.py`** - Main detection orchestration (574 lines) - - ✅ Extracted main pipeline execution from `pympta.py` with full orchestration - - ✅ Implemented detection flow coordination with async execution - - ✅ Added pipeline state management with comprehensive statistics - - ✅ Handled pipeline result aggregation with branch synchronization - - ✅ Redis and database integration with error handling - - ✅ Immediate and parallel action execution with template resolution - -- ✅ **Create `branches.py`** - Parallel branch processing (442 lines) - - ✅ Extracted parallel branch execution from `pympta.py` - - ✅ Implemented ThreadPoolExecutor-based parallel processing - - ✅ Added branch synchronization and result collection - - ✅ Handled branch failure and retry logic with graceful degradation - - ✅ Support for nested branches and model caching - - ✅ Both detection and classification model support - -### 5.2 Storage Module (`core/storage/`) ✅ -- ✅ **Create `redis.py`** - Redis operations (410 lines) - - ✅ Extracted Redis action execution from `pympta.py` - - ✅ Implemented async image storage with region cropping - - ✅ Added pub/sub messaging functionality with JSON support - - ✅ Handled Redis connection management and retry logic - - ✅ Added statistics tracking and health monitoring - - ✅ Support for various image formats (JPEG, PNG) with quality control - -- ✅ **Move `database.py`** - PostgreSQL operations (339 lines) - - ✅ Moved existing `archive/siwatsystem/database.py` to `core/storage/` - - ✅ Updated imports and integration points - - ✅ Ensured compatibility with new module structure - - ✅ Added session management and statistics methods - - ✅ Enhanced error handling and connection management - -### 5.3 Integration Updates ✅ -- ✅ **Updated `core/tracking/integration.py`** - - ✅ Added DetectionPipeline integration - - ✅ Replaced placeholder `_execute_pipeline` with real implementation - - ✅ Added detection pipeline initialization and cleanup - - ✅ Integrated with existing tracking system flow - - ✅ Maintained backward compatibility with test mode - -### 5.4 Testing Phase 5 ✅ -- ✅ Verified module imports work correctly -- ✅ All new modules follow established coding patterns -- ✅ Integration points properly connected -- ✅ Error handling and cleanup methods implemented -- ✅ Statistics and monitoring capabilities added - -### 5.5 Phase 5 Results ✅ -- ✅ **DetectionPipeline**: Complete detection orchestration with Redis/PostgreSQL integration, async execution, and comprehensive error handling -- ✅ **BranchProcessor**: Parallel branch execution with ThreadPoolExecutor, model caching, and nested branch support -- ✅ **RedisManager**: Async Redis operations with image storage, pub/sub messaging, and connection management -- ✅ **DatabaseManager**: Enhanced PostgreSQL operations with session management and statistics -- ✅ **Module Integration**: Seamless integration with existing tracking system while maintaining compatibility -- ✅ **Error Handling**: Comprehensive error handling and graceful degradation throughout all components -- ✅ **Performance**: Optimized parallel processing and caching for high-performance pipeline execution - -## ✅ Additional Implemented Features (Not in Original Plan) - -### License Plate Recognition Integration (`core/storage/license_plate.py`) ✅ -- ✅ **LicensePlateManager**: Subscribes to Redis channel `license_results` for external LPR service -- ✅ **Multi-format Support**: Handles various message formats from LPR service -- ✅ **Result Caching**: 5-minute TTL for license plate results -- ✅ **WebSocket Integration**: Sends combined `imageDetection` messages with license data -- ✅ **Asynchronous Processing**: Non-blocking Redis pub/sub listener - -### Advanced Session State Management (`core/communication/state.py`) ✅ -- ✅ **Session ID Mapping**: Per-display session identifier tracking -- ✅ **Progression Stage Tracking**: Workflow state per display (welcome, car_wait_staff, finished, cleared) -- ✅ **Thread-Safe Operations**: RLock-based synchronization for concurrent access -- ✅ **Comprehensive State Reporting**: Full system state for debugging - -### Car Abandonment Detection (`core/tracking/integration.py`) ✅ -- ✅ **Abandonment Monitoring**: Detects cars leaving without completing fueling -- ✅ **Timeout Configuration**: 3-second abandonment timeout -- ✅ **Null Detection Messages**: Sends `detection: null` to backend for abandoned cars -- ✅ **Automatic Cleanup**: Removes abandoned sessions from tracking - -### Enhanced Message Protocol (`core/communication/models.py`) ✅ -- ✅ **PatchSessionResult**: Session data patching support -- ✅ **SetProgressionStage**: Workflow stage management messages -- ✅ **Null Detection Handling**: Support for abandonment notifications -- ✅ **Complex Detection Structure**: Supports both classification and null states - -### Comprehensive Timeout and Cooldown Systems ✅ -- ✅ **Post-Session Cooldown**: 30-second cooldown after session clearing -- ✅ **Processing Cooldown**: 10-second cooldown for repeated processing -- ✅ **Abandonment Timeout**: 3-second timeout for car abandonment detection -- ✅ **Vehicle Expiration**: 2-second timeout for tracking cleanup -- ✅ **Stream Timeouts**: 30-second connection timeout management - -## 📋 Phase 6: Integration & Final Testing - -### 6.1 Main Application Refactoring -- [ ] **Refactor `app.py`** - - [ ] Remove extracted functionality - - [ ] Update to use new modular structure - - [ ] Maintain FastAPI application structure - - [ ] Update imports and dependencies - -- [ ] **Clean up `siwatsystem/pympta.py`** - - [ ] Remove extracted functionality - - [ ] Keep only necessary legacy compatibility code - - [ ] Update imports to use new modules - -### 6.2 Post-Session Tracking Validation -- [ ] Implement same-car validation after sessionId cleared -- [ ] Add logic to prevent duplicate pipeline execution -- [ ] Test tracking persistence through session lifecycle -- [ ] Verify correct behavior during edge cases - -### 6.3 Configuration & Documentation -- [ ] Update configuration handling for new structure -- [ ] Ensure `config.json` compatibility maintained -- [ ] Update logging configuration for all modules -- [ ] Add module-level documentation - -### 6.4 Comprehensive Testing -- [ ] **Integration Testing** - - [ ] Test complete system flow end-to-end - - [ ] Test all WebSocket message types - - [ ] Test HTTP API endpoints - - [ ] Test error handling and recovery - -- [ ] **Performance Testing** - - [ ] Verify system performance is maintained - - [ ] Test memory usage optimization - - [ ] Test GPU utilization efficiency - - [ ] Benchmark against original implementation - -- [ ] **Edge Case Testing** - - [ ] Test connection failures and reconnection - - [ ] Test model loading failures - - [ ] Test stream interruption handling - - [ ] Test concurrent subscription management - -### 6.5 Logging Optimization & Cleanup ✅ -- ✅ **Removed Debug Frame Saving** - - ✅ Removed hard-coded debug frame saving in `core/detection/pipeline.py` - - ✅ Removed hard-coded debug frame saving in `core/detection/branches.py` - - ✅ Eliminated absolute debug paths for production use - -- ✅ **Eliminated Test/Mock Functionality** - - ✅ Removed `save_frame_for_testing` function from `core/streaming/buffers.py` - - ✅ Removed `save_test_frames` configuration from `StreamConfig` - - ✅ Cleaned up test frame saving calls in stream manager - - ✅ Updated module exports to remove test functions - -- ✅ **Reduced Verbose Logging** - - ✅ Commented out verbose frame storage logging (every frame) - - ✅ Converted debug-level info logs to proper debug level - - ✅ Reduced repetitive frame dimension logging - - ✅ Maintained important model results and detection confidence logging - - ✅ Kept critical pipeline execution and error messages - -- ✅ **Production-Ready Logging** - - ✅ Clean startup and initialization messages - - ✅ Clear model loading and pipeline status - - ✅ Preserved detection results with confidence scores - - ✅ Maintained session management and tracking messages - - ✅ Kept important error and warning messages - -### 6.6 Final Cleanup -- [ ] Remove any remaining duplicate code -- [ ] Optimize imports across all modules -- [ ] Clean up temporary files and debugging code -- [ ] Update project documentation - -## 📋 Post-Refactoring Tasks - -### Documentation Updates -- [ ] Update `CLAUDE.md` with new architecture -- [ ] Create module-specific documentation -- [ ] Update installation and deployment guides -- [ ] Add troubleshooting guide for new structure - -### Code Quality -- [ ] Add type hints to all new modules -- [ ] Implement proper error handling patterns -- [ ] Add logging consistency across modules -- [ ] Ensure proper resource cleanup - -### Future Enhancements (Optional) -- [ ] Add unit tests for each module -- [ ] Implement monitoring and metrics collection -- [ ] Add configuration validation -- [ ] Consider adding dependency injection container - ---- - -## Success Criteria - -✅ **Modularity**: Each module has a single, clear responsibility -✅ **Testability**: Each phase can be tested independently -✅ **Maintainability**: Code is easy to understand and modify -✅ **Compatibility**: All existing functionality preserved -✅ **Performance**: System performance is maintained or improved -✅ **Documentation**: Clear documentation for new architecture - -## Risk Mitigation - -- **Feature-by-feature testing** ensures functionality is preserved at each step -- **Gradual migration** minimizes risk of breaking existing functionality -- **Preserve critical interfaces** (WebSocket protocol, HTTP endpoints) -- **Maintain backward compatibility** with existing configurations -- **Comprehensive testing** at each phase before proceeding - ---- - -## 🎯 Current Status Summary - -### ✅ Completed Phases (95% Complete) -- **Phase 1**: Communication Layer - ✅ COMPLETED -- **Phase 2**: Pipeline Configuration & Model Management - ✅ COMPLETED -- **Phase 3**: Streaming System - ✅ COMPLETED -- **Phase 4**: Vehicle Tracking System - ✅ COMPLETED -- **Phase 5**: Detection Pipeline System - ✅ COMPLETED -- **Additional Features**: License Plate Recognition, Car Abandonment, Session Management - ✅ COMPLETED - -### 📋 Remaining Work (5%) -- **Phase 6**: Final Integration & Testing - - Main application cleanup (`app.py` and `pympta.py`) - - Comprehensive integration testing - - Performance benchmarking - - Documentation updates - -### 🚀 Production Ready Features -- ✅ **Modular Architecture**: ~4000 lines refactored into 20+ focused modules -- ✅ **WebSocket Protocol**: Full compliance with all message types -- ✅ **License Plate Recognition**: External LPR service integration via Redis -- ✅ **Car Abandonment Detection**: Automatic detection and notification -- ✅ **Session Management**: Complete lifecycle with progression stages -- ✅ **Parallel Processing**: ThreadPoolExecutor for branch execution -- ✅ **Redis Integration**: Pub/sub, image storage, LPR subscription -- ✅ **PostgreSQL Integration**: Automatic schema management, combined updates -- ✅ **Stream Optimization**: Shared streams, format-specific handling -- ✅ **Error Recovery**: H.264 corruption detection, automatic reconnection -- ✅ **Production Logging**: Clean, informative logging without debug clutter - -### 📊 Metrics -- **Modules Created**: 20+ specialized modules -- **Lines Per Module**: ~200-500 (highly maintainable) -- **Test Coverage**: Feature-by-feature validation completed -- **Performance**: Maintained or improved from original implementation -- **Backward Compatibility**: 100% preserved \ No newline at end of file diff --git a/app.py b/app.py index 21d89db..0ac238f 100644 --- a/app.py +++ b/app.py @@ -1,576 +1,2348 @@ -""" -Detector Worker - Main FastAPI Application -Refactored modular architecture for computer vision pipeline processing. -""" -import json -import logging +from typing import Any, Dict import os +import json import time +import queue +import torch import cv2 -from contextlib import asynccontextmanager -from typing import Dict, Any +import numpy as np +import base64 +import logging +import threading +import requests +import asyncio +import psutil +import zipfile +import ssl +import urllib3 +import subprocess +import tempfile +import redis +from urllib.parse import urlparse +from requests.adapters import HTTPAdapter +from urllib3.util.ssl_ import create_urllib3_context from fastapi import FastAPI, WebSocket, HTTPException +from fastapi.websockets import WebSocketDisconnect from fastapi.responses import Response +from websockets.exceptions import ConnectionClosedError +from ultralytics import YOLO -# Import new modular communication system -from core.communication.websocket import websocket_endpoint -from core.communication.state import worker_state +# Import shared pipeline functions +from siwatsystem.pympta import load_pipeline_from_zip, run_pipeline, cleanup_camera_stability, cleanup_pipeline_node +from siwatsystem.model_registry import get_registry_status, cleanup_registry +from siwatsystem.mpta_manager import get_or_download_mpta, release_mpta, get_mpta_manager_status, cleanup_mpta_manager + +app = FastAPI() + +# Global dictionaries to keep track of models and streams +# "models" now holds a nested dict: { camera_id: { modelId: model_tree } } +models: Dict[str, Dict[str, Any]] = {} +streams: Dict[str, Dict[str, Any]] = {} +# Store session IDs per display +session_ids: Dict[str, int] = {} +# Track shared camera streams by camera URL +camera_streams: Dict[str, Dict[str, Any]] = {} +# Map subscriptions to their camera URL +subscription_to_camera: Dict[str, str] = {} +# Store latest frames for REST API access (separate from processing buffer) +latest_frames: Dict[str, Any] = {} +# Store cached detection dict after successful pipeline completion +cached_detections: Dict[str, Dict[str, Any]] = {} +# Enhanced caching system for LPR integration +session_detections: Dict[str, Dict[str, Any]] = {} # session_id -> detection data +session_to_camera: Dict[str, str] = {} # session_id -> camera_id +detection_timestamps: Dict[str, float] = {} # session_id -> timestamp (for cleanup) +# Track frame skipping for pipeline buffer after detection +frame_skip_flags: Dict[str, bool] = {} +# Track camera connection states for immediate error handling +camera_states: Dict[str, Dict[str, Any]] = {} +# Track session ID states and pipeline modes per camera +session_pipeline_states: Dict[str, Dict[str, Any]] = {} +# Store full pipeline results for caching +cached_full_pipeline_results: Dict[str, Dict[str, Any]] = {} + +with open("config.json", "r") as f: + config = json.load(f) + +poll_interval = config.get("poll_interval_ms", 100) +reconnect_interval = config.get("reconnect_interval_sec", 5) +TARGET_FPS = config.get("target_fps", 10) +poll_interval = 1000 / TARGET_FPS +logging.info(f"Poll interval: {poll_interval}ms") +max_streams = config.get("max_streams", 5) +max_retries = config.get("max_retries", 3) # Configure logging logging.basicConfig( - level=logging.DEBUG, + level=logging.INFO, # Set to INFO level for less verbose output format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", handlers=[ - logging.FileHandler("detector_worker.log"), - logging.StreamHandler() + logging.FileHandler("detector_worker.log"), # Write logs to a file + logging.StreamHandler() # Also output to console ] ) +# Create a logger specifically for this application logger = logging.getLogger("detector_worker") -logger.setLevel(logging.DEBUG) +logger.setLevel(logging.DEBUG) # Set app-specific logger to DEBUG level -# Frames are now stored in the shared cache buffer from core.streaming.buffers -# latest_frames = {} # Deprecated - using shared_cache_buffer instead +# Create WebSocket communication logger +ws_logger = logging.getLogger("websocket_comm") +ws_logger.setLevel(logging.INFO) +ws_handler = logging.FileHandler("websocket_comm.log", encoding='utf-8') +ws_formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s") +ws_handler.setFormatter(ws_formatter) +ws_logger.addHandler(ws_handler) +ws_logger.propagate = False # Don't propagate to root logger +# Ensure all other libraries (including root) use at least INFO level +logging.getLogger().setLevel(logging.INFO) -# Health monitoring recovery handlers -def _handle_stream_restart_recovery(component: str, details: Dict[str, Any]) -> bool: - """Handle stream restart recovery at the application level.""" - try: - from core.streaming.manager import shared_stream_manager +logger.info("Starting detector worker application") +logger.info(f"Configuration: Target FPS: {TARGET_FPS}, Max streams: {max_streams}, Max retries: {max_retries}") +ws_logger.info("WebSocket communication logging started - TX/RX format") +logger.info("WebSocket communication will be logged to websocket_comm.log") - # Extract camera ID from component name (e.g., "stream_cam-001" -> "cam-001") - if component.startswith("stream_"): - camera_id = component[7:] # Remove "stream_" prefix - else: - camera_id = component - - logger.info(f"Attempting stream restart recovery for {camera_id}") - - # Find and restart the subscription - subscriptions = shared_stream_manager.get_all_subscriptions() - for sub_info in subscriptions: - if sub_info.camera_id == camera_id: - # Remove and re-add the subscription - shared_stream_manager.remove_subscription(sub_info.subscription_id) - time.sleep(1.0) # Brief delay - - # Re-add subscription - success = shared_stream_manager.add_subscription( - sub_info.subscription_id, - sub_info.stream_config, - sub_info.crop_coords, - sub_info.model_id, - sub_info.model_url, - sub_info.tracking_integration - ) - - if success: - logger.info(f"Stream restart recovery successful for {camera_id}") - return True - else: - logger.error(f"Stream restart recovery failed for {camera_id}") - return False - - logger.warning(f"No subscription found for camera {camera_id} during recovery") - return False - - except Exception as e: - logger.error(f"Error in stream restart recovery for {component}: {e}") - return False - - -def _handle_stream_reconnect_recovery(component: str, details: Dict[str, Any]) -> bool: - """Handle stream reconnect recovery at the application level.""" - try: - from core.streaming.manager import shared_stream_manager - - # Extract camera ID from component name - if component.startswith("stream_"): - camera_id = component[7:] - else: - camera_id = component - - logger.info(f"Attempting stream reconnect recovery for {camera_id}") - - # For reconnect, we just need to trigger the stream's internal reconnect - # The stream readers handle their own reconnection logic - active_cameras = shared_stream_manager.get_active_cameras() - - if camera_id in active_cameras: - logger.info(f"Stream reconnect recovery triggered for {camera_id}") - return True - else: - logger.warning(f"Camera {camera_id} not found in active cameras during reconnect recovery") - return False - - except Exception as e: - logger.error(f"Error in stream reconnect recovery for {component}: {e}") - return False - -# Lifespan event handler (modern FastAPI approach) -@asynccontextmanager -async def lifespan(app: FastAPI): - """Application lifespan management.""" - # Startup - logger.info("Detector Worker started successfully") - - # Initialize health monitoring system - try: - from core.monitoring.health import health_monitor - from core.monitoring.stream_health import stream_health_tracker - from core.monitoring.thread_health import thread_health_monitor - from core.monitoring.recovery import recovery_manager - - # Start health monitoring - health_monitor.start() - logger.info("Health monitoring system started") - - # Register recovery handlers for stream management - from core.streaming.manager import shared_stream_manager - recovery_manager.register_recovery_handler( - "restart_stream", - _handle_stream_restart_recovery - ) - recovery_manager.register_recovery_handler( - "reconnect", - _handle_stream_reconnect_recovery - ) - - logger.info("Recovery handlers registered") - - except Exception as e: - logger.error(f"Failed to initialize health monitoring: {e}") - - logger.info("WebSocket endpoint available at: ws://0.0.0.0:8001/") - logger.info("HTTP camera endpoint available at: http://0.0.0.0:8001/camera/{camera_id}/image") - logger.info("Health check available at: http://0.0.0.0:8001/health") - logger.info("Detailed health monitoring available at: http://0.0.0.0:8001/health/detailed") - logger.info("Ready and waiting for backend WebSocket connections") - - yield - - # Shutdown - logger.info("Detector Worker shutting down...") - - # Stop health monitoring - try: - from core.monitoring.health import health_monitor - health_monitor.stop() - logger.info("Health monitoring system stopped") - except Exception as e: - logger.error(f"Error stopping health monitoring: {e}") - - # Clear all state - worker_state.set_subscriptions([]) - worker_state.session_ids.clear() - worker_state.progression_stages.clear() - # latest_frames.clear() # No longer needed - frames are in shared_cache_buffer - logger.info("Detector Worker shutdown complete") - -# Create FastAPI application with detailed WebSocket logging -app = FastAPI(title="Detector Worker", version="2.0.0", lifespan=lifespan) - -# Add middleware to log all requests -@app.middleware("http") -async def log_requests(request, call_next): - start_time = time.time() - response = await call_next(request) - process_time = time.time() - start_time - logger.debug(f"HTTP {request.method} {request.url} - {response.status_code} ({process_time:.3f}s)") - return response - -# Load configuration -config_path = "config.json" -if os.path.exists(config_path): - with open(config_path, "r") as f: - config = json.load(f) - logger.info(f"Loaded configuration from {config_path}") -else: - # Default configuration - config = { - "poll_interval_ms": 100, - "reconnect_interval_sec": 5, - "target_fps": 10, - "max_streams": 20, - "max_retries": 3 - } - logger.warning(f"Configuration file {config_path} not found, using defaults") - -# Ensure models directory exists +# Ensure the models directory exists os.makedirs("models", exist_ok=True) logger.info("Ensured models directory exists") -# Stream manager already initialized at module level with max_streams=20 -# Calling initialize_stream_manager() creates a NEW instance, breaking references -# from core.streaming import initialize_stream_manager -# initialize_stream_manager(max_streams=config.get('max_streams', 10)) -logger.info(f"Using stream manager with max_streams=20 (module-level initialization)") +# Constants for heartbeat and timeouts +HEARTBEAT_INTERVAL = 2 # seconds -# Frames are now stored in the shared cache buffer from core.streaming.buffers -# latest_frames = {} # Deprecated - using shared_cache_buffer instead - -logger.info("Starting detector worker application (refactored)") -logger.info(f"Configuration: Target FPS: {config.get('target_fps', 10)}, " - f"Max streams: {config.get('max_streams', 5)}, " - f"Max retries: {config.get('max_retries', 3)}") - - -@app.websocket("/") -async def websocket_handler(websocket: WebSocket): - """ - Main WebSocket endpoint for backend communication. - Handles all protocol messages according to worker.md specification. - """ - client_info = f"{websocket.client.host}:{websocket.client.port}" if websocket.client else "unknown" - logger.info(f"[RX ← Backend] New WebSocket connection request from {client_info}") +# Global Redis connection for LPR integration +redis_client_global = None +lpr_listener_thread = None +cleanup_timer_thread = None +lpr_integration_started = False +# Redis connection helper functions +def get_redis_config_from_model(camera_id: str) -> Dict[str, Any]: + """Extract Redis configuration from loaded model pipeline""" try: - await websocket_endpoint(websocket) + for model_id, model_tree in models.get(camera_id, {}).items(): + if hasattr(model_tree, 'get') and 'redis_client' in model_tree: + # Extract config from existing Redis client + client = model_tree['redis_client'] + if client: + return { + 'host': client.connection_pool.connection_kwargs['host'], + 'port': client.connection_pool.connection_kwargs['port'], + 'password': client.connection_pool.connection_kwargs.get('password'), + 'db': client.connection_pool.connection_kwargs.get('db', 0) + } except Exception as e: - logger.error(f"WebSocket handler error for {client_info}: {e}", exc_info=True) + logger.debug(f"Could not extract Redis config from model: {e}") + + # Fallback - try to read from pipeline.json directly + try: + pipeline_dirs = [] + models_dir = "models" + if os.path.exists(models_dir): + for root, dirs, files in os.walk(models_dir): + if "pipeline.json" in files: + with open(os.path.join(root, "pipeline.json"), 'r') as f: + config = json.load(f) + if 'redis' in config: + return config['redis'] + except Exception as e: + logger.debug(f"Could not read Redis config from pipeline.json: {e}") + + return None +def create_redis_connection() -> redis.Redis: + """Create Redis connection using config from pipeline""" + global redis_client_global + + if redis_client_global is not None: + try: + redis_client_global.ping() + return redis_client_global + except: + redis_client_global = None + + # Find any camera with a loaded model to get Redis config + redis_config = None + for camera_id in models.keys(): + redis_config = get_redis_config_from_model(camera_id) + if redis_config: + break + + if not redis_config: + logger.error("No Redis configuration found in any loaded models") + return None + + try: + redis_client_global = redis.Redis( + host=redis_config['host'], + port=redis_config['port'], + password=redis_config.get('password'), + db=redis_config.get('db', 0), + decode_responses=True, + socket_connect_timeout=5, + socket_timeout=5 + ) + redis_client_global.ping() + logger.info(f"✅ Connected to Redis for LPR at {redis_config['host']}:{redis_config['port']}") + return redis_client_global + except Exception as e: + logger.error(f"❌ Failed to connect to Redis for LPR: {e}") + redis_client_global = None + return None + +# LPR Integration Functions +def process_license_result(lpr_data: Dict[str, Any]): + """Process incoming LPR result and update backend""" + try: + # Enhanced debugging for LPR data reception + logger.info("=" * 60) + logger.info("🚗 LPR SERVICE DATA RECEIVED") + logger.info("=" * 60) + logger.info(f"📥 Raw LPR data: {json.dumps(lpr_data, indent=2)}") + + session_id = str(lpr_data.get('session_id', '')) + license_text = lpr_data.get('license_character', '') + + logger.info(f"🔍 Extracted session_id: '{session_id}'") + logger.info(f"🔍 Extracted license_character: '{license_text}'") + logger.info(f"📊 Current cached sessions count: {len(session_detections)}") + logger.info(f"📊 Available session IDs: {list(session_detections.keys())}") + + # Find cached detection by session_id + if session_id not in session_detections: + logger.warning("❌ LPR SESSION ID NOT FOUND!") + logger.warning(f" Looking for session_id: '{session_id}'") + logger.warning(f" Available sessions: {list(session_detections.keys())}") + logger.warning(f" Session count: {len(session_detections)}") + + # Additional debugging - show session timestamps + if session_detections: + logger.warning("📅 Available session details:") + for sid, timestamp in detection_timestamps.items(): + age = time.time() - timestamp + camera = session_to_camera.get(sid, 'unknown') + logger.warning(f" Session {sid}: camera={camera}, age={age:.1f}s") + else: + logger.warning(" No cached sessions available - worker may not have processed any detections yet") + + logger.warning("💡 Possible causes:") + logger.warning(" 1. Session expired (TTL: 10 minutes)") + logger.warning(" 2. Session ID mismatch between detection and LPR service") + logger.warning(" 3. Detection was not cached (no sessionId from backend)") + logger.warning(" 4. Worker restarted after detection but before LPR result") + return + + # Get the original detection data + detection_data = session_detections[session_id].copy() + camera_id = session_to_camera.get(session_id, 'unknown') + + logger.info("✅ LPR SESSION FOUND!") + logger.info(f" 📹 Camera ID: {camera_id}") + logger.info(f" ⏰ Session age: {time.time() - detection_timestamps.get(session_id, 0):.1f} seconds") + + # Show original detection structure before update + original_license = detection_data.get('data', {}).get('detection', {}).get('licensePlateText') + logger.info(f" 🔍 Original licensePlateText: {original_license}") + logger.info(f" 🆕 New licensePlateText: '{license_text}'") + + # Update licensePlateText in detection + if 'data' in detection_data and 'detection' in detection_data['data']: + detection_data['data']['detection']['licensePlateText'] = license_text + + logger.info("🎯 LICENSE PLATE UPDATE SUCCESS!") + logger.info(f" ✅ Updated detection for session {session_id}") + logger.info(f" ✅ Set licensePlateText = '{license_text}'") + + # Show full detection structure after update + detection_dict = detection_data['data']['detection'] + logger.info("📋 Updated detection dictionary:") + logger.info(f" carModel: {detection_dict.get('carModel')}") + logger.info(f" carBrand: {detection_dict.get('carBrand')}") + logger.info(f" bodyType: {detection_dict.get('bodyType')}") + logger.info(f" licensePlateText: {detection_dict.get('licensePlateText')} ← UPDATED") + logger.info(f" licensePlateConfidence: {detection_dict.get('licensePlateConfidence')}") + else: + logger.error("❌ INVALID DETECTION DATA STRUCTURE!") + logger.error(f" Session {session_id} has malformed detection data") + logger.error(f" Detection data keys: {list(detection_data.keys())}") + if 'data' in detection_data: + logger.error(f" Data keys: {list(detection_data['data'].keys())}") + return + + # Update timestamp to indicate this is an LPR update + detection_data['timestamp'] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + + # Update all caches with new data + session_detections[session_id] = detection_data.copy() + cached_detections[camera_id] = detection_data.copy() + + # CRITICAL: Also update the pipeline state cached detection dict (used by lightweight mode) + if camera_id in session_pipeline_states: + pipeline_state = session_pipeline_states[camera_id] + current_cached_dict = pipeline_state.get("cached_detection_dict", {}) + + # Update the pipeline cached detection dict with new license plate + updated_dict = current_cached_dict.copy() if current_cached_dict else {} + updated_dict['licensePlateText'] = license_text + + pipeline_state["cached_detection_dict"] = updated_dict + logger.info(f"✅ LPR: Updated pipeline state cached_detection_dict for camera {camera_id}") + logger.debug(f"🔍 Pipeline cached dict now: {updated_dict}") + else: + logger.warning(f"⚠️ Camera {camera_id} not found in session_pipeline_states - pipeline cache not updated") + + logger.info("📡 SENDING UPDATED DETECTION TO BACKEND") + logger.info(f" 📹 Camera ID: {camera_id}") + logger.info(f" 📨 Updated licensePlateText: '{license_text}'") + logger.info(" 🔄 Updated both cache systems:") + logger.info(f" 1️⃣ cached_detections[{camera_id}] ✅") + logger.info(f" 2️⃣ session_pipeline_states[{camera_id}].cached_detection_dict ✅") + + # Log the full message being sent + logger.info("📋 Updated detection data in cache:") + logger.info(json.dumps(detection_data, indent=2)) + + logger.info("✅ ALL CACHES UPDATED!") + logger.info(f" 🎯 Lightweight mode will now use updated licensePlateText") + logger.info(f" 📤 Backend will receive: licensePlateText = '{license_text}'") + logger.info(" 🔄 Both cache systems synchronized with LPR data") + + logger.info("=" * 60) + logger.info("🏁 LPR PROCESSING COMPLETE") + logger.info(f" Session: {session_id}") + logger.info(f" License: '{license_text}'") + logger.info(f" Status: ✅ SUCCESS - DETECTION CACHE UPDATED") + logger.info("=" * 60) + + except Exception as e: + logger.error("=" * 60) + logger.error("❌ LPR PROCESSING FAILED") + logger.error("=" * 60) + logger.error(f"Error: {e}") + import traceback + logger.error(f"Traceback: {traceback.format_exc()}") + logger.error("=" * 60) + +# LPR integration now uses cached detection mechanism instead of direct WebSocket sending + +def license_results_listener(): + """Background thread to listen for LPR results from Redis""" + logger.info("🎧 Starting LPR listener thread...") + + while True: + try: + redis_client = create_redis_connection() + if not redis_client: + logger.error("❌ No Redis connection available for LPR listener") + time.sleep(10) + continue + + pubsub = redis_client.pubsub() + pubsub.subscribe("license_results") + logger.info("✅ LPR listener subscribed to 'license_results' channel") + + for message in pubsub.listen(): + try: + if message['type'] == 'message': + logger.info("🔔 REDIS MESSAGE RECEIVED!") + logger.info(f" 📡 Channel: {message['channel']}") + logger.info(f" 📥 Raw data: {message['data']}") + logger.info(f" 📏 Data size: {len(str(message['data']))} bytes") + + try: + lpr_data = json.loads(message['data']) + logger.info("✅ JSON parsing successful") + logger.info("🏁 Starting LPR processing...") + process_license_result(lpr_data) + logger.info("✅ LPR processing completed") + except json.JSONDecodeError as e: + logger.error("❌ JSON PARSING FAILED!") + logger.error(f" Error: {e}") + logger.error(f" Raw data: {message['data']}") + logger.error(f" Data type: {type(message['data'])}") + except Exception as e: + logger.error("❌ LPR PROCESSING ERROR!") + logger.error(f" Error: {e}") + import traceback + logger.error(f" Traceback: {traceback.format_exc()}") + elif message['type'] == 'subscribe': + logger.info(f"📡 LPR listener subscribed to channel: {message['channel']}") + logger.info("🎧 Ready to receive license plate results...") + elif message['type'] == 'unsubscribe': + logger.warning(f"📡 LPR listener unsubscribed from channel: {message['channel']}") + else: + logger.debug(f"📡 Redis message type: {message['type']}") + + except Exception as e: + logger.error(f"❌ Error in LPR message processing loop: {e}") + break + + except redis.exceptions.ConnectionError as e: + logger.error(f"❌ Redis connection lost in LPR listener: {e}") + time.sleep(5) # Wait before reconnecting + except Exception as e: + logger.error(f"❌ Unexpected error in LPR listener: {e}") + time.sleep(10) + + logger.warning("🛑 LPR listener thread stopped") + +def cleanup_expired_sessions(): + """Remove sessions older than TTL (10 minutes)""" + try: + current_time = time.time() + ttl_seconds = 600 # 10 minutes + + expired_sessions = [ + session_id for session_id, timestamp in detection_timestamps.items() + if current_time - timestamp > ttl_seconds + ] + + if expired_sessions: + logger.info(f"🧹 Cleaning up {len(expired_sessions)} expired sessions") + + for session_id in expired_sessions: + session_detections.pop(session_id, None) + camera_id = session_to_camera.pop(session_id, None) + detection_timestamps.pop(session_id, None) + logger.debug(f"Cleaned up expired session: {session_id} (camera: {camera_id})") + + else: + logger.debug(f"🧹 No expired sessions to clean up ({len(detection_timestamps)} active)") + + except Exception as e: + logger.error(f"❌ Error in session cleanup: {e}") + +def cleanup_timer(): + """Background thread for periodic session cleanup""" + logger.info("⏰ Starting session cleanup timer thread...") + + while True: + try: + time.sleep(120) # Run cleanup every 2 minutes + cleanup_expired_sessions() + except Exception as e: + logger.error(f"❌ Error in cleanup timer: {e}") + time.sleep(120) + +def start_lpr_integration(): + """Start LPR integration threads""" + global lpr_listener_thread, cleanup_timer_thread + + # Start LPR listener thread + lpr_listener_thread = threading.Thread(target=license_results_listener, daemon=True, name="LPR-Listener") + lpr_listener_thread.start() + logger.info("✅ LPR listener thread started") + + # Start cleanup timer thread + cleanup_timer_thread = threading.Thread(target=cleanup_timer, daemon=True, name="Session-Cleanup") + cleanup_timer_thread.start() + logger.info("✅ Session cleanup timer thread started") + +WORKER_TIMEOUT_MS = 10000 +logger.debug(f"Heartbeat interval set to {HEARTBEAT_INTERVAL} seconds") + +# Locks for thread-safe operations +streams_lock = threading.Lock() +models_lock = threading.Lock() +logger.debug("Initialized thread locks") + + +# Add helper to fetch snapshot image from HTTP/HTTPS URL +def fetch_snapshot(url: str): + try: + from requests.auth import HTTPBasicAuth, HTTPDigestAuth + import requests.adapters + import urllib3 + + # Parse URL to extract credentials + parsed = urlparse(url) + + # Prepare headers - some cameras require User-Agent and specific headers + headers = { + 'User-Agent': 'Mozilla/5.0 (compatible; DetectorWorker/1.0)', + 'Accept': 'image/jpeg,image/*,*/*', + 'Connection': 'close', + 'Cache-Control': 'no-cache' + } + + # Create a session with custom adapter for better connection handling + session = requests.Session() + adapter = requests.adapters.HTTPAdapter( + pool_connections=1, + pool_maxsize=1, + max_retries=urllib3.util.retry.Retry( + total=2, + backoff_factor=0.1, + status_forcelist=[500, 502, 503, 504] + ) + ) + session.mount('http://', adapter) + session.mount('https://', adapter) + + # Reconstruct URL without credentials + clean_url = f"{parsed.scheme}://{parsed.hostname}" + if parsed.port: + clean_url += f":{parsed.port}" + clean_url += parsed.path + if parsed.query: + clean_url += f"?{parsed.query}" + + auth = None + response = None + + if parsed.username and parsed.password: + # Try HTTP Digest authentication first (common for IP cameras) + try: + auth = HTTPDigestAuth(parsed.username, parsed.password) + response = session.get(clean_url, auth=auth, headers=headers, timeout=(5, 15), stream=True) + if response.status_code == 200: + logger.debug(f"Successfully authenticated using HTTP Digest for {clean_url}") + elif response.status_code == 401: + # If Digest fails, try Basic auth + logger.debug(f"HTTP Digest failed, trying Basic auth for {clean_url}") + auth = HTTPBasicAuth(parsed.username, parsed.password) + response = session.get(clean_url, auth=auth, headers=headers, timeout=(5, 15), stream=True) + if response.status_code == 200: + logger.debug(f"Successfully authenticated using HTTP Basic for {clean_url}") + except Exception as auth_error: + logger.debug(f"Authentication setup error: {auth_error}") + # Fallback to original URL with embedded credentials + response = session.get(url, headers=headers, timeout=(5, 15), stream=True) + else: + # No credentials in URL, make request as-is + response = session.get(url, headers=headers, timeout=(5, 15), stream=True) + + if response and response.status_code == 200: + # Read content with size limit to prevent memory issues + content = b'' + max_size = 10 * 1024 * 1024 # 10MB limit + for chunk in response.iter_content(chunk_size=8192): + content += chunk + if len(content) > max_size: + logger.error(f"Snapshot too large (>{max_size} bytes) from {clean_url}") + return None + + # Convert response content to numpy array + nparr = np.frombuffer(content, np.uint8) + # Decode image + frame = cv2.imdecode(nparr, cv2.IMREAD_COLOR) + if frame is not None: + logger.debug(f"Successfully fetched snapshot from {clean_url}, shape: {frame.shape}, size: {len(content)} bytes") + return frame + else: + logger.error(f"Failed to decode image from snapshot URL: {clean_url} (content size: {len(content)} bytes)") + return None + elif response: + logger.error(f"Failed to fetch snapshot (status code {response.status_code}): {clean_url}") + # Log response headers and first part of content for debugging + logger.debug(f"Response headers: {dict(response.headers)}") + if len(response.content) < 1000: + logger.debug(f"Response content: {response.content[:500]}") + return None + else: + logger.error(f"No response received from snapshot URL: {clean_url}") + return None + except requests.exceptions.Timeout as e: + logger.error(f"Timeout fetching snapshot from {url}: {str(e)}") + return None + except requests.exceptions.ConnectionError as e: + logger.error(f"Connection error fetching snapshot from {url}: {str(e)}") + return None + except Exception as e: + logger.error(f"Exception fetching snapshot from {url}: {str(e)}", exc_info=True) + return None + +# Helper to get crop coordinates from stream +def get_crop_coords(stream): + return { + "cropX1": stream.get("cropX1"), + "cropY1": stream.get("cropY1"), + "cropX2": stream.get("cropX2"), + "cropY2": stream.get("cropY2") + } + +# Camera state management functions +def set_camera_connected(camera_id, connected=True, error_msg=None): + """Set camera connection state and track error information""" + current_time = time.time() + + if camera_id not in camera_states: + camera_states[camera_id] = { + "connected": True, + "last_error": None, + "last_error_time": None, + "consecutive_failures": 0, + "disconnection_notified": False + } + + state = camera_states[camera_id] + was_connected = state["connected"] + + if connected: + state["connected"] = True + state["consecutive_failures"] = 0 + state["disconnection_notified"] = False + if not was_connected: + logger.info(f"📶 CAMERA RECONNECTED: {camera_id}") + else: + state["connected"] = False + state["last_error"] = error_msg + state["last_error_time"] = current_time + state["consecutive_failures"] += 1 + + # Distinguish between temporary and permanent disconnection + is_permanent = state["consecutive_failures"] >= 3 + + if was_connected and is_permanent: + logger.error(f"📵 CAMERA DISCONNECTED: {camera_id} - {error_msg} (consecutive failures: {state['consecutive_failures']})") + logger.info(f"🚨 CAMERA ERROR DETECTED - Will send detection: null to reset backend session for {camera_id}") + +def is_camera_connected(camera_id): + """Check if camera is currently connected""" + return camera_states.get(camera_id, {}).get("connected", True) + +def should_notify_disconnection(camera_id): + """Check if we should notify backend about disconnection""" + state = camera_states.get(camera_id, {}) + is_disconnected = not state.get("connected", True) + not_yet_notified = not state.get("disconnection_notified", False) + has_enough_failures = state.get("consecutive_failures", 0) >= 3 + + return is_disconnected and not_yet_notified and has_enough_failures + +def mark_disconnection_notified(camera_id): + """Mark that we've notified backend about this disconnection""" + if camera_id in camera_states: + camera_states[camera_id]["disconnection_notified"] = True + logger.debug(f"Marked disconnection notification sent for camera {camera_id}") + +def get_or_init_session_pipeline_state(camera_id): + """Get or initialize session pipeline state for a camera""" + if camera_id not in session_pipeline_states: + session_pipeline_states[camera_id] = { + "mode": "validation_detecting", # "validation_detecting", "send_detections", "waiting_for_session_id", "full_pipeline", "lightweight", "car_gone_waiting" + "session_id_received": False, + "full_pipeline_completed": False, + "absence_counter": 0, + "validation_counter": 0, # Counter for validation phase + "validation_threshold": 4, # Default validation threshold + "max_absence_frames": 3, + "yolo_inference_enabled": True, # Controls whether to run YOLO inference + "cached_detection_dict": None, # Cached detection dict for lightweight mode + "stable_track_id": None, # The stable track ID we're monitoring + "validated_detection": None, # Stored detection result from validation phase for full_pipeline reuse + "progression_stage": None # Tracks current progression stage (welcome, car_wait_staff, car_fueling, car_waitpayment) + } + return session_pipeline_states[camera_id] + +def update_session_pipeline_mode(camera_id, new_mode, session_id=None): + """Update session pipeline mode and related state""" + state = get_or_init_session_pipeline_state(camera_id) + old_mode = state["mode"] + state["mode"] = new_mode + + # Reset counters based on mode transition + if new_mode == "validation_detecting": + # Transitioning to validation mode - reset both counters for fresh start + old_validation_counter = state.get("validation_counter", 0) + old_absence_counter = state.get("absence_counter", 0) + state["validation_counter"] = 0 + state["absence_counter"] = 0 + if old_validation_counter > 0 or old_absence_counter > 0: + logger.info(f"🧹 Camera {camera_id}: VALIDATION MODE RESET - validation_counter: {old_validation_counter}→0, absence_counter: {old_absence_counter}→0") + + if session_id: + state["session_id_received"] = True + state["absence_counter"] = 0 # Reset absence counter when session starts + + logger.info(f"📊 Camera {camera_id}: Pipeline mode changed from '{old_mode}' to '{new_mode}'") + return state + +#################################################### +# REST API endpoint for image retrieval +#################################################### +@app.get("/lpr/debug") +async def get_lpr_debug_info(): + """Debug endpoint to inspect LPR integration state""" + try: + return { + "status": "success", + "lpr_integration_started": lpr_integration_started, + "redis_connected": redis_client_global is not None and redis_client_global.ping() if redis_client_global else False, + "active_sessions": len(session_detections), + "session_details": { + session_id: { + "camera_id": session_to_camera.get(session_id, "unknown"), + "timestamp": detection_timestamps.get(session_id, 0), + "age_seconds": time.time() - detection_timestamps.get(session_id, time.time()), + "has_license": session_detections[session_id].get('data', {}).get('detection', {}).get('licensePlateText') is not None + } + for session_id in session_detections.keys() + }, + "thread_status": { + "lpr_listener_alive": lpr_listener_thread.is_alive() if lpr_listener_thread else False, + "cleanup_timer_alive": cleanup_timer_thread.is_alive() if cleanup_timer_thread else False, + "model_registry": get_registry_status(), + "mpta_manager": get_mpta_manager_status() + }, + "cached_detections_by_camera": list(cached_detections.keys()) + } + except Exception as e: + return { + "status": "error", + "error": str(e), + "lpr_integration_started": lpr_integration_started + } @app.get("/camera/{camera_id}/image") async def get_camera_image(camera_id: str): """ - HTTP endpoint to retrieve the latest frame from a camera as JPEG image. - - This endpoint is preserved for backward compatibility with existing systems. - - Args: - camera_id: The subscription identifier (e.g., "display-001;cam-001") - - Returns: - JPEG image as binary response - - Raises: - HTTPException: 404 if camera not found or no frame available - HTTPException: 500 if encoding fails + Get the current frame from a camera as JPEG image """ try: + # URL decode the camera_id to handle encoded characters like %3B for semicolon from urllib.parse import unquote - - # URL decode the camera_id to handle encoded characters original_camera_id = camera_id camera_id = unquote(camera_id) logger.debug(f"REST API request: original='{original_camera_id}', decoded='{camera_id}'") - - # Check if camera is in active subscriptions - subscription = worker_state.get_subscription(camera_id) - if not subscription: - logger.warning(f"Camera ID '{camera_id}' not found in active subscriptions") - available_cameras = list(worker_state.subscriptions.keys()) - logger.debug(f"Available cameras: {available_cameras}") - raise HTTPException( - status_code=404, - detail=f"Camera {camera_id} not found or not active" - ) - - # Extract actual camera_id from subscription identifier (displayId;cameraId) - # Frames are stored using just the camera_id part - actual_camera_id = camera_id.split(';')[-1] if ';' in camera_id else camera_id - - # Get frame from the shared cache buffer - from core.streaming.buffers import shared_cache_buffer - - # Only show buffer debug info if camera not found (to reduce log spam) - available_cameras = shared_cache_buffer.frame_buffer.get_camera_list() - - frame = shared_cache_buffer.get_frame(actual_camera_id) - if frame is None: - logger.warning(f"\033[93m[API] No frame for '{actual_camera_id}' - Available: {available_cameras}\033[0m") - raise HTTPException( - status_code=404, - detail=f"No frame available for camera {actual_camera_id}" - ) - - # Successful frame retrieval - log only occasionally to avoid spam - + + with streams_lock: + if camera_id not in streams: + logger.warning(f"Camera ID '{camera_id}' not found in streams. Current streams: {list(streams.keys())}") + raise HTTPException(status_code=404, detail=f"Camera {camera_id} not found or not active") + + # Check if we have a cached frame for this camera + if camera_id not in latest_frames: + logger.warning(f"No cached frame available for camera '{camera_id}'.") + raise HTTPException(status_code=404, detail=f"No frame available for camera {camera_id}") + + frame = latest_frames[camera_id] + logger.debug(f"Retrieved cached frame for camera '{camera_id}', frame shape: {frame.shape}") # Encode frame as JPEG success, buffer_img = cv2.imencode('.jpg', frame, [cv2.IMWRITE_JPEG_QUALITY, 85]) if not success: raise HTTPException(status_code=500, detail="Failed to encode image as JPEG") - + # Return image as binary response return Response(content=buffer_img.tobytes(), media_type="image/jpeg") - + except HTTPException: raise except Exception as e: logger.error(f"Error retrieving image for camera {camera_id}: {str(e)}", exc_info=True) raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}") +#################################################### +# Detection and frame processing functions +#################################################### +@app.websocket("/") +async def detect(websocket: WebSocket): + logger.info("WebSocket connection accepted") + persistent_data_dict = {} -@app.get("/session-image/{session_id}") -async def get_session_image(session_id: int): - """ - HTTP endpoint to retrieve the saved session image by session ID. - - Args: - session_id: The session ID to retrieve the image for - - Returns: - JPEG image as binary response - - Raises: - HTTPException: 404 if no image found for the session - HTTPException: 500 if reading image fails - """ - try: - from pathlib import Path - import glob - - # Images directory - images_dir = Path("images") - - if not images_dir.exists(): - logger.warning(f"Images directory does not exist") - raise HTTPException( - status_code=404, - detail=f"No images directory found" - ) - - # Search for files matching session ID pattern: {session_id}_* - pattern = str(images_dir / f"{session_id}_*.jpg") - matching_files = glob.glob(pattern) - - if not matching_files: - logger.warning(f"No image found for session {session_id}") - raise HTTPException( - status_code=404, - detail=f"No image found for session {session_id}" - ) - - # Get the most recent file if multiple exist - most_recent_file = max(matching_files, key=os.path.getmtime) - logger.info(f"Found session image for session {session_id}: {most_recent_file}") - - # Read the image file - image_data = open(most_recent_file, 'rb').read() - - # Return image as binary response - return Response(content=image_data, media_type="image/jpeg") - - except HTTPException: - raise - except Exception as e: - logger.error(f"Error retrieving session image for session {session_id}: {str(e)}", exc_info=True) - raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}") - - -@app.get("/health") -async def health_check(): - """Health check endpoint for monitoring.""" - return { - "status": "healthy", - "version": "2.0.0", - "active_subscriptions": len(worker_state.subscriptions), - "active_sessions": len(worker_state.session_ids) - } - - -@app.get("/health/detailed") -async def detailed_health_check(): - """Comprehensive health status with detailed monitoring data.""" - try: - from core.monitoring.health import health_monitor - from core.monitoring.stream_health import stream_health_tracker - from core.monitoring.thread_health import thread_health_monitor - from core.monitoring.recovery import recovery_manager - - # Get comprehensive health status - overall_health = health_monitor.get_health_status() - stream_metrics = stream_health_tracker.get_all_metrics() - thread_info = thread_health_monitor.get_all_thread_info() - recovery_stats = recovery_manager.get_recovery_stats() - - return { - "timestamp": time.time(), - "overall_health": overall_health, - "stream_metrics": stream_metrics, - "thread_health": thread_info, - "recovery_stats": recovery_stats, - "system_info": { - "active_subscriptions": len(worker_state.subscriptions), - "active_sessions": len(worker_state.session_ids), - "version": "2.0.0" - } - } - - except Exception as e: - logger.error(f"Error generating detailed health report: {e}") - raise HTTPException(status_code=500, detail=f"Health monitoring error: {str(e)}") - - -@app.get("/health/streams") -async def stream_health_status(): - """Stream-specific health monitoring.""" - try: - from core.monitoring.stream_health import stream_health_tracker - from core.streaming.buffers import shared_cache_buffer - - stream_metrics = stream_health_tracker.get_all_metrics() - buffer_stats = shared_cache_buffer.get_stats() - - return { - "timestamp": time.time(), - "stream_count": len(stream_metrics), - "stream_metrics": stream_metrics, - "buffer_stats": buffer_stats, - "frame_ages": { - camera_id: { - "age_seconds": time.time() - info["last_frame_time"] if info and info.get("last_frame_time") else None, - "total_frames": info.get("frame_count", 0) if info else 0 - } - for camera_id, info in stream_metrics.items() - } - } - - except Exception as e: - logger.error(f"Error generating stream health report: {e}") - raise HTTPException(status_code=500, detail=f"Stream health error: {str(e)}") - - -@app.get("/health/threads") -async def thread_health_status(): - """Thread-specific health monitoring.""" - try: - from core.monitoring.thread_health import thread_health_monitor - - thread_info = thread_health_monitor.get_all_thread_info() - deadlocks = thread_health_monitor.detect_deadlocks() - - return { - "timestamp": time.time(), - "thread_count": len(thread_info), - "thread_info": thread_info, - "potential_deadlocks": deadlocks, - "summary": { - "responsive_threads": sum(1 for info in thread_info.values() if info.get("is_responsive", False)), - "unresponsive_threads": sum(1 for info in thread_info.values() if not info.get("is_responsive", True)), - "deadlock_count": len(deadlocks) - } - } - - except Exception as e: - logger.error(f"Error generating thread health report: {e}") - raise HTTPException(status_code=500, detail=f"Thread health error: {str(e)}") - - -@app.get("/health/recovery") -async def recovery_status(): - """Recovery system status and history.""" - try: - from core.monitoring.recovery import recovery_manager - - recovery_stats = recovery_manager.get_recovery_stats() - - return { - "timestamp": time.time(), - "recovery_stats": recovery_stats, - "summary": { - "total_recoveries_last_hour": recovery_stats.get("total_recoveries_last_hour", 0), - "components_with_recovery_state": len(recovery_stats.get("recovery_states", {})), - "total_recovery_failures": sum( - state.get("failure_count", 0) - for state in recovery_stats.get("recovery_states", {}).values() - ), - "total_recovery_successes": sum( - state.get("success_count", 0) - for state in recovery_stats.get("recovery_states", {}).values() - ) - } - } - - except Exception as e: - logger.error(f"Error generating recovery status report: {e}") - raise HTTPException(status_code=500, detail=f"Recovery status error: {str(e)}") - - -@app.post("/health/recovery/force/{component}") -async def force_recovery(component: str, action: str = "restart_stream"): - """Force recovery action for a specific component.""" - try: - from core.monitoring.recovery import recovery_manager, RecoveryAction - - # Validate action + async def handle_detection(camera_id, stream, frame, websocket, model_tree, persistent_data): try: - recovery_action = RecoveryAction(action) - except ValueError: - raise HTTPException( - status_code=400, - detail=f"Invalid recovery action: {action}. Valid actions: {[a.value for a in RecoveryAction]}" - ) + # Check camera connection state first - handle disconnection immediately + if should_notify_disconnection(camera_id): + logger.error(f"🚨 CAMERA DISCONNECTION DETECTED: {camera_id} - sending immediate detection: null") + + # Clear cached detections and occupancy state + cached_detections.pop(camera_id, None) + frame_skip_flags.pop(camera_id, None) + cached_full_pipeline_results.pop(camera_id, None) # Clear cached pipeline results + session_pipeline_states.pop(camera_id, None) # Reset session pipeline state + + # Reset pipeline state immediately + from siwatsystem.pympta import reset_tracking_state + model_id = stream.get("modelId", "unknown") + reset_tracking_state(camera_id, model_id, "camera disconnected") + + # Send immediate detection: null to backend + detection_data = { + "type": "imageDetection", + "subscriptionIdentifier": stream["subscriptionIdentifier"], + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "data": { + "detection": None, # null detection for disconnection + "modelId": stream["modelId"], + "modelName": stream["modelName"] + } + } + + try: + ws_logger.info(f"TX -> {json.dumps(detection_data, separators=(',', ':'))}") + await websocket.send_json(detection_data) + except RuntimeError as e: + if "websocket.close" in str(e): + logger.warning(f"WebSocket connection closed - cannot send disconnection signal for camera {camera_id}") + return persistent_data + else: + raise + mark_disconnection_notified(camera_id) + logger.info(f"📡 SENT DISCONNECTION SIGNAL - detection: null for camera {camera_id}, backend should clear session") + + return persistent_data + + # Apply crop if specified + cropped_frame = frame + if all(coord is not None for coord in [stream.get("cropX1"), stream.get("cropY1"), stream.get("cropX2"), stream.get("cropY2")]): + cropX1, cropY1, cropX2, cropY2 = stream["cropX1"], stream["cropY1"], stream["cropX2"], stream["cropY2"] + cropped_frame = frame[cropY1:cropY2, cropX1:cropX2] + logger.debug(f"Applied crop coordinates ({cropX1}, {cropY1}, {cropX2}, {cropY2}) to frame for camera {camera_id}") + + logger.debug(f"Processing frame for camera {camera_id} with model {stream['modelId']}") + start_time = time.time() + + # Extract display identifier for pipeline context + subscription_parts = stream["subscriptionIdentifier"].split(';') + display_identifier = subscription_parts[0] if subscription_parts else None + + # Get backend session ID if available + backend_session_id = session_ids.get(display_identifier) + + # Get or initialize session pipeline state + pipeline_state = get_or_init_session_pipeline_state(camera_id) + current_mode = pipeline_state["mode"] + + logger.debug(f"🔍 SESSIONID LOOKUP: display='{display_identifier}', session_id={repr(backend_session_id)}, mode='{current_mode}'") + logger.debug(f"🔍 Available session_ids: {session_ids}") + logger.debug(f"🔍 VALIDATED_DETECTION TRACE: {pipeline_state.get('validated_detection')}") + + # ═══ SESSION ID-BASED PROCESSING MODE ═══ + if not backend_session_id: + # No session ID - handle different modes appropriately + if current_mode == "lightweight": + # Check if we're in car_waitpayment stage - if so, don't reset immediately + current_progression = pipeline_state.get("progression_stage") + if current_progression == "car_waitpayment": + # Stay in lightweight mode - let absence counter + sessionId null logic handle reset + logger.debug(f"🔍 Camera {camera_id}: No session ID but in car_waitpayment - staying in lightweight mode for dual reset condition") + else: + # Not in car_waitpayment - reset immediately (situation 1) + update_session_pipeline_mode(camera_id, "validation_detecting") + current_mode = "validation_detecting" + logger.debug(f"🔍 Camera {camera_id}: No session ID - reset to validation_detecting (not in car_waitpayment)") + elif current_mode not in ["validation_detecting", "send_detections", "waiting_for_session_id"]: + # Other modes - reset to validation_detecting + update_session_pipeline_mode(camera_id, "validation_detecting") + current_mode = "validation_detecting" + logger.debug(f"🔍 Camera {camera_id}: No session ID - reset to validation_detecting from {current_mode}") + else: + logger.debug(f"🔍 Camera {camera_id}: No session ID - staying in {current_mode} mode") + else: + # Session ID available - switch to full pipeline mode + if current_mode in ["send_detections", "waiting_for_session_id"]: + # Session ID just arrived - switch to full pipeline mode + update_session_pipeline_mode(camera_id, "full_pipeline", backend_session_id) + current_mode = "full_pipeline" + logger.info(f"🔥 Camera {camera_id}: Session ID received ({backend_session_id}) - switching to FULL PIPELINE mode") + + # Create context for pipeline execution + pipeline_context = { + "camera_id": camera_id, + "display_id": display_identifier, + "backend_session_id": backend_session_id, + "current_mode": current_mode # Pass current mode to pipeline + } + + start_time = time.time() + detection_result = None + + if current_mode == "validation_detecting": + # ═══ TRACK VALIDATION MODE ═══ + # Run tracking-based validation with track ID stability + logger.debug(f"🔍 Camera {camera_id}: In validation_detecting mode - running track-based validation") + + # Get tracking configuration from model_tree + tracking_config = model_tree.get("tracking", {}) + tracking_enabled = tracking_config.get("enabled", True) + stability_threshold = tracking_config.get("stabilityThreshold", 4) + + # Default to "none" - only proceed after track validation + detection_result = {"class": "none", "confidence": 1.0, "bbox": [0, 0, 0, 0]} + + if tracking_enabled: + # Run full tracking detection to get track IDs + from siwatsystem.pympta import run_detection_with_tracking + all_detections, regions_dict, track_validation_result = run_detection_with_tracking(cropped_frame, model_tree, pipeline_context) + + if track_validation_result.get("validation_complete", False): + # Track validation completed - we have stable track IDs + stable_tracks = track_validation_result.get("stable_tracks", []) + logger.info(f"🎯 Camera {camera_id}: TRACK VALIDATION COMPLETED - stable tracks: {stable_tracks}") + + # Switch to send_detections mode + update_session_pipeline_mode(camera_id, "send_detections") + + # Send the best detection with stable track + if all_detections: + # Find detection with stable track ID + stable_detection = None + for detection in all_detections: + if detection.get("id") in stable_tracks: + stable_detection = detection + break + + if stable_detection: + detection_result = { + "class": stable_detection.get("class", "car"), + "confidence": stable_detection.get("confidence", 0.0), + "bbox": stable_detection.get("bbox", [0, 0, 0, 0]), + "track_id": stable_detection.get("id") + } + + # Store validated detection for full_pipeline mode to reuse + pipeline_state["validated_detection"] = detection_result.copy() + logger.debug(f"🔍 Camera {camera_id}: VALIDATION DEBUG - storing detection_result = {detection_result}") + logger.debug(f"🔍 Camera {camera_id}: VALIDATION DEBUG - pipeline_state after storing = {pipeline_state.get('validated_detection')}") + logger.info(f"🚗 Camera {camera_id}: SENDING STABLE DETECTION - track ID {detection_result['track_id']}") + logger.info(f"💾 Camera {camera_id}: STORED VALIDATED DETECTION for full_pipeline reuse") + else: + logger.warning(f"⚠️ Camera {camera_id}: Stable tracks found but no matching detection") + else: + # Track validation still in progress + stable_tracks = track_validation_result.get("stable_tracks", []) + current_tracks = track_validation_result.get("current_tracks", []) + + if current_tracks: + track_id = current_tracks[0] if current_tracks else "None" + stable_status = "STABLE" if stable_tracks else "validating" + logger.info(f"🔍 Camera {camera_id}: TRACK VALIDATION - car track_id {track_id} ({stable_status}, need {stability_threshold} consecutive frames)") + else: + logger.debug(f"👻 Camera {camera_id}: No car detected") + + logger.debug(f"📤 Camera {camera_id}: Sending 'none' (track validation in progress)") + else: + # Tracking disabled - fall back to basic detection validation + logger.debug(f"🔍 Camera {camera_id}: Tracking disabled - using basic detection validation") + from siwatsystem.pympta import run_lightweight_detection + basic_detection = run_lightweight_detection(cropped_frame, model_tree) + + if basic_detection and basic_detection.get("car_detected"): + best_detection = basic_detection.get("best_detection") + + # Increment validation counter for basic detection + pipeline_state["validation_counter"] += 1 + current_count = pipeline_state["validation_counter"] + threshold = pipeline_state["validation_threshold"] + + if current_count >= threshold: + update_session_pipeline_mode(camera_id, "send_detections") + detection_result = { + "class": best_detection.get("class", "car"), + "confidence": best_detection.get("confidence", 0.0), + "bbox": best_detection.get("bbox", [0, 0, 0, 0]) + } + + # Store validated detection for full_pipeline mode to reuse + pipeline_state["validated_detection"] = detection_result.copy() + logger.debug(f"🔍 Camera {camera_id}: BASIC VALIDATION DEBUG - storing detection_result = {detection_result}") + logger.info(f"💾 Camera {camera_id}: STORED BASIC VALIDATED DETECTION for full_pipeline reuse") + logger.info(f"🎯 Camera {camera_id}: BASIC VALIDATION COMPLETED after {current_count} frames") + else: + logger.info(f"📊 Camera {camera_id}: Basic validation progress {current_count}/{threshold}") + else: + # Reset validation counter + if pipeline_state["validation_counter"] > 0: + pipeline_state["validation_counter"] = 0 + logger.info(f"🔄 Camera {camera_id}: Reset validation counter (no detection)") + + elif current_mode == "send_detections": + # ═══ SEND DETECTIONS MODE ═══ + # Validation completed - now send detection_dict for car detections, detection: null for no car + logger.debug(f"📤 Camera {camera_id}: In send_detections mode - sending detection_dict for cars") + from siwatsystem.pympta import run_lightweight_detection + basic_detection = run_lightweight_detection(cropped_frame, model_tree) + + if basic_detection and basic_detection.get("car_detected"): + # Car detected - send detection_dict + best_detection = basic_detection.get("best_detection") + detection_result = { + "class": best_detection.get("class", "car"), + "confidence": best_detection.get("confidence", 0.0), + "bbox": best_detection.get("bbox", [0, 0, 0, 0]) + } + logger.info(f"🚗 Camera {camera_id}: SENDING DETECTION_DICT - {detection_result['class']} (conf={detection_result['confidence']:.3f}) - backend should generate session ID") + else: + # No car detected - send "none" + detection_result = {"class": "none", "confidence": 1.0, "bbox": [0, 0, 0, 0]} + logger.debug(f"👻 Camera {camera_id}: No car detected - sending 'none'") + + elif current_mode == "waiting_for_session_id": + # ═══ WAITING FOR SESSION ID MODE ═══ + # Stop processing snapshots, wait for session ID + logger.debug(f"⏳ Camera {camera_id}: In waiting_for_session_id mode - not processing snapshots") + return persistent_data # Don't process or send anything + + elif current_mode == "full_pipeline": + # ═══ FULL PIPELINE MODE ═══ + logger.info(f"🔥 Camera {camera_id}: Running FULL PIPELINE (classification branches + Redis + PostgreSQL)") + + # Use validated detection from validation phase instead of detecting again + validated_detection = pipeline_state.get("validated_detection") + logger.debug(f"🔍 Camera {camera_id}: FULL_PIPELINE DEBUG - validated_detection = {validated_detection}") + logger.debug(f"🔍 Camera {camera_id}: FULL_PIPELINE DEBUG - pipeline_state keys = {list(pipeline_state.keys())}") + if validated_detection: + logger.info(f"🔄 Camera {camera_id}: Using validated detection for full pipeline: track_id={validated_detection.get('track_id')}") + detection_result = run_pipeline(cropped_frame, model_tree, context=pipeline_context, validated_detection=validated_detection) + # Clear the validated detection after using it + pipeline_state["validated_detection"] = None + else: + logger.warning(f"⚠️ Camera {camera_id}: No validated detection found for full pipeline - this shouldn't happen") + detection_result = run_pipeline(cropped_frame, model_tree, context=pipeline_context) + + if detection_result and isinstance(detection_result, dict): + # Cache the full pipeline result + cached_full_pipeline_results[camera_id] = { + "result": detection_result.copy(), + "timestamp": time.time() + } + + # Note: Will cache detection_dict after branch processing completes + + # Store the stable track ID for lightweight monitoring + track_id = detection_result.get("track_id") or detection_result.get("id") + if track_id is not None: + pipeline_state["stable_track_id"] = track_id + logger.info(f"💾 Camera {camera_id}: Cached stable track_id={track_id}") + else: + logger.warning(f"⚠️ Camera {camera_id}: No track_id found in detection_result: {detection_result.keys()}") + + # Ensure we have a cached detection dict for lightweight mode + if not pipeline_state.get("cached_detection_dict"): + # Create fallback cached detection dict if branch processing didn't populate it + fallback_detection = { + "carModel": None, + "carBrand": None, + "carYear": None, + "bodyType": None, + "licensePlateText": None, + "licensePlateConfidence": None + } + pipeline_state["cached_detection_dict"] = fallback_detection + logger.warning(f"⚠️ Camera {camera_id}: Created fallback cached detection dict (branch processing may have failed)") + + # Switch to lightweight mode + update_session_pipeline_mode(camera_id, "lightweight") + logger.info(f"✅ Camera {camera_id}: Full pipeline completed - switching to LIGHTWEIGHT mode") + + elif current_mode == "lightweight": + # ═══ SIMPLIFIED LIGHTWEIGHT MODE ═══ + # Send cached detection dict + check for 2 consecutive empty frames to reset + + stable_track_id = pipeline_state.get("stable_track_id") + cached_detection_dict = pipeline_state.get("cached_detection_dict") + + logger.debug(f"🪶 Camera {camera_id}: LIGHTWEIGHT MODE - stable_track_id={stable_track_id}") + + if not pipeline_state.get("yolo_inference_enabled", True): + # YOLO inference disabled during car_fueling - continue sending cached detection dict + logger.debug(f"🛑 Camera {camera_id}: YOLO inference disabled during car_fueling - continue sending cached detection dict") + if cached_detection_dict: + detection_result = cached_detection_dict # Continue sending cached data + logger.info(f"⛽ Camera {camera_id}: YOLO disabled during car_fueling but sending cached detection dict") + else: + logger.warning(f"⚠️ Camera {camera_id}: YOLO disabled but no cached detection dict available") + detection_result = None + else: + # Run lightweight YOLO inference to check car presence for reset logic (no tracking validation needed) + from siwatsystem.pympta import run_lightweight_detection + basic_detection = run_lightweight_detection(cropped_frame, model_tree) + + # Enhanced car detection: requires both confidence pass AND bbox >= 50% of frame + car_detected_confidence = basic_detection and basic_detection.get("car_detected", False) + car_detected_with_bbox_validation = False + + if car_detected_confidence: + # Car passed confidence - now check bbox area + best_detection = basic_detection.get("best_detection") + if best_detection and best_detection.get("bbox"): + bbox = best_detection["bbox"] + x1, y1, x2, y2 = bbox + bbox_area = (x2 - x1) * (y2 - y1) + frame_height, frame_width = cropped_frame.shape[:2] + frame_area = frame_height * frame_width + bbox_area_ratio = bbox_area / frame_area if frame_area > 0 else 0 + + min_area_ratio = 0.2 # 20% of frame + car_detected_with_bbox_validation = bbox_area_ratio >= min_area_ratio + + if not car_detected_with_bbox_validation: + logger.info(f"🚫 Camera {camera_id}: LIGHTWEIGHT - car detected but bbox {bbox_area_ratio:.1%} < {min_area_ratio:.0%} (too distant) - counting as absent") + else: + logger.debug(f"✅ Camera {camera_id}: LIGHTWEIGHT - car detected with valid bbox {bbox_area_ratio:.1%} >= {min_area_ratio:.0%}") + else: + logger.debug(f"⚠️ Camera {camera_id}: LIGHTWEIGHT - car detected but no bbox info available") + + logger.debug(f"🔍 Camera {camera_id}: LIGHTWEIGHT - enhanced car presence check: confidence={car_detected_confidence}, bbox_valid={car_detected_with_bbox_validation}") + + if car_detected_with_bbox_validation: + # Car detected - reset absence counter, continue sending cached detection dict + pipeline_state["absence_counter"] = 0 # Reset absence since cars are present + + if cached_detection_dict: + detection_result = cached_detection_dict # Always send cached data + logger.info(f"💾 Camera {camera_id}: LIGHTWEIGHT - car detected, sending cached detection dict") + else: + logger.warning(f"⚠️ Camera {camera_id}: LIGHTWEIGHT - car detected but no cached detection dict available") + detection_result = None + else: + # No car detected - increment absence counter + pipeline_state["absence_counter"] += 1 + absence_count = pipeline_state["absence_counter"] + max_absence = 3 # Need 3 consecutive empty frames + + logger.info(f"👻 Camera {camera_id}: LIGHTWEIGHT - no car detected (absence {absence_count}/{max_absence})") + + # Check if we should reset: Need BOTH 3 consecutive absence frames AND sessionId: null + current_progression = pipeline_state.get("progression_stage") + should_check_session_null = current_progression == "car_waitpayment" + + if absence_count >= max_absence: + if should_check_session_null: + # In car_waitpayment stage - require BOTH conditions + if backend_session_id is None: + # Both conditions met: 3 absence frames + sessionId: null + logger.info(f"🔄 Camera {camera_id}: DUAL RESET CONDITIONS MET - {max_absence} consecutive absence frames + sessionId: null") + + # Clear all state and prepare for next car + cached_full_pipeline_results.pop(camera_id, None) + pipeline_state["cached_detection_dict"] = None + pipeline_state["stable_track_id"] = None + pipeline_state["validated_detection"] = None + pipeline_state["progression_stage"] = None + old_absence_counter = pipeline_state["absence_counter"] + old_validation_counter = pipeline_state.get("validation_counter", 0) + pipeline_state["absence_counter"] = 0 + pipeline_state["validation_counter"] = 0 + pipeline_state["yolo_inference_enabled"] = True + + logger.info(f"🧹 Camera {camera_id}: DUAL RESET - absence_counter: {old_absence_counter}→0, validation_counter: {old_validation_counter}→0, progression_stage: {current_progression}→None") + + # Clear stability tracking data for this camera + from siwatsystem.pympta import reset_camera_stability_tracking + reset_camera_stability_tracking(camera_id, model_tree.get("modelId", "unknown")) + + # Switch back to validation phase + update_session_pipeline_mode(camera_id, "validation_detecting") + logger.info(f"✅ Camera {camera_id}: DUAL RESET TO VALIDATION COMPLETE - ready for new car") + + # Now in validation mode - send what YOLO detection finds (will be null since no car) + detection_result = {"class": "none", "confidence": 1.0, "bbox": [0, 0, 0, 0]} + else: + # Only absence frames met, but sessionId is not null - continue sending cached detection + logger.info(f"⏳ Camera {camera_id}: {max_absence} absence frames reached but sessionId={backend_session_id} (not null) - continuing with cached detection") + if cached_detection_dict: + detection_result = cached_detection_dict + else: + logger.warning(f"⚠️ Camera {camera_id}: No cached detection dict available") + detection_result = None + else: + # Not in car_waitpayment - use original simple reset condition (situation 1) + logger.info(f"🔄 Camera {camera_id}: SIMPLE RESET CONDITION MET - {max_absence} consecutive empty frames (not in car_waitpayment)") + + # Clear all state and prepare for next car + cached_full_pipeline_results.pop(camera_id, None) + pipeline_state["cached_detection_dict"] = None + pipeline_state["stable_track_id"] = None + pipeline_state["validated_detection"] = None + pipeline_state["progression_stage"] = None + old_absence_counter = pipeline_state["absence_counter"] + old_validation_counter = pipeline_state.get("validation_counter", 0) + pipeline_state["absence_counter"] = 0 + pipeline_state["validation_counter"] = 0 + pipeline_state["yolo_inference_enabled"] = True + + logger.info(f"🧹 Camera {camera_id}: SIMPLE RESET - absence_counter: {old_absence_counter}→0, validation_counter: {old_validation_counter}→0") + + # Clear stability tracking data for this camera + from siwatsystem.pympta import reset_camera_stability_tracking + reset_camera_stability_tracking(camera_id, model_tree.get("modelId", "unknown")) + + # Switch back to validation phase + update_session_pipeline_mode(camera_id, "validation_detecting") + logger.info(f"✅ Camera {camera_id}: SIMPLE RESET TO VALIDATION COMPLETE - ready for new car") + + # Now in validation mode - send what YOLO detection finds (will be null since no car) + detection_result = {"class": "none", "confidence": 1.0, "bbox": [0, 0, 0, 0]} + else: + # Still within absence threshold - continue sending cached detection dict + if cached_detection_dict: + detection_result = cached_detection_dict # Send cached data + logger.info(f"⏳ Camera {camera_id}: LIGHTWEIGHT - no car but absence<{max_absence}, still sending cached detection dict") + else: + logger.warning(f"⚠️ Camera {camera_id}: LIGHTWEIGHT - no cached detection dict available") + detection_result = None + + elif current_mode == "car_gone_waiting": + # ═══ CAR GONE WAITING STATE ═══ + # Car is gone (both conditions met), YOLO inference disabled, waiting for new session + + logger.debug(f"🛑 Camera {camera_id}: CAR GONE WAITING - YOLO inference stopped") + + # Check if backend has started a new session (indicates new car scenario) + if backend_session_id is not None: + # Backend started new session - re-enable YOLO and reset to validation + pipeline_state["yolo_inference_enabled"] = True + pipeline_state["absence_counter"] = 0 + pipeline_state["stable_track_id"] = None + pipeline_state["cached_detection_dict"] = None + pipeline_state["validated_detection"] = None + + # Clear stability tracking data for this camera + from siwatsystem.pympta import reset_camera_stability_tracking + reset_camera_stability_tracking(camera_id, model_tree.get("modelId", "unknown")) + + update_session_pipeline_mode(camera_id, "validation_detecting") + logger.info(f"🔄 Camera {camera_id}: New session detected (id={backend_session_id}) - re-enabling YOLO inference") + logger.info(f"✅ Camera {camera_id}: Reset to validation mode - cleared all tracking, ready for new car detection") + + # Don't run detection this frame - let next frame start fresh + detection_result = {"class": "none", "confidence": 1.0, "bbox": [0, 0, 0, 0]} + else: + # Still waiting - no sessionId, no detection to send + logger.debug(f"🛑 Camera {camera_id}: Car gone waiting - no YOLO inference, no data sent") + detection_result = None + + process_time = (time.time() - start_time) * 1000 + logger.debug(f"Detection for camera {camera_id} completed in {process_time:.2f}ms (mode: {current_mode})") + + # Skip processing if no detection result (blocked by session gating) + if detection_result is None: + logger.debug(f"No detection result to process for camera {camera_id}") + return persistent_data + + # Log the raw detection result for debugging + logger.debug(f"Raw detection result for camera {camera_id}:\n{json.dumps(detection_result, indent=2, default=str)}") + + # Extract session_id from pipeline result (always use backend sessionId) + session_id = backend_session_id + logger.debug(f"Using backend session_id: {session_id}") + + + # Process detection result based on current mode + if current_mode == "validation_detecting": + # ═══ VALIDATION DETECTING MODE ═══ + # Always send detection: null during validation phase + detection_dict = None + logger.debug(f"🔍 SENDING 'NONE' - validation_detecting mode for camera {camera_id}") + + elif current_mode == "send_detections": + # ═══ SEND DETECTIONS MODE ═══ + if detection_result.get("class") == "none": + # No car detected - send detection: null + detection_dict = None + logger.debug(f"📤 SENDING 'NONE' - send_detections mode (no car) for camera {camera_id}") + else: + # Car detected in send_detections mode - ALWAYS send empty dict to trigger backend sessionId + # Purpose: Tell backend "car is here, please create sessionId" + detection_dict = {} + logger.info(f"📤 SENDING EMPTY DETECTION_DICT - send_detections mode, requesting backend to create sessionId (conf={detection_result.get('confidence', 0):.3f}) for camera {camera_id}") + + if backend_session_id: + logger.debug(f"🔄 Camera {camera_id}: Note - sessionId {backend_session_id} exists but still in send_detections mode (transition pending)") + + elif current_mode == "lightweight": + # ═══ SIMPLIFIED LIGHTWEIGHT MODE DETECTION PROCESSING ═══ + if detection_result.get("class") == "none": + # No car detected - this happens when resetting to validation + detection_dict = None # Send detection: null + logger.info(f"🚫 LIGHTWEIGHT - no car detected, sending detection=null") + elif isinstance(detection_result, dict) and ("carBrand" in detection_result or "carModel" in detection_result): + # Check if we're waiting for dual reset condition + current_progression = pipeline_state.get("progression_stage") + if current_progression == "car_waitpayment" and backend_session_id is None: + # In car_waitpayment + sessionId: null - STOP sending cached detection to prevent new session creation + detection_dict = None + logger.info(f"🛑 LIGHTWEIGHT - in car_waitpayment with sessionId: null, NOT sending cached detection (waiting for dual reset)") + else: + # Normal lightweight mode - send cached detection dict + detection_dict = detection_result + logger.info(f"💾 LIGHTWEIGHT - sending cached detection dict") + else: + logger.warning(f"⚠️ LIGHTWEIGHT - unexpected detection_result type: {type(detection_result)}") + detection_dict = None + + elif detection_result.get("class") == "none": + # Other modes - send null to clear session + detection_dict = None + logger.info(f"📤 SENDING 'NONE' (detection: null) - Car absent, expecting backend to clear session for camera {camera_id}") + elif detection_result and "carBrand" in detection_result: + # Handle cached detection dict format (fallback for compatibility) + detection_dict = detection_result + logger.info(f"💾 Camera {camera_id}: LIGHTWEIGHT MODE - using detection_result as detection_dict:") + logger.info(f"💾 Camera {camera_id}: - detection_dict: {detection_dict}") + else: + # Valid detection - convert to backend format (will be populated by branch processing) + detection_dict = { + "carModel": None, + "carBrand": None, + "carYear": None, + "bodyType": None, + "licensePlateText": None, + "licensePlateConfidence": None + } + + # Extract and process branch results from parallel classification (only for valid detections, skip cached mode) + if detection_result.get("class") != "none" and "branch_results" in detection_result and not detection_result.get("cached_mode", False): + def process_branch_results(branch_results, depth=0): + """Recursively process branch results including nested branches.""" + if not isinstance(branch_results, dict): + return + + indent = " " * depth + for branch_id, branch_data in branch_results.items(): + if isinstance(branch_data, dict): + logger.debug(f"{indent}Processing branch {branch_id}: {branch_data}") + + # Map common classification fields to backend-expected names + if "brand" in branch_data: + detection_dict["carBrand"] = branch_data["brand"] + logger.debug(f"{indent}Mapped carBrand: {branch_data['brand']}") + if "body_type" in branch_data: + detection_dict["bodyType"] = branch_data["body_type"] + logger.debug(f"{indent}Mapped bodyType: {branch_data['body_type']}") + if "class" in branch_data: + class_name = branch_data["class"] + + # Map based on branch/model type + if "brand" in branch_id.lower(): + detection_dict["carBrand"] = class_name + logger.debug(f"{indent}Mapped carBrand from class: {class_name}") + elif "bodytype" in branch_id.lower() or "body" in branch_id.lower(): + detection_dict["bodyType"] = class_name + logger.debug(f"{indent}Mapped bodyType from class: {class_name}") + + # Process nested branch results recursively + if "branch_results" in branch_data: + logger.debug(f"{indent}Processing nested branches in {branch_id}") + process_branch_results(branch_data["branch_results"], depth + 1) - # Force recovery - success = recovery_manager.force_recovery(component, recovery_action, "manual_api_request") + branch_results = detection_result.get("branch_results", {}) + if branch_results: + logger.debug(f"Processing branch results: {branch_results}") + process_branch_results(branch_results) + logger.info(f"Detection payload after branch processing: {detection_dict}") + + # Cache the detection_dict for lightweight mode (after branch processing completes) + if current_mode == "full_pipeline": + pipeline_state = get_or_init_session_pipeline_state(camera_id) + pipeline_state["cached_detection_dict"] = detection_dict.copy() + logger.info(f"💾 Camera {camera_id}: CACHED DETECTION DICT after branch processing: {detection_dict}") + + else: + logger.debug("No branch results found in detection result") + + detection_data = { + "type": "imageDetection", + "subscriptionIdentifier": stream["subscriptionIdentifier"], + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + # "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) + f".{int(time.time() * 1000) % 1000:03d}Z", + "data": { + "detection": detection_dict, + "modelId": stream["modelId"], + "modelName": stream["modelName"] + } + } + + # SessionId should NEVER be sent from worker to backend - it's uni-directional (backend -> worker only) + # Backend manages sessionIds independently based on detection content + logger.debug(f"TX message prepared (no sessionId) - detection_dict type: {type(detection_dict)}") + + # Log detection details for different modes + if current_mode == "lightweight": + if detection_result and detection_result.get("class") == "none": + logger.info(f"🚫 Camera {camera_id}: LIGHTWEIGHT - No car detected (resetting to validation)") + elif isinstance(detection_result, dict) and ("carBrand" in detection_result or "carModel" in detection_result): + logger.info(f"💾 Camera {camera_id}: LIGHTWEIGHT - Sending cached detection data") + else: + logger.info(f"🪶 Camera {camera_id}: LIGHTWEIGHT - Processing detection") + elif detection_result and "class" in detection_result and detection_result.get("class") != "none": + confidence = detection_result.get("confidence", 0.0) + logger.info(f"🚗 Camera {camera_id}: Detected {detection_result['class']} with confidence {confidence:.2f} using model {stream['modelName']}") + + # Send detection data to backend (session gating handled above in processing logic) + logger.debug(f"📤 SENDING TO BACKEND for camera {camera_id}: {json.dumps(detection_data, indent=2)}") + try: + ws_logger.info(f"TX -> {json.dumps(detection_data, separators=(',', ':'))}") + await websocket.send_json(detection_data) + logger.debug(f"Sent detection data to client for camera {camera_id}") + + # Cache the detection data for potential resubscriptions (only if not null detection) + if detection_dict is not None and detection_result.get("class") != "none": + cached_detections[camera_id] = detection_data.copy() + logger.debug(f"Cached detection for camera {camera_id}: {detection_dict}") + + # Enhanced caching: Store by session_id for LPR integration + session_id = detection_data.get('sessionId') + if session_id: + session_id_str = str(session_id) + session_detections[session_id_str] = detection_data.copy() + session_to_camera[session_id_str] = camera_id + detection_timestamps[session_id_str] = time.time() + logger.debug(f"🔑 Cached detection for LPR by session_id {session_id_str}: {camera_id}") + else: + # Don't cache null/none detections - let them reset properly + cached_detections.pop(camera_id, None) + logger.debug(f"Not caching null/none detection for camera {camera_id}") + + except RuntimeError as e: + if "websocket.close" in str(e): + logger.warning(f"WebSocket connection closed - cannot send detection data for camera {camera_id}") + return persistent_data + else: + raise + + # Log status after sending (no sessionId sent to backend) + if detection_dict is None: + logger.info(f"📡 SENT 'none' detection - backend should clear session for camera {camera_id}") + elif detection_dict == {}: + logger.info(f"📡 SENT empty detection - backend should create sessionId for camera {camera_id}") + else: + logger.info(f"📡 SENT detection data - backend manages sessionId independently for camera {camera_id}") + return persistent_data + except Exception as e: + logger.error(f"Error in handle_detection for camera {camera_id}: {str(e)}", exc_info=True) + return persistent_data - return { - "timestamp": time.time(), - "component": component, - "action": action, - "success": success, - "message": f"Recovery {'successful' if success else 'failed'} for component {component}" - } + def frame_reader(camera_id, cap, buffer, stop_event): + retries = 0 + logger.info(f"Starting frame reader thread for camera {camera_id}") + frame_count = 0 + last_log_time = time.time() + + try: + # Log initial camera status and properties + if cap.isOpened(): + width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + fps = cap.get(cv2.CAP_PROP_FPS) + logger.info(f"Camera {camera_id} opened successfully with resolution {width}x{height}, FPS: {fps}") + set_camera_connected(camera_id, True) + else: + logger.error(f"Camera {camera_id} failed to open initially") + set_camera_connected(camera_id, False, "Failed to open camera initially") + + while not stop_event.is_set(): + try: + if not cap.isOpened(): + logger.error(f"Camera {camera_id} is not open before trying to read") + # Attempt to reopen + cap = cv2.VideoCapture(streams[camera_id]["rtsp_url"]) + time.sleep(reconnect_interval) + continue + + logger.debug(f"Attempting to read frame from camera {camera_id}") + ret, frame = cap.read() + + if not ret: + error_msg = f"Connection lost for camera: {camera_id}, retry {retries+1}/{max_retries}" + logger.warning(error_msg) + set_camera_connected(camera_id, False, error_msg) + cap.release() + time.sleep(reconnect_interval) + retries += 1 + if retries > max_retries and max_retries != -1: + logger.error(f"Max retries reached for camera: {camera_id}, stopping frame reader") + set_camera_connected(camera_id, False, "Max retries reached") + break + # Re-open + logger.info(f"Attempting to reopen RTSP stream for camera: {camera_id}") + cap = cv2.VideoCapture(streams[camera_id]["rtsp_url"]) + if not cap.isOpened(): + logger.error(f"Failed to reopen RTSP stream for camera: {camera_id}") + set_camera_connected(camera_id, False, "Failed to reopen RTSP stream") + continue + logger.info(f"Successfully reopened RTSP stream for camera: {camera_id}") + set_camera_connected(camera_id, True) + continue + + # Successfully read a frame + frame_count += 1 + current_time = time.time() + # Log frame stats every 5 seconds + if current_time - last_log_time > 5: + logger.info(f"Camera {camera_id}: Read {frame_count} frames in the last {current_time - last_log_time:.1f} seconds") + frame_count = 0 + last_log_time = current_time + + logger.debug(f"Successfully read frame from camera {camera_id}, shape: {frame.shape}") + retries = 0 + set_camera_connected(camera_id, True) # Mark as connected on successful frame read + + # Overwrite old frame if buffer is full + if not buffer.empty(): + try: + buffer.get_nowait() + logger.debug(f"[frame_reader] Removed old frame from buffer for camera {camera_id}") + except queue.Empty: + pass + buffer.put(frame) + logger.debug(f"[frame_reader] Added new frame to buffer for camera {camera_id}. Buffer size: {buffer.qsize()}") + + # Short sleep to avoid CPU overuse + time.sleep(0.01) + + except cv2.error as e: + error_msg = f"OpenCV error for camera {camera_id}: {e}" + logger.error(error_msg, exc_info=True) + set_camera_connected(camera_id, False, error_msg) + cap.release() + time.sleep(reconnect_interval) + retries += 1 + if retries > max_retries and max_retries != -1: + logger.error(f"Max retries reached after OpenCV error for camera {camera_id}") + set_camera_connected(camera_id, False, "Max retries reached after OpenCV error") + break + logger.info(f"Attempting to reopen RTSP stream after OpenCV error for camera: {camera_id}") + cap = cv2.VideoCapture(streams[camera_id]["rtsp_url"]) + if not cap.isOpened(): + logger.error(f"Failed to reopen RTSP stream for camera {camera_id} after OpenCV error") + set_camera_connected(camera_id, False, "Failed to reopen after OpenCV error") + continue + logger.info(f"Successfully reopened RTSP stream after OpenCV error for camera: {camera_id}") + set_camera_connected(camera_id, True) + except Exception as e: + error_msg = f"Unexpected error for camera {camera_id}: {str(e)}" + logger.error(error_msg, exc_info=True) + set_camera_connected(camera_id, False, error_msg) + cap.release() + break + except Exception as e: + logger.error(f"Error in frame_reader thread for camera {camera_id}: {str(e)}", exc_info=True) + finally: + logger.info(f"Frame reader thread for camera {camera_id} is exiting") + if cap and cap.isOpened(): + cap.release() - except HTTPException: - raise - except Exception as e: - logger.error(f"Error forcing recovery for {component}: {e}") - raise HTTPException(status_code=500, detail=f"Recovery error: {str(e)}") + def snapshot_reader(camera_id, snapshot_url, snapshot_interval, buffer, stop_event): + """Frame reader that fetches snapshots from HTTP/HTTPS URL at specified intervals""" + retries = 0 + consecutive_failures = 0 # Track consecutive failures for backoff + logger.info(f"Starting snapshot reader thread for camera {camera_id} from {snapshot_url}") + frame_count = 0 + last_log_time = time.time() + + # Initialize camera state + set_camera_connected(camera_id, True) + + try: + interval_seconds = snapshot_interval / 1000.0 # Convert milliseconds to seconds + logger.info(f"Snapshot interval for camera {camera_id}: {interval_seconds}s") + + while not stop_event.is_set(): + try: + start_time = time.time() + frame = fetch_snapshot(snapshot_url) + + if frame is None: + consecutive_failures += 1 + error_msg = f"Failed to fetch snapshot for camera: {camera_id}, consecutive failures: {consecutive_failures}" + logger.warning(error_msg) + set_camera_connected(camera_id, False, error_msg) + retries += 1 + + # Check network connectivity with a simple ping-like test + if consecutive_failures % 5 == 1: # Every 5th failure, test connectivity + try: + test_response = requests.get(snapshot_url, timeout=(2, 5), stream=False) + logger.info(f"Camera {camera_id}: Connectivity test result: {test_response.status_code}") + except Exception as test_error: + logger.warning(f"Camera {camera_id}: Connectivity test failed: {test_error}") + + if retries > max_retries and max_retries != -1: + logger.error(f"Max retries reached for snapshot camera: {camera_id}, stopping reader") + set_camera_connected(camera_id, False, "Max retries reached for snapshot camera") + break + + # Exponential backoff based on consecutive failures + backoff_delay = min(30, max(1, min(2 ** min(consecutive_failures - 1, 6), interval_seconds * 2))) # Start with 1s, max 30s + logger.debug(f"Camera {camera_id}: Backing off for {backoff_delay:.1f}s (consecutive failures: {consecutive_failures})") + if stop_event.wait(backoff_delay): # Use wait with timeout instead of sleep + break # Exit if stop_event is set during backoff + continue + + # Successfully fetched a frame - reset consecutive failures + consecutive_failures = 0 # Reset backoff on success + frame_count += 1 + current_time = time.time() + # Log frame stats every 5 seconds + if current_time - last_log_time > 5: + logger.info(f"Camera {camera_id}: Fetched {frame_count} snapshots in the last {current_time - last_log_time:.1f} seconds") + frame_count = 0 + last_log_time = current_time + + logger.debug(f"Successfully fetched snapshot from camera {camera_id}, shape: {frame.shape}") + retries = 0 + set_camera_connected(camera_id, True) # Mark as connected on successful snapshot + + # Overwrite old frame if buffer is full + if not buffer.empty(): + try: + buffer.get_nowait() + logger.debug(f"[snapshot_reader] Removed old snapshot from buffer for camera {camera_id}") + except queue.Empty: + pass + buffer.put(frame) + logger.debug(f"[snapshot_reader] Added new snapshot to buffer for camera {camera_id}. Buffer size: {buffer.qsize()}") + + # Wait for the specified interval + elapsed = time.time() - start_time + sleep_time = max(interval_seconds - elapsed, 0) + if sleep_time > 0: + time.sleep(sleep_time) + + except Exception as e: + consecutive_failures += 1 + error_msg = f"Unexpected error fetching snapshot for camera {camera_id}: {str(e)}" + logger.error(error_msg, exc_info=True) + set_camera_connected(camera_id, False, error_msg) + retries += 1 + if retries > max_retries and max_retries != -1: + logger.error(f"Max retries reached after error for snapshot camera {camera_id}") + set_camera_connected(camera_id, False, "Max retries reached after error") + break + + # Exponential backoff for exceptions too + backoff_delay = min(30, max(1, min(2 ** min(consecutive_failures - 1, 6), interval_seconds * 2))) # Start with 1s, max 30s + logger.debug(f"Camera {camera_id}: Exception backoff for {backoff_delay:.1f}s (consecutive failures: {consecutive_failures})") + if stop_event.wait(backoff_delay): # Use wait with timeout instead of sleep + break # Exit if stop_event is set during backoff + except Exception as e: + logger.error(f"Error in snapshot_reader thread for camera {camera_id}: {str(e)}", exc_info=True) + finally: + logger.info(f"Snapshot reader thread for camera {camera_id} is exiting") + async def reconcile_subscriptions(desired_subscriptions, websocket): + """ + Declarative reconciliation: Compare desired vs current subscriptions and make changes + """ + logger.info(f"Reconciling subscriptions: {len(desired_subscriptions)} desired") + + with streams_lock: + # Get current subscriptions + current_subscription_ids = set(streams.keys()) + desired_subscription_ids = set(sub["subscriptionIdentifier"] for sub in desired_subscriptions) + + # Find what to add and remove + to_add = desired_subscription_ids - current_subscription_ids + to_remove = current_subscription_ids - desired_subscription_ids + to_check_for_changes = current_subscription_ids & desired_subscription_ids + + logger.info(f"Reconciliation: {len(to_add)} to add, {len(to_remove)} to remove, {len(to_check_for_changes)} to check for changes") + + # Remove subscriptions that are no longer wanted + for subscription_id in to_remove: + await unsubscribe_internal(subscription_id) + + # Check existing subscriptions for parameter changes + for subscription_id in to_check_for_changes: + desired_sub = next(sub for sub in desired_subscriptions if sub["subscriptionIdentifier"] == subscription_id) + current_stream = streams[subscription_id] + + # Check if parameters changed + if has_subscription_changed(desired_sub, current_stream): + logger.info(f"Parameters changed for {subscription_id}, resubscribing") + logger.debug(f"Parameter comparison for {subscription_id}:") + logger.debug(f" rtspUrl: '{desired_sub.get('rtspUrl')}' vs '{current_stream.get('rtsp_url')}'") + logger.debug(f" snapshotUrl: '{desired_sub.get('snapshotUrl')}' vs '{current_stream.get('snapshot_url')}'") + logger.debug(f" modelUrl: '{extract_model_file_identifier(desired_sub.get('modelUrl'))}' vs '{extract_model_file_identifier(current_stream.get('modelUrl'))}'") + logger.debug(f" modelId: {desired_sub.get('modelId')} vs {current_stream.get('modelId')}") + + # Preserve detection state for resubscription + cached_detection = cached_detections.get(subscription_id) + logger.debug(f"Preserving detection state for resubscription: {cached_detection is not None}") + + await unsubscribe_internal(subscription_id, preserve_detection=True) + await subscribe_internal(desired_sub, websocket, cached_detection=cached_detection) + + # Add new subscriptions + for subscription_id in to_add: + desired_sub = next(sub for sub in desired_subscriptions if sub["subscriptionIdentifier"] == subscription_id) + await subscribe_internal(desired_sub, websocket) -@app.get("/health/metrics") -async def health_metrics(): - """Performance and health metrics in a format suitable for monitoring systems.""" + def extract_model_file_identifier(model_url): + """Extract the core model file identifier from S3 URLs, ignoring timestamp parameters""" + if not model_url: + return None + + # For S3 URLs, extract just the path portion before query parameters + try: + from urllib.parse import urlparse + parsed = urlparse(model_url) + # Return the path which contains the actual model file identifier + # e.g. "/adsist-cms-staging/models/bangchak_poc-1756312318569.mpta" + return parsed.path + except Exception as e: + logger.warning(f"Failed to parse model URL {model_url}: {e}") + return model_url + + def has_subscription_changed(desired_sub, current_stream): + """Check if subscription parameters have changed""" + # Smart model URL comparison - ignore timestamp changes in signed URLs + desired_model_id = extract_model_file_identifier(desired_sub.get("modelUrl")) + current_model_id = extract_model_file_identifier(current_stream.get("modelUrl")) + + return ( + desired_sub.get("rtspUrl") != current_stream.get("rtsp_url") or + desired_sub.get("snapshotUrl") != current_stream.get("snapshot_url") or + desired_sub.get("snapshotInterval") != current_stream.get("snapshot_interval") or + desired_sub.get("cropX1") != current_stream.get("cropX1") or + desired_sub.get("cropY1") != current_stream.get("cropY1") or + desired_sub.get("cropX2") != current_stream.get("cropX2") or + desired_sub.get("cropY2") != current_stream.get("cropY2") or + desired_sub.get("modelId") != current_stream.get("modelId") or + desired_sub.get("modelName") != current_stream.get("modelName") or + desired_model_id != current_model_id + ) + + async def subscribe_internal(subscription, websocket, cached_detection=None): + """Internal subscription logic extracted from original subscribe handler""" + subscriptionIdentifier = subscription.get("subscriptionIdentifier") + rtsp_url = subscription.get("rtspUrl") + snapshot_url = subscription.get("snapshotUrl") + snapshot_interval = subscription.get("snapshotInterval") + model_url = subscription.get("modelUrl") + modelId = subscription.get("modelId") + modelName = subscription.get("modelName") + cropX1 = subscription.get("cropX1") + cropY1 = subscription.get("cropY1") + cropX2 = subscription.get("cropX2") + cropY2 = subscription.get("cropY2") + + # Extract camera_id from subscriptionIdentifier + parts = subscriptionIdentifier.split(';') + if len(parts) != 2: + logger.error(f"Invalid subscriptionIdentifier format: {subscriptionIdentifier}") + return + + display_identifier, camera_identifier = parts + camera_id = subscriptionIdentifier + + # Load model if needed using shared MPTA manager + if model_url: + with models_lock: + if (camera_id not in models) or (modelId not in models[camera_id]): + logger.info(f"Getting shared MPTA for camera {camera_id}, modelId {modelId}") + + # Use shared MPTA manager for optimized downloads + mpta_result = get_or_download_mpta(modelId, model_url, camera_id) + if not mpta_result: + logger.error(f"Failed to get/download MPTA for modelId {modelId}") + return + + shared_extraction_path, local_mpta_file = mpta_result + + # Load pipeline from local MPTA file + model_tree = load_pipeline_from_zip(local_mpta_file, shared_extraction_path) + if model_tree is None: + logger.error(f"Failed to load model {modelId} from shared MPTA") + return + + if camera_id not in models: + models[camera_id] = {} + models[camera_id][modelId] = model_tree + + # Start LPR integration threads after first model is loaded (only once) + global lpr_integration_started + if not lpr_integration_started and hasattr(model_tree, 'get') and model_tree.get('redis_client'): + try: + start_lpr_integration() + lpr_integration_started = True + logger.info("🚀 LPR integration started after first model load") + except Exception as e: + logger.error(f"❌ Failed to start LPR integration: {e}") + + # Create stream (same logic as original) + if camera_id and (rtsp_url or snapshot_url) and len(streams) < max_streams: + camera_url = snapshot_url if snapshot_url else rtsp_url + + # Check if we already have a stream for this camera URL + shared_stream = camera_streams.get(camera_url) + + if shared_stream: + # Reuse existing stream + buffer = shared_stream["buffer"] + stop_event = shared_stream["stop_event"] + thread = shared_stream["thread"] + mode = shared_stream["mode"] + shared_stream["ref_count"] = shared_stream.get("ref_count", 0) + 1 + else: + # Create new stream + buffer = queue.Queue(maxsize=1) + stop_event = threading.Event() + + if snapshot_url and snapshot_interval: + thread = threading.Thread(target=snapshot_reader, args=(camera_id, snapshot_url, snapshot_interval, buffer, stop_event)) + thread.daemon = True + thread.start() + mode = "snapshot" + shared_stream = { + "buffer": buffer, "thread": thread, "stop_event": stop_event, + "mode": mode, "url": snapshot_url, "snapshot_interval": snapshot_interval, "ref_count": 1 + } + camera_streams[camera_url] = shared_stream + elif rtsp_url: + cap = cv2.VideoCapture(rtsp_url) + if not cap.isOpened(): + logger.error(f"Failed to open RTSP stream for camera {camera_id}") + return + thread = threading.Thread(target=frame_reader, args=(camera_id, cap, buffer, stop_event)) + thread.daemon = True + thread.start() + mode = "rtsp" + shared_stream = { + "buffer": buffer, "thread": thread, "stop_event": stop_event, + "mode": mode, "url": rtsp_url, "cap": cap, "ref_count": 1 + } + camera_streams[camera_url] = shared_stream + else: + logger.error(f"No valid URL provided for camera {camera_id}") + return + + # Create stream info + stream_info = { + "buffer": buffer, "thread": thread, "stop_event": stop_event, + "modelId": modelId, "modelName": modelName, "subscriptionIdentifier": subscriptionIdentifier, + "cropX1": cropX1, "cropY1": cropY1, "cropX2": cropX2, "cropY2": cropY2, + "mode": mode, "camera_url": camera_url, "modelUrl": model_url, + # Always store both URLs for comparison consistency + "rtsp_url": rtsp_url, + "snapshot_url": snapshot_url, + "snapshot_interval": snapshot_interval + } + + if mode == "rtsp": + stream_info["cap"] = shared_stream["cap"] + + streams[camera_id] = stream_info + subscription_to_camera[camera_id] = camera_url + logger.info(f"Subscribed to camera {camera_id}") + + # Send initial detection to backend - use cached if available, otherwise "none" + if cached_detection: + # Restore cached detection with updated timestamp (RESUBSCRIPTION STATUS UPDATE) + initial_detection_data = cached_detection.copy() + initial_detection_data["timestamp"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + logger.info(f"📡 RESUBSCRIPTION: Restoring cached detection for camera {camera_id}") + logger.debug(f"📡 RESUBSCRIPTION: Cached detection has sessionId: {initial_detection_data.get('sessionId', 'None')}") + else: + # Send "none" detection for new subscriptions + initial_detection_data = { + "type": "imageDetection", + "subscriptionIdentifier": subscriptionIdentifier, + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "data": { + "detection": None, + "modelId": modelId, + "modelName": modelName + } + } + logger.info(f"📡 NEW SUBSCRIPTION: Sending initial 'none' detection for camera {camera_id}") + + ws_logger.info(f"TX -> {json.dumps(initial_detection_data, separators=(',', ':'))}") + await websocket.send_json(initial_detection_data) + logger.debug(f"Initial detection data sent (resubscription={cached_detection is not None}): {initial_detection_data}") + + # This cached detection was just a one-time status update for resubscription + # Normal frame processing will continue independently + + async def unsubscribe_internal(subscription_id, preserve_detection=False): + """Internal unsubscription logic""" + if subscription_id in streams: + stream = streams.pop(subscription_id) + camera_url = subscription_to_camera.pop(subscription_id, None) + + # Clean up model references for this camera + with models_lock: + if subscription_id in models: + camera_models = models[subscription_id] + for model_id, model_tree in camera_models.items(): + logger.info(f"🧹 Cleaning up model references for camera {subscription_id}, modelId {model_id}") + # Release model registry references + cleanup_pipeline_node(model_tree) + # Release MPTA manager reference + release_mpta(model_id, subscription_id) + del models[subscription_id] + + if camera_url and camera_url in camera_streams: + shared_stream = camera_streams[camera_url] + shared_stream["ref_count"] -= 1 + + if shared_stream["ref_count"] <= 0: + shared_stream["stop_event"].set() + shared_stream["thread"].join() + if "cap" in shared_stream: + shared_stream["cap"].release() + del camera_streams[camera_url] + + latest_frames.pop(subscription_id, None) + if not preserve_detection: + cached_detections.pop(subscription_id, None) # Clear cached detection only if not preserving + frame_skip_flags.pop(subscription_id, None) # Clear frame skip flag + camera_states.pop(subscription_id, None) # Clear camera state + cached_full_pipeline_results.pop(subscription_id, None) # Clear cached pipeline results + session_pipeline_states.pop(subscription_id, None) # Clear session pipeline state + cleanup_camera_stability(subscription_id) + logger.info(f"Unsubscribed from camera {subscription_id} (preserve_detection={preserve_detection})") + + async def process_streams(): + logger.info("Started processing streams") + try: + while True: + start_time = time.time() + with streams_lock: + current_streams = list(streams.items()) + if current_streams: + logger.debug(f"Processing {len(current_streams)} active streams") + else: + logger.debug("No active streams to process") + + for camera_id, stream in current_streams: + buffer = stream["buffer"] + if buffer.empty(): + logger.debug(f"Frame buffer is empty for camera {camera_id}") + continue + + logger.debug(f"Got frame from buffer for camera {camera_id}") + frame = buffer.get() + + # Cache the frame for REST API access + latest_frames[camera_id] = frame.copy() + logger.debug(f"Cached frame for REST API access for camera {camera_id}") + + with models_lock: + model_tree = models.get(camera_id, {}).get(stream["modelId"]) + if not model_tree: + logger.warning(f"Model not found for camera {camera_id}, modelId {stream['modelId']}") + continue + logger.debug(f"Found model tree for camera {camera_id}, modelId {stream['modelId']}") + + key = (camera_id, stream["modelId"]) + persistent_data = persistent_data_dict.get(key, {}) + logger.debug(f"Starting detection for camera {camera_id} with modelId {stream['modelId']}") + updated_persistent_data = await handle_detection( + camera_id, stream, frame, websocket, model_tree, persistent_data + ) + persistent_data_dict[key] = updated_persistent_data + + elapsed_time = (time.time() - start_time) * 1000 # ms + sleep_time = max(poll_interval - elapsed_time, 0) + logger.debug(f"Frame processing cycle: {elapsed_time:.2f}ms, sleeping for: {sleep_time:.2f}ms") + await asyncio.sleep(sleep_time / 1000.0) + except asyncio.CancelledError: + logger.info("Stream processing task cancelled") + except Exception as e: + logger.error(f"Error in process_streams: {str(e)}", exc_info=True) + + async def send_heartbeat(): + while True: + try: + cpu_usage = psutil.cpu_percent() + memory_usage = psutil.virtual_memory().percent + if torch.cuda.is_available(): + gpu_usage = torch.cuda.utilization() if hasattr(torch.cuda, 'utilization') else None + gpu_memory_usage = torch.cuda.memory_reserved() / (1024 ** 2) + else: + gpu_usage = None + gpu_memory_usage = None + + camera_connections = [ + { + "subscriptionIdentifier": stream["subscriptionIdentifier"], + "modelId": stream["modelId"], + "modelName": stream["modelName"], + "online": True, + # Include all subscription parameters for proper change detection + "rtspUrl": stream.get("rtsp_url"), + "snapshotUrl": stream.get("snapshot_url"), + "snapshotInterval": stream.get("snapshot_interval"), + **{k: v for k, v in get_crop_coords(stream).items() if v is not None} + } + for camera_id, stream in streams.items() + ] + + state_report = { + "type": "stateReport", + "cpuUsage": cpu_usage, + "memoryUsage": memory_usage, + "gpuUsage": gpu_usage, + "gpuMemoryUsage": gpu_memory_usage, + "cameraConnections": camera_connections + } + await websocket.send_text(json.dumps(state_report)) + logger.debug(f"Sent stateReport as heartbeat: CPU {cpu_usage:.1f}%, Memory {memory_usage:.1f}%, {len(camera_connections)} active cameras") + await asyncio.sleep(HEARTBEAT_INTERVAL) + except Exception as e: + logger.error(f"Error sending stateReport heartbeat: {e}") + break + + async def on_message(): + while True: + try: + msg = await websocket.receive_text() + ws_logger.info(f"RX <- {msg}") + logger.debug(f"Received message: {msg}") + data = json.loads(msg) + msg_type = data.get("type") + + if msg_type == "setSubscriptionList": + # Declarative approach: Backend sends list of subscriptions this worker should have + desired_subscriptions = data.get("subscriptions", []) + logger.info(f"Received subscription list with {len(desired_subscriptions)} subscriptions") + + await reconcile_subscriptions(desired_subscriptions, websocket) + + elif msg_type == "subscribe": + # Legacy support - convert single subscription to list + payload = data.get("payload", {}) + await reconcile_subscriptions([payload], websocket) + + elif msg_type == "unsubscribe": + # Legacy support - remove subscription + payload = data.get("payload", {}) + subscriptionIdentifier = payload.get("subscriptionIdentifier") + # Remove from current subscriptions and reconcile + current_subs = [] + with streams_lock: + for camera_id, stream in streams.items(): + if stream["subscriptionIdentifier"] != subscriptionIdentifier: + # Convert stream back to subscription format + current_subs.append({ + "subscriptionIdentifier": stream["subscriptionIdentifier"], + "rtspUrl": stream.get("rtsp_url"), + "snapshotUrl": stream.get("snapshot_url"), + "snapshotInterval": stream.get("snapshot_interval"), + "modelId": stream["modelId"], + "modelName": stream["modelName"], + "modelUrl": stream.get("modelUrl", ""), + "cropX1": stream.get("cropX1"), + "cropY1": stream.get("cropY1"), + "cropX2": stream.get("cropX2"), + "cropY2": stream.get("cropY2") + }) + await reconcile_subscriptions(current_subs, websocket) + + elif msg_type == "unsubscribe": + payload = data.get("payload", {}) + subscriptionIdentifier = payload.get("subscriptionIdentifier") + camera_id = subscriptionIdentifier + with streams_lock: + if camera_id and camera_id in streams: + stream = streams.pop(camera_id) + camera_url = subscription_to_camera.pop(camera_id, None) + + if camera_url and camera_url in camera_streams: + shared_stream = camera_streams[camera_url] + shared_stream["ref_count"] -= 1 + + # If no more references, stop the shared stream + if shared_stream["ref_count"] <= 0: + logger.info(f"Stopping shared stream for camera URL: {camera_url}") + shared_stream["stop_event"].set() + shared_stream["thread"].join() + if "cap" in shared_stream: + shared_stream["cap"].release() + del camera_streams[camera_url] + else: + logger.info(f"Shared stream for {camera_url} still has {shared_stream['ref_count']} references") + + # Clean up cached frame and stability tracking + latest_frames.pop(camera_id, None) + cached_detections.pop(camera_id, None) # Clear cached detection + frame_skip_flags.pop(camera_id, None) # Clear frame skip flag + camera_states.pop(camera_id, None) # Clear camera state + cleanup_camera_stability(camera_id) + logger.info(f"Unsubscribed from camera {camera_id}") + # Note: Keep models in memory for potential reuse + elif msg_type == "requestState": + cpu_usage = psutil.cpu_percent() + memory_usage = psutil.virtual_memory().percent + if torch.cuda.is_available(): + gpu_usage = torch.cuda.utilization() if hasattr(torch.cuda, 'utilization') else None + gpu_memory_usage = torch.cuda.memory_reserved() / (1024 ** 2) + else: + gpu_usage = None + gpu_memory_usage = None + + camera_connections = [ + { + "subscriptionIdentifier": stream["subscriptionIdentifier"], + "modelId": stream["modelId"], + "modelName": stream["modelName"], + "online": True, + # Include all subscription parameters for proper change detection + "rtspUrl": stream.get("rtsp_url"), + "snapshotUrl": stream.get("snapshot_url"), + "snapshotInterval": stream.get("snapshot_interval"), + **{k: v for k, v in get_crop_coords(stream).items() if v is not None} + } + for camera_id, stream in streams.items() + ] + + state_report = { + "type": "stateReport", + "cpuUsage": cpu_usage, + "memoryUsage": memory_usage, + "gpuUsage": gpu_usage, + "gpuMemoryUsage": gpu_memory_usage, + "cameraConnections": camera_connections + } + await websocket.send_text(json.dumps(state_report)) + + elif msg_type == "setSessionId": + payload = data.get("payload", {}) + display_identifier = payload.get("displayIdentifier") + session_id = payload.get("sessionId") + + # Debug sessionId value types and contents + session_id_type = type(session_id).__name__ + if session_id is None: + logger.info(f"🆔 BACKEND SESSIONID RECEIVED: displayId={display_identifier}, sessionId=None (type: {session_id_type})") + logger.info(f"🔄 BACKEND WANTS TO CLEAR SESSION for display {display_identifier}") + elif session_id == "null": + logger.info(f"🆔 BACKEND SESSIONID RECEIVED: displayId={display_identifier}, sessionId='null' (type: {session_id_type})") + logger.info(f"🔄 BACKEND SENT STRING 'null' for display {display_identifier}") + elif session_id == "": + logger.info(f"🆔 BACKEND SESSIONID RECEIVED: displayId={display_identifier}, sessionId='' (empty string, type: {session_id_type})") + logger.info(f"🔄 BACKEND SENT EMPTY STRING for display {display_identifier}") + else: + logger.info(f"🆔 BACKEND SESSIONID RECEIVED: displayId={display_identifier}, sessionId='{session_id}' (type: {session_id_type}, length: {len(str(session_id))})") + logger.info(f"🔄 BACKEND CREATED/UPDATED SESSION: {session_id} for display {display_identifier}") + + logger.debug(f"Full setSessionId payload: {payload}") + logger.debug(f"WebSocket message raw data: {json.dumps(data, indent=2)}") + logger.debug(f"Current active cameras: {list(streams.keys())}") + + if display_identifier: + # Store session ID for this display + if session_id is None or session_id == "null" or session_id == "": + old_session_id = session_ids.get(display_identifier) + session_ids.pop(display_identifier, None) + + if session_id is None: + logger.info(f"🚫 BACKEND ENDED SESSION: Cleared session ID for display {display_identifier} (was: {old_session_id}) - received None") + elif session_id == "null": + logger.info(f"🚫 BACKEND ENDED SESSION: Cleared session ID for display {display_identifier} (was: {old_session_id}) - received string 'null'") + elif session_id == "": + logger.info(f"🚫 BACKEND ENDED SESSION: Cleared session ID for display {display_identifier} (was: {old_session_id}) - received empty string") + + logger.debug(f"Session IDs after clearing: {session_ids}") + + # Reset tracking state for all cameras associated with this display + with streams_lock: + affected_cameras = [] + for camera_id, stream in streams.items(): + if stream["subscriptionIdentifier"].startswith(display_identifier + ";"): + affected_cameras.append(camera_id) + # Import here to avoid circular import + from siwatsystem.pympta import reset_tracking_state + model_id = stream.get("modelId", "unknown") + reset_tracking_state(camera_id, model_id, "backend session ended") + + + logger.info(f"Reset tracking for camera {camera_id} (display: {display_identifier})") + logger.debug(f"Reset tracking for {len(affected_cameras)} cameras: {affected_cameras}") + else: + old_session_id = session_ids.get(display_identifier) + session_ids[display_identifier] = session_id + logger.info(f"✅ BACKEND SESSION STARTED: Set session ID {session_id} for display {display_identifier} (previous: {old_session_id})") + logger.debug(f"Session IDs after update: {session_ids}") + logger.debug(f"🎯 CMS Backend created sessionId {session_id} after receiving detection data") + + # 🔑 LPR Integration: Retroactively cache the last detection by this new session_id + session_id_str = str(session_id) + logger.info(f"🔑 LPR: Attempting to retroactively cache detection for session_id {session_id_str}") + + # Find cameras associated with this display + display_cameras = [] + with streams_lock: + for camera_id, stream in streams.items(): + if stream["subscriptionIdentifier"].startswith(display_identifier + ";"): + display_cameras.append(camera_id) + + logger.debug(f"🔍 Found {len(display_cameras)} cameras for display {display_identifier}: {display_cameras}") + + # Cache the most recent detection for each camera by the new session_id + cached_count = 0 + for camera_id in display_cameras: + if camera_id in cached_detections: + detection_data = cached_detections[camera_id].copy() + + # Add sessionId to the detection data + detection_data['sessionId'] = session_id + + # Cache by session_id for LPR lookup + session_detections[session_id_str] = detection_data + session_to_camera[session_id_str] = camera_id + detection_timestamps[session_id_str] = time.time() + cached_count += 1 + + logger.info(f"✅ LPR: Cached detection for session_id {session_id_str} -> camera {camera_id}") + logger.debug(f"🔍 Detection data: {detection_data.get('data', {}).get('detection', {})}") + else: + logger.debug(f"⚠️ No cached detection available for camera {camera_id}") + + if cached_count > 0: + logger.info(f"🎉 LPR: Successfully cached {cached_count} detection(s) for session_id {session_id_str}") + logger.info(f"📊 Total LPR sessions now cached: {len(session_detections)}") + else: + logger.warning(f"⚠️ LPR: No detections could be cached for session_id {session_id_str}") + logger.warning(f" Display cameras: {display_cameras}") + logger.warning(f" Available cached detections: {list(cached_detections.keys())}") + + # Clear waiting state for cameras associated with this display + with streams_lock: + affected_cameras = [] + for camera_id, stream in streams.items(): + if stream["subscriptionIdentifier"].startswith(display_identifier + ";"): + affected_cameras.append(camera_id) + from siwatsystem.pympta import get_camera_stability_data + model_id = stream.get("modelId", "unknown") + stability_data = get_camera_stability_data(camera_id, model_id) + session_state = stability_data["session_state"] + if session_state.get("waiting_for_backend_session", False): + session_state["waiting_for_backend_session"] = False + session_state["wait_start_time"] = 0.0 + logger.info(f"🚀 PIPELINE UNBLOCKED: Backend sessionId {session_id} received - camera {camera_id} can proceed with database operations") + logger.debug(f"📋 Camera {camera_id}: SessionId {session_id} now available for future database operations") + logger.debug(f"Updated session state for {len(affected_cameras)} cameras: {affected_cameras}") + else: + logger.warning(f"🚨 Invalid setSessionId message: missing displayIdentifier in payload") + + elif msg_type == "patchSession": + session_id = data.get("sessionId") + patch_data = data.get("data", {}) + + # For now, just acknowledge the patch - actual implementation depends on backend requirements + response = { + "type": "patchSessionResult", + "payload": { + "sessionId": session_id, + "success": True, + "message": "Session patch acknowledged" + } + } + ws_logger.info(f"TX -> {json.dumps(response, separators=(',', ':'))}") + await websocket.send_json(response) + logger.info(f"Acknowledged patch for session {session_id}") + + elif msg_type == "setProgressionStage": + payload = data.get("payload", {}) + display_identifier = payload.get("displayIdentifier") + progression_stage = payload.get("progressionStage") + + logger.info(f"🏁 PROGRESSION STAGE RECEIVED: displayId={display_identifier}, stage={progression_stage}") + + if display_identifier: + # Find all cameras associated with this display + with streams_lock: + affected_cameras = [] + for camera_id, stream in streams.items(): + if stream["subscriptionIdentifier"].startswith(display_identifier + ";"): + affected_cameras.append(camera_id) + + logger.debug(f"🎯 Found {len(affected_cameras)} cameras for display {display_identifier}: {affected_cameras}") + + # Handle different progression stages + for camera_id in affected_cameras: + pipeline_state = get_or_init_session_pipeline_state(camera_id) + current_mode = pipeline_state.get("mode", "validation_detecting") + + if progression_stage == "car_fueling": + # Situation 2: Stop YOLO inference, continue sending cached detection dict + if current_mode == "lightweight": + pipeline_state["yolo_inference_enabled"] = False + pipeline_state["progression_stage"] = "car_fueling" + logger.info(f"⏸️ Camera {camera_id}: YOLO inference DISABLED for car_fueling stage (still sending cached detection dict)") + else: + logger.debug(f"📊 Camera {camera_id}: car_fueling received but not in lightweight mode (mode: {current_mode})") + + elif progression_stage == "car_waitpayment": + # Resume YOLO inference for absence counter + pipeline_state["yolo_inference_enabled"] = True + pipeline_state["progression_stage"] = "car_waitpayment" + logger.info(f"▶️ Camera {camera_id}: YOLO inference RE-ENABLED for car_waitpayment stage") + + elif progression_stage == "welcome": + # Ignore welcome messages during car_waitpayment as per requirement + current_progression = pipeline_state.get("progression_stage") + if current_progression == "car_waitpayment": + logger.info(f"🚫 Camera {camera_id}: IGNORING welcome stage (currently in car_waitpayment)") + else: + pipeline_state["progression_stage"] = "welcome" + logger.info(f"🎉 Camera {camera_id}: Progression stage set to welcome") + + elif progression_stage in ["car_wait_staff"]: + pipeline_state["progression_stage"] = progression_stage + logger.info(f"📋 Camera {camera_id}: Progression stage set to {progression_stage}") + else: + logger.warning(f"🚨 Invalid setProgressionStage message: missing displayIdentifier in payload") + + else: + logger.error(f"Unknown message type: {msg_type}") + except json.JSONDecodeError: + logger.error("Received invalid JSON message") + except (WebSocketDisconnect, ConnectionClosedError) as e: + logger.warning(f"WebSocket disconnected: {e}") + break + except Exception as e: + logger.error(f"Error handling message: {e}") + break try: - from core.monitoring.health import health_monitor - from core.monitoring.stream_health import stream_health_tracker - from core.streaming.buffers import shared_cache_buffer - - # Get basic metrics - overall_health = health_monitor.get_health_status() - stream_metrics = stream_health_tracker.get_all_metrics() - buffer_stats = shared_cache_buffer.get_stats() - - # Format for monitoring systems (Prometheus-style) - metrics = { - "detector_worker_up": 1, - "detector_worker_streams_total": len(stream_metrics), - "detector_worker_subscriptions_total": len(worker_state.subscriptions), - "detector_worker_sessions_total": len(worker_state.session_ids), - "detector_worker_memory_mb": buffer_stats.get("total_memory_mb", 0), - "detector_worker_health_status": { - "healthy": 1, - "warning": 2, - "critical": 3, - "unknown": 4 - }.get(overall_health.get("overall_status", "unknown"), 4) - } - - # Add per-stream metrics - for camera_id, stream_info in stream_metrics.items(): - safe_camera_id = camera_id.replace("-", "_").replace(".", "_") - metrics.update({ - f"detector_worker_stream_frames_total{{camera=\"{safe_camera_id}\"}}": stream_info.get("frame_count", 0), - f"detector_worker_stream_errors_total{{camera=\"{safe_camera_id}\"}}": stream_info.get("error_count", 0), - f"detector_worker_stream_fps{{camera=\"{safe_camera_id}\"}}": stream_info.get("frames_per_second", 0), - f"detector_worker_stream_frame_age_seconds{{camera=\"{safe_camera_id}\"}}": stream_info.get("last_frame_age_seconds") or 0 - }) - - return { - "timestamp": time.time(), - "metrics": metrics - } - + await websocket.accept() + stream_task = asyncio.create_task(process_streams()) + heartbeat_task = asyncio.create_task(send_heartbeat()) + message_task = asyncio.create_task(on_message()) + await asyncio.gather(heartbeat_task, message_task) except Exception as e: - logger.error(f"Error generating health metrics: {e}") - raise HTTPException(status_code=500, detail=f"Metrics error: {str(e)}") - - - - -if __name__ == "__main__": - import uvicorn - uvicorn.run(app, host="0.0.0.0", port=8001) \ No newline at end of file + logger.error(f"Error in detect websocket: {e}") + finally: + stream_task.cancel() + await stream_task + with streams_lock: + # Clean up shared camera streams + for camera_url, shared_stream in camera_streams.items(): + shared_stream["stop_event"].set() + shared_stream["thread"].join() + if "cap" in shared_stream: + shared_stream["cap"].release() + while not shared_stream["buffer"].empty(): + try: + shared_stream["buffer"].get_nowait() + except queue.Empty: + pass + logger.info(f"Released shared camera stream for {camera_url}") + + streams.clear() + camera_streams.clear() + subscription_to_camera.clear() + with models_lock: + # Clean up all model references before clearing models dict + for camera_id, camera_models in models.items(): + for model_id, model_tree in camera_models.items(): + logger.info(f"🧹 Shutdown cleanup: Releasing model {model_id} for camera {camera_id}") + # Release model registry references + cleanup_pipeline_node(model_tree) + # Release MPTA manager reference + release_mpta(model_id, camera_id) + models.clear() + + # Clean up the entire model registry and MPTA manager + # logger.info("🏭 Performing final model registry cleanup...") + # cleanup_registry() + # logger.info("🏭 Performing final MPTA manager cleanup...") + # cleanup_mpta_manager() + + latest_frames.clear() + cached_detections.clear() + frame_skip_flags.clear() + camera_states.clear() + cached_full_pipeline_results.clear() + session_pipeline_states.clear() + session_ids.clear() + # Clean up LPR integration caches + session_detections.clear() + session_to_camera.clear() + detection_timestamps.clear() + logger.info("WebSocket connection closed") diff --git a/archive/app.py b/archive/app.py deleted file mode 100644 index 09cb227..0000000 --- a/archive/app.py +++ /dev/null @@ -1,903 +0,0 @@ -from typing import Any, Dict -import os -import json -import time -import queue -import torch -import cv2 -import numpy as np -import base64 -import logging -import threading -import requests -import asyncio -import psutil -import zipfile -from urllib.parse import urlparse -from fastapi import FastAPI, WebSocket, HTTPException -from fastapi.websockets import WebSocketDisconnect -from fastapi.responses import Response -from websockets.exceptions import ConnectionClosedError -from ultralytics import YOLO - -# Import shared pipeline functions -from siwatsystem.pympta import load_pipeline_from_zip, run_pipeline - -app = FastAPI() - -# Global dictionaries to keep track of models and streams -# "models" now holds a nested dict: { camera_id: { modelId: model_tree } } -models: Dict[str, Dict[str, Any]] = {} -streams: Dict[str, Dict[str, Any]] = {} -# Store session IDs per display -session_ids: Dict[str, int] = {} -# Track shared camera streams by camera URL -camera_streams: Dict[str, Dict[str, Any]] = {} -# Map subscriptions to their camera URL -subscription_to_camera: Dict[str, str] = {} -# Store latest frames for REST API access (separate from processing buffer) -latest_frames: Dict[str, Any] = {} - -with open("config.json", "r") as f: - config = json.load(f) - -poll_interval = config.get("poll_interval_ms", 100) -reconnect_interval = config.get("reconnect_interval_sec", 5) -TARGET_FPS = config.get("target_fps", 10) -poll_interval = 1000 / TARGET_FPS -logging.info(f"Poll interval: {poll_interval}ms") -max_streams = config.get("max_streams", 5) -max_retries = config.get("max_retries", 3) - -# Configure logging -logging.basicConfig( - level=logging.INFO, # Set to INFO level for less verbose output - format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", - handlers=[ - logging.FileHandler("detector_worker.log"), # Write logs to a file - logging.StreamHandler() # Also output to console - ] -) - -# Create a logger specifically for this application -logger = logging.getLogger("detector_worker") -logger.setLevel(logging.DEBUG) # Set app-specific logger to DEBUG level - -# Ensure all other libraries (including root) use at least INFO level -logging.getLogger().setLevel(logging.INFO) - -logger.info("Starting detector worker application") -logger.info(f"Configuration: Target FPS: {TARGET_FPS}, Max streams: {max_streams}, Max retries: {max_retries}") - -# Ensure the models directory exists -os.makedirs("models", exist_ok=True) -logger.info("Ensured models directory exists") - -# Constants for heartbeat and timeouts -HEARTBEAT_INTERVAL = 2 # seconds -WORKER_TIMEOUT_MS = 10000 -logger.debug(f"Heartbeat interval set to {HEARTBEAT_INTERVAL} seconds") - -# Locks for thread-safe operations -streams_lock = threading.Lock() -models_lock = threading.Lock() -logger.debug("Initialized thread locks") - -# Add helper to download mpta ZIP file from a remote URL -def download_mpta(url: str, dest_path: str) -> str: - try: - logger.info(f"Starting download of model from {url} to {dest_path}") - os.makedirs(os.path.dirname(dest_path), exist_ok=True) - response = requests.get(url, stream=True) - if response.status_code == 200: - file_size = int(response.headers.get('content-length', 0)) - logger.info(f"Model file size: {file_size/1024/1024:.2f} MB") - downloaded = 0 - with open(dest_path, "wb") as f: - for chunk in response.iter_content(chunk_size=8192): - f.write(chunk) - downloaded += len(chunk) - if file_size > 0 and downloaded % (file_size // 10) < 8192: # Log approximately every 10% - logger.debug(f"Download progress: {downloaded/file_size*100:.1f}%") - logger.info(f"Successfully downloaded mpta file from {url} to {dest_path}") - return dest_path - else: - logger.error(f"Failed to download mpta file (status code {response.status_code}): {response.text}") - return None - except Exception as e: - logger.error(f"Exception downloading mpta file from {url}: {str(e)}", exc_info=True) - return None - -# Add helper to fetch snapshot image from HTTP/HTTPS URL -def fetch_snapshot(url: str): - try: - from requests.auth import HTTPBasicAuth, HTTPDigestAuth - - # Parse URL to extract credentials - parsed = urlparse(url) - - # Prepare headers - some cameras require User-Agent - headers = { - 'User-Agent': 'Mozilla/5.0 (compatible; DetectorWorker/1.0)' - } - - # Reconstruct URL without credentials - clean_url = f"{parsed.scheme}://{parsed.hostname}" - if parsed.port: - clean_url += f":{parsed.port}" - clean_url += parsed.path - if parsed.query: - clean_url += f"?{parsed.query}" - - auth = None - if parsed.username and parsed.password: - # Try HTTP Digest authentication first (common for IP cameras) - try: - auth = HTTPDigestAuth(parsed.username, parsed.password) - response = requests.get(clean_url, auth=auth, headers=headers, timeout=10) - if response.status_code == 200: - logger.debug(f"Successfully authenticated using HTTP Digest for {clean_url}") - elif response.status_code == 401: - # If Digest fails, try Basic auth - logger.debug(f"HTTP Digest failed, trying Basic auth for {clean_url}") - auth = HTTPBasicAuth(parsed.username, parsed.password) - response = requests.get(clean_url, auth=auth, headers=headers, timeout=10) - if response.status_code == 200: - logger.debug(f"Successfully authenticated using HTTP Basic for {clean_url}") - except Exception as auth_error: - logger.debug(f"Authentication setup error: {auth_error}") - # Fallback to original URL with embedded credentials - response = requests.get(url, headers=headers, timeout=10) - else: - # No credentials in URL, make request as-is - response = requests.get(url, headers=headers, timeout=10) - - if response.status_code == 200: - # Convert response content to numpy array - nparr = np.frombuffer(response.content, np.uint8) - # Decode image - frame = cv2.imdecode(nparr, cv2.IMREAD_COLOR) - if frame is not None: - logger.debug(f"Successfully fetched snapshot from {clean_url}, shape: {frame.shape}") - return frame - else: - logger.error(f"Failed to decode image from snapshot URL: {clean_url}") - return None - else: - logger.error(f"Failed to fetch snapshot (status code {response.status_code}): {clean_url}") - return None - except Exception as e: - logger.error(f"Exception fetching snapshot from {url}: {str(e)}") - return None - -# Helper to get crop coordinates from stream -def get_crop_coords(stream): - return { - "cropX1": stream.get("cropX1"), - "cropY1": stream.get("cropY1"), - "cropX2": stream.get("cropX2"), - "cropY2": stream.get("cropY2") - } - -#################################################### -# REST API endpoint for image retrieval -#################################################### -@app.get("/camera/{camera_id}/image") -async def get_camera_image(camera_id: str): - """ - Get the current frame from a camera as JPEG image - """ - try: - # URL decode the camera_id to handle encoded characters like %3B for semicolon - from urllib.parse import unquote - original_camera_id = camera_id - camera_id = unquote(camera_id) - logger.debug(f"REST API request: original='{original_camera_id}', decoded='{camera_id}'") - - with streams_lock: - if camera_id not in streams: - logger.warning(f"Camera ID '{camera_id}' not found in streams. Current streams: {list(streams.keys())}") - raise HTTPException(status_code=404, detail=f"Camera {camera_id} not found or not active") - - # Check if we have a cached frame for this camera - if camera_id not in latest_frames: - logger.warning(f"No cached frame available for camera '{camera_id}'.") - raise HTTPException(status_code=404, detail=f"No frame available for camera {camera_id}") - - frame = latest_frames[camera_id] - logger.debug(f"Retrieved cached frame for camera '{camera_id}', frame shape: {frame.shape}") - # Encode frame as JPEG - success, buffer_img = cv2.imencode('.jpg', frame, [cv2.IMWRITE_JPEG_QUALITY, 85]) - if not success: - raise HTTPException(status_code=500, detail="Failed to encode image as JPEG") - - # Return image as binary response - return Response(content=buffer_img.tobytes(), media_type="image/jpeg") - - except HTTPException: - raise - except Exception as e: - logger.error(f"Error retrieving image for camera {camera_id}: {str(e)}", exc_info=True) - raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}") - -#################################################### -# Detection and frame processing functions -#################################################### -@app.websocket("/") -async def detect(websocket: WebSocket): - logger.info("WebSocket connection accepted") - persistent_data_dict = {} - - async def handle_detection(camera_id, stream, frame, websocket, model_tree, persistent_data): - try: - # Apply crop if specified - cropped_frame = frame - if all(coord is not None for coord in [stream.get("cropX1"), stream.get("cropY1"), stream.get("cropX2"), stream.get("cropY2")]): - cropX1, cropY1, cropX2, cropY2 = stream["cropX1"], stream["cropY1"], stream["cropX2"], stream["cropY2"] - cropped_frame = frame[cropY1:cropY2, cropX1:cropX2] - logger.debug(f"Applied crop coordinates ({cropX1}, {cropY1}, {cropX2}, {cropY2}) to frame for camera {camera_id}") - - logger.debug(f"Processing frame for camera {camera_id} with model {stream['modelId']}") - start_time = time.time() - - # Extract display identifier for session ID lookup - subscription_parts = stream["subscriptionIdentifier"].split(';') - display_identifier = subscription_parts[0] if subscription_parts else None - session_id = session_ids.get(display_identifier) if display_identifier else None - - # Create context for pipeline execution - pipeline_context = { - "camera_id": camera_id, - "display_id": display_identifier, - "session_id": session_id - } - - detection_result = run_pipeline(cropped_frame, model_tree, context=pipeline_context) - process_time = (time.time() - start_time) * 1000 - logger.debug(f"Detection for camera {camera_id} completed in {process_time:.2f}ms") - - # Log the raw detection result for debugging - logger.debug(f"Raw detection result for camera {camera_id}:\n{json.dumps(detection_result, indent=2, default=str)}") - - # Direct class result (no detections/classifications structure) - if detection_result and isinstance(detection_result, dict) and "class" in detection_result and "confidence" in detection_result: - highest_confidence_detection = { - "class": detection_result.get("class", "none"), - "confidence": detection_result.get("confidence", 1.0), - "box": [0, 0, 0, 0] # Empty bounding box for classifications - } - # Handle case when no detections found or result is empty - elif not detection_result or not detection_result.get("detections"): - # Check if we have classification results - if detection_result and detection_result.get("classifications"): - # Get the highest confidence classification - classifications = detection_result.get("classifications", []) - highest_confidence_class = max(classifications, key=lambda x: x.get("confidence", 0)) if classifications else None - - if highest_confidence_class: - highest_confidence_detection = { - "class": highest_confidence_class.get("class", "none"), - "confidence": highest_confidence_class.get("confidence", 1.0), - "box": [0, 0, 0, 0] # Empty bounding box for classifications - } - else: - highest_confidence_detection = { - "class": "none", - "confidence": 1.0, - "box": [0, 0, 0, 0] - } - else: - highest_confidence_detection = { - "class": "none", - "confidence": 1.0, - "box": [0, 0, 0, 0] - } - else: - # Find detection with highest confidence - detections = detection_result.get("detections", []) - highest_confidence_detection = max(detections, key=lambda x: x.get("confidence", 0)) if detections else { - "class": "none", - "confidence": 1.0, - "box": [0, 0, 0, 0] - } - - # Convert detection format to match protocol - flatten detection attributes - detection_dict = {} - - # Handle different detection result formats - if isinstance(highest_confidence_detection, dict): - # Copy all fields from the detection result - for key, value in highest_confidence_detection.items(): - if key not in ["box", "id"]: # Skip internal fields - detection_dict[key] = value - - detection_data = { - "type": "imageDetection", - "subscriptionIdentifier": stream["subscriptionIdentifier"], - "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S.%fZ", time.gmtime()), - "data": { - "detection": detection_dict, - "modelId": stream["modelId"], - "modelName": stream["modelName"] - } - } - - # Add session ID if available - if session_id is not None: - detection_data["sessionId"] = session_id - - if highest_confidence_detection["class"] != "none": - logger.info(f"Camera {camera_id}: Detected {highest_confidence_detection['class']} with confidence {highest_confidence_detection['confidence']:.2f} using model {stream['modelName']}") - - # Log session ID if available - if session_id: - logger.debug(f"Detection associated with session ID: {session_id}") - - await websocket.send_json(detection_data) - logger.debug(f"Sent detection data to client for camera {camera_id}") - return persistent_data - except Exception as e: - logger.error(f"Error in handle_detection for camera {camera_id}: {str(e)}", exc_info=True) - return persistent_data - - def frame_reader(camera_id, cap, buffer, stop_event): - retries = 0 - logger.info(f"Starting frame reader thread for camera {camera_id}") - frame_count = 0 - last_log_time = time.time() - - try: - # Log initial camera status and properties - if cap.isOpened(): - width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) - height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) - fps = cap.get(cv2.CAP_PROP_FPS) - logger.info(f"Camera {camera_id} opened successfully with resolution {width}x{height}, FPS: {fps}") - else: - logger.error(f"Camera {camera_id} failed to open initially") - - while not stop_event.is_set(): - try: - if not cap.isOpened(): - logger.error(f"Camera {camera_id} is not open before trying to read") - # Attempt to reopen - cap = cv2.VideoCapture(streams[camera_id]["rtsp_url"]) - time.sleep(reconnect_interval) - continue - - logger.debug(f"Attempting to read frame from camera {camera_id}") - ret, frame = cap.read() - - if not ret: - logger.warning(f"Connection lost for camera: {camera_id}, retry {retries+1}/{max_retries}") - cap.release() - time.sleep(reconnect_interval) - retries += 1 - if retries > max_retries and max_retries != -1: - logger.error(f"Max retries reached for camera: {camera_id}, stopping frame reader") - break - # Re-open - logger.info(f"Attempting to reopen RTSP stream for camera: {camera_id}") - cap = cv2.VideoCapture(streams[camera_id]["rtsp_url"]) - if not cap.isOpened(): - logger.error(f"Failed to reopen RTSP stream for camera: {camera_id}") - continue - logger.info(f"Successfully reopened RTSP stream for camera: {camera_id}") - continue - - # Successfully read a frame - frame_count += 1 - current_time = time.time() - # Log frame stats every 5 seconds - if current_time - last_log_time > 5: - logger.info(f"Camera {camera_id}: Read {frame_count} frames in the last {current_time - last_log_time:.1f} seconds") - frame_count = 0 - last_log_time = current_time - - logger.debug(f"Successfully read frame from camera {camera_id}, shape: {frame.shape}") - retries = 0 - - # Overwrite old frame if buffer is full - if not buffer.empty(): - try: - buffer.get_nowait() - logger.debug(f"[frame_reader] Removed old frame from buffer for camera {camera_id}") - except queue.Empty: - pass - buffer.put(frame) - logger.debug(f"[frame_reader] Added new frame to buffer for camera {camera_id}. Buffer size: {buffer.qsize()}") - - # Short sleep to avoid CPU overuse - time.sleep(0.01) - - except cv2.error as e: - logger.error(f"OpenCV error for camera {camera_id}: {e}", exc_info=True) - cap.release() - time.sleep(reconnect_interval) - retries += 1 - if retries > max_retries and max_retries != -1: - logger.error(f"Max retries reached after OpenCV error for camera {camera_id}") - break - logger.info(f"Attempting to reopen RTSP stream after OpenCV error for camera: {camera_id}") - cap = cv2.VideoCapture(streams[camera_id]["rtsp_url"]) - if not cap.isOpened(): - logger.error(f"Failed to reopen RTSP stream for camera {camera_id} after OpenCV error") - continue - logger.info(f"Successfully reopened RTSP stream after OpenCV error for camera: {camera_id}") - except Exception as e: - logger.error(f"Unexpected error for camera {camera_id}: {str(e)}", exc_info=True) - cap.release() - break - except Exception as e: - logger.error(f"Error in frame_reader thread for camera {camera_id}: {str(e)}", exc_info=True) - finally: - logger.info(f"Frame reader thread for camera {camera_id} is exiting") - if cap and cap.isOpened(): - cap.release() - - def snapshot_reader(camera_id, snapshot_url, snapshot_interval, buffer, stop_event): - """Frame reader that fetches snapshots from HTTP/HTTPS URL at specified intervals""" - retries = 0 - logger.info(f"Starting snapshot reader thread for camera {camera_id} from {snapshot_url}") - frame_count = 0 - last_log_time = time.time() - - try: - interval_seconds = snapshot_interval / 1000.0 # Convert milliseconds to seconds - logger.info(f"Snapshot interval for camera {camera_id}: {interval_seconds}s") - - while not stop_event.is_set(): - try: - start_time = time.time() - frame = fetch_snapshot(snapshot_url) - - if frame is None: - logger.warning(f"Failed to fetch snapshot for camera: {camera_id}, retry {retries+1}/{max_retries}") - retries += 1 - if retries > max_retries and max_retries != -1: - logger.error(f"Max retries reached for snapshot camera: {camera_id}, stopping reader") - break - time.sleep(min(interval_seconds, reconnect_interval)) - continue - - # Successfully fetched a frame - frame_count += 1 - current_time = time.time() - # Log frame stats every 5 seconds - if current_time - last_log_time > 5: - logger.info(f"Camera {camera_id}: Fetched {frame_count} snapshots in the last {current_time - last_log_time:.1f} seconds") - frame_count = 0 - last_log_time = current_time - - logger.debug(f"Successfully fetched snapshot from camera {camera_id}, shape: {frame.shape}") - retries = 0 - - # Overwrite old frame if buffer is full - if not buffer.empty(): - try: - buffer.get_nowait() - logger.debug(f"[snapshot_reader] Removed old snapshot from buffer for camera {camera_id}") - except queue.Empty: - pass - buffer.put(frame) - logger.debug(f"[snapshot_reader] Added new snapshot to buffer for camera {camera_id}. Buffer size: {buffer.qsize()}") - - # Wait for the specified interval - elapsed = time.time() - start_time - sleep_time = max(interval_seconds - elapsed, 0) - if sleep_time > 0: - time.sleep(sleep_time) - - except Exception as e: - logger.error(f"Unexpected error fetching snapshot for camera {camera_id}: {str(e)}", exc_info=True) - retries += 1 - if retries > max_retries and max_retries != -1: - logger.error(f"Max retries reached after error for snapshot camera {camera_id}") - break - time.sleep(min(interval_seconds, reconnect_interval)) - except Exception as e: - logger.error(f"Error in snapshot_reader thread for camera {camera_id}: {str(e)}", exc_info=True) - finally: - logger.info(f"Snapshot reader thread for camera {camera_id} is exiting") - - async def process_streams(): - logger.info("Started processing streams") - try: - while True: - start_time = time.time() - with streams_lock: - current_streams = list(streams.items()) - if current_streams: - logger.debug(f"Processing {len(current_streams)} active streams") - else: - logger.debug("No active streams to process") - - for camera_id, stream in current_streams: - buffer = stream["buffer"] - if buffer.empty(): - logger.debug(f"Frame buffer is empty for camera {camera_id}") - continue - - logger.debug(f"Got frame from buffer for camera {camera_id}") - frame = buffer.get() - - # Cache the frame for REST API access - latest_frames[camera_id] = frame.copy() - logger.debug(f"Cached frame for REST API access for camera {camera_id}") - - with models_lock: - model_tree = models.get(camera_id, {}).get(stream["modelId"]) - if not model_tree: - logger.warning(f"Model not found for camera {camera_id}, modelId {stream['modelId']}") - continue - logger.debug(f"Found model tree for camera {camera_id}, modelId {stream['modelId']}") - - key = (camera_id, stream["modelId"]) - persistent_data = persistent_data_dict.get(key, {}) - logger.debug(f"Starting detection for camera {camera_id} with modelId {stream['modelId']}") - updated_persistent_data = await handle_detection( - camera_id, stream, frame, websocket, model_tree, persistent_data - ) - persistent_data_dict[key] = updated_persistent_data - - elapsed_time = (time.time() - start_time) * 1000 # ms - sleep_time = max(poll_interval - elapsed_time, 0) - logger.debug(f"Frame processing cycle: {elapsed_time:.2f}ms, sleeping for: {sleep_time:.2f}ms") - await asyncio.sleep(sleep_time / 1000.0) - except asyncio.CancelledError: - logger.info("Stream processing task cancelled") - except Exception as e: - logger.error(f"Error in process_streams: {str(e)}", exc_info=True) - - async def send_heartbeat(): - while True: - try: - cpu_usage = psutil.cpu_percent() - memory_usage = psutil.virtual_memory().percent - if torch.cuda.is_available(): - gpu_usage = torch.cuda.utilization() if hasattr(torch.cuda, 'utilization') else None - gpu_memory_usage = torch.cuda.memory_reserved() / (1024 ** 2) - else: - gpu_usage = None - gpu_memory_usage = None - - camera_connections = [ - { - "subscriptionIdentifier": stream["subscriptionIdentifier"], - "modelId": stream["modelId"], - "modelName": stream["modelName"], - "online": True, - **{k: v for k, v in get_crop_coords(stream).items() if v is not None} - } - for camera_id, stream in streams.items() - ] - - state_report = { - "type": "stateReport", - "cpuUsage": cpu_usage, - "memoryUsage": memory_usage, - "gpuUsage": gpu_usage, - "gpuMemoryUsage": gpu_memory_usage, - "cameraConnections": camera_connections - } - await websocket.send_text(json.dumps(state_report)) - logger.debug(f"Sent stateReport as heartbeat: CPU {cpu_usage:.1f}%, Memory {memory_usage:.1f}%, {len(camera_connections)} active cameras") - await asyncio.sleep(HEARTBEAT_INTERVAL) - except Exception as e: - logger.error(f"Error sending stateReport heartbeat: {e}") - break - - async def on_message(): - while True: - try: - msg = await websocket.receive_text() - logger.debug(f"Received message: {msg}") - data = json.loads(msg) - msg_type = data.get("type") - - if msg_type == "subscribe": - payload = data.get("payload", {}) - subscriptionIdentifier = payload.get("subscriptionIdentifier") - rtsp_url = payload.get("rtspUrl") - snapshot_url = payload.get("snapshotUrl") - snapshot_interval = payload.get("snapshotInterval") - model_url = payload.get("modelUrl") - modelId = payload.get("modelId") - modelName = payload.get("modelName") - cropX1 = payload.get("cropX1") - cropY1 = payload.get("cropY1") - cropX2 = payload.get("cropX2") - cropY2 = payload.get("cropY2") - - # Extract camera_id from subscriptionIdentifier (format: displayIdentifier;cameraIdentifier) - parts = subscriptionIdentifier.split(';') - if len(parts) != 2: - logger.error(f"Invalid subscriptionIdentifier format: {subscriptionIdentifier}") - continue - - display_identifier, camera_identifier = parts - camera_id = subscriptionIdentifier # Use full subscriptionIdentifier as camera_id for mapping - - if model_url: - with models_lock: - if (camera_id not in models) or (modelId not in models[camera_id]): - logger.info(f"Loading model from {model_url} for camera {camera_id}, modelId {modelId}") - extraction_dir = os.path.join("models", camera_identifier, str(modelId)) - os.makedirs(extraction_dir, exist_ok=True) - # If model_url is remote, download it first. - parsed = urlparse(model_url) - if parsed.scheme in ("http", "https"): - logger.info(f"Downloading remote .mpta file from {model_url}") - filename = os.path.basename(parsed.path) or f"model_{modelId}.mpta" - local_mpta = os.path.join(extraction_dir, filename) - logger.debug(f"Download destination: {local_mpta}") - local_path = download_mpta(model_url, local_mpta) - if not local_path: - logger.error(f"Failed to download the remote .mpta file from {model_url}") - error_response = { - "type": "error", - "subscriptionIdentifier": subscriptionIdentifier, - "error": f"Failed to download model from {model_url}" - } - await websocket.send_json(error_response) - continue - model_tree = load_pipeline_from_zip(local_path, extraction_dir) - else: - logger.info(f"Loading local .mpta file from {model_url}") - # Check if file exists before attempting to load - if not os.path.exists(model_url): - logger.error(f"Local .mpta file not found: {model_url}") - logger.debug(f"Current working directory: {os.getcwd()}") - error_response = { - "type": "error", - "subscriptionIdentifier": subscriptionIdentifier, - "error": f"Model file not found: {model_url}" - } - await websocket.send_json(error_response) - continue - model_tree = load_pipeline_from_zip(model_url, extraction_dir) - if model_tree is None: - logger.error(f"Failed to load model {modelId} from .mpta file for camera {camera_id}") - error_response = { - "type": "error", - "subscriptionIdentifier": subscriptionIdentifier, - "error": f"Failed to load model {modelId}" - } - await websocket.send_json(error_response) - continue - if camera_id not in models: - models[camera_id] = {} - models[camera_id][modelId] = model_tree - logger.info(f"Successfully loaded model {modelId} for camera {camera_id}") - logger.debug(f"Model extraction directory: {extraction_dir}") - if camera_id and (rtsp_url or snapshot_url): - with streams_lock: - # Determine camera URL for shared stream management - camera_url = snapshot_url if snapshot_url else rtsp_url - - if camera_id not in streams and len(streams) < max_streams: - # Check if we already have a stream for this camera URL - shared_stream = camera_streams.get(camera_url) - - if shared_stream: - # Reuse existing stream - logger.info(f"Reusing existing stream for camera URL: {camera_url}") - buffer = shared_stream["buffer"] - stop_event = shared_stream["stop_event"] - thread = shared_stream["thread"] - mode = shared_stream["mode"] - - # Increment reference count - shared_stream["ref_count"] = shared_stream.get("ref_count", 0) + 1 - else: - # Create new stream - buffer = queue.Queue(maxsize=1) - stop_event = threading.Event() - - if snapshot_url and snapshot_interval: - logger.info(f"Creating new snapshot stream for camera {camera_id}: {snapshot_url}") - thread = threading.Thread(target=snapshot_reader, args=(camera_id, snapshot_url, snapshot_interval, buffer, stop_event)) - thread.daemon = True - thread.start() - mode = "snapshot" - - # Store shared stream info - shared_stream = { - "buffer": buffer, - "thread": thread, - "stop_event": stop_event, - "mode": mode, - "url": snapshot_url, - "snapshot_interval": snapshot_interval, - "ref_count": 1 - } - camera_streams[camera_url] = shared_stream - - elif rtsp_url: - logger.info(f"Creating new RTSP stream for camera {camera_id}: {rtsp_url}") - cap = cv2.VideoCapture(rtsp_url) - if not cap.isOpened(): - logger.error(f"Failed to open RTSP stream for camera {camera_id}") - continue - thread = threading.Thread(target=frame_reader, args=(camera_id, cap, buffer, stop_event)) - thread.daemon = True - thread.start() - mode = "rtsp" - - # Store shared stream info - shared_stream = { - "buffer": buffer, - "thread": thread, - "stop_event": stop_event, - "mode": mode, - "url": rtsp_url, - "cap": cap, - "ref_count": 1 - } - camera_streams[camera_url] = shared_stream - else: - logger.error(f"No valid URL provided for camera {camera_id}") - continue - - # Create stream info for this subscription - stream_info = { - "buffer": buffer, - "thread": thread, - "stop_event": stop_event, - "modelId": modelId, - "modelName": modelName, - "subscriptionIdentifier": subscriptionIdentifier, - "cropX1": cropX1, - "cropY1": cropY1, - "cropX2": cropX2, - "cropY2": cropY2, - "mode": mode, - "camera_url": camera_url - } - - if mode == "snapshot": - stream_info["snapshot_url"] = snapshot_url - stream_info["snapshot_interval"] = snapshot_interval - elif mode == "rtsp": - stream_info["rtsp_url"] = rtsp_url - stream_info["cap"] = shared_stream["cap"] - - streams[camera_id] = stream_info - subscription_to_camera[camera_id] = camera_url - - elif camera_id and camera_id in streams: - # If already subscribed, unsubscribe first - logger.info(f"Resubscribing to camera {camera_id}") - # Note: Keep models in memory for reuse across subscriptions - elif msg_type == "unsubscribe": - payload = data.get("payload", {}) - subscriptionIdentifier = payload.get("subscriptionIdentifier") - camera_id = subscriptionIdentifier - with streams_lock: - if camera_id and camera_id in streams: - stream = streams.pop(camera_id) - camera_url = subscription_to_camera.pop(camera_id, None) - - if camera_url and camera_url in camera_streams: - shared_stream = camera_streams[camera_url] - shared_stream["ref_count"] -= 1 - - # If no more references, stop the shared stream - if shared_stream["ref_count"] <= 0: - logger.info(f"Stopping shared stream for camera URL: {camera_url}") - shared_stream["stop_event"].set() - shared_stream["thread"].join() - if "cap" in shared_stream: - shared_stream["cap"].release() - del camera_streams[camera_url] - else: - logger.info(f"Shared stream for {camera_url} still has {shared_stream['ref_count']} references") - - # Clean up cached frame - latest_frames.pop(camera_id, None) - logger.info(f"Unsubscribed from camera {camera_id}") - # Note: Keep models in memory for potential reuse - elif msg_type == "requestState": - cpu_usage = psutil.cpu_percent() - memory_usage = psutil.virtual_memory().percent - if torch.cuda.is_available(): - gpu_usage = torch.cuda.utilization() if hasattr(torch.cuda, 'utilization') else None - gpu_memory_usage = torch.cuda.memory_reserved() / (1024 ** 2) - else: - gpu_usage = None - gpu_memory_usage = None - - camera_connections = [ - { - "subscriptionIdentifier": stream["subscriptionIdentifier"], - "modelId": stream["modelId"], - "modelName": stream["modelName"], - "online": True, - **{k: v for k, v in get_crop_coords(stream).items() if v is not None} - } - for camera_id, stream in streams.items() - ] - - state_report = { - "type": "stateReport", - "cpuUsage": cpu_usage, - "memoryUsage": memory_usage, - "gpuUsage": gpu_usage, - "gpuMemoryUsage": gpu_memory_usage, - "cameraConnections": camera_connections - } - await websocket.send_text(json.dumps(state_report)) - - elif msg_type == "setSessionId": - payload = data.get("payload", {}) - display_identifier = payload.get("displayIdentifier") - session_id = payload.get("sessionId") - - if display_identifier: - # Store session ID for this display - if session_id is None: - session_ids.pop(display_identifier, None) - logger.info(f"Cleared session ID for display {display_identifier}") - else: - session_ids[display_identifier] = session_id - logger.info(f"Set session ID {session_id} for display {display_identifier}") - - elif msg_type == "patchSession": - session_id = data.get("sessionId") - patch_data = data.get("data", {}) - - # For now, just acknowledge the patch - actual implementation depends on backend requirements - response = { - "type": "patchSessionResult", - "payload": { - "sessionId": session_id, - "success": True, - "message": "Session patch acknowledged" - } - } - await websocket.send_json(response) - logger.info(f"Acknowledged patch for session {session_id}") - - else: - logger.error(f"Unknown message type: {msg_type}") - except json.JSONDecodeError: - logger.error("Received invalid JSON message") - except (WebSocketDisconnect, ConnectionClosedError) as e: - logger.warning(f"WebSocket disconnected: {e}") - break - except Exception as e: - logger.error(f"Error handling message: {e}") - break - try: - await websocket.accept() - stream_task = asyncio.create_task(process_streams()) - heartbeat_task = asyncio.create_task(send_heartbeat()) - message_task = asyncio.create_task(on_message()) - await asyncio.gather(heartbeat_task, message_task) - except Exception as e: - logger.error(f"Error in detect websocket: {e}") - finally: - stream_task.cancel() - await stream_task - with streams_lock: - # Clean up shared camera streams - for camera_url, shared_stream in camera_streams.items(): - shared_stream["stop_event"].set() - shared_stream["thread"].join() - if "cap" in shared_stream: - shared_stream["cap"].release() - while not shared_stream["buffer"].empty(): - try: - shared_stream["buffer"].get_nowait() - except queue.Empty: - pass - logger.info(f"Released shared camera stream for {camera_url}") - - streams.clear() - camera_streams.clear() - subscription_to_camera.clear() - with models_lock: - models.clear() - latest_frames.clear() - session_ids.clear() - logger.info("WebSocket connection closed") diff --git a/archive/siwatsystem/pympta.py b/archive/siwatsystem/pympta.py deleted file mode 100644 index d21232d..0000000 --- a/archive/siwatsystem/pympta.py +++ /dev/null @@ -1,798 +0,0 @@ -import os -import json -import logging -import torch -import cv2 -import zipfile -import shutil -import traceback -import redis -import time -import uuid -import concurrent.futures -from ultralytics import YOLO -from urllib.parse import urlparse -from .database import DatabaseManager - -# Create a logger specifically for this module -logger = logging.getLogger("detector_worker.pympta") - -def validate_redis_config(redis_config: dict) -> bool: - """Validate Redis configuration parameters.""" - required_fields = ["host", "port"] - for field in required_fields: - if field not in redis_config: - logger.error(f"Missing required Redis config field: {field}") - return False - - if not isinstance(redis_config["port"], int) or redis_config["port"] <= 0: - logger.error(f"Invalid Redis port: {redis_config['port']}") - return False - - return True - -def validate_postgresql_config(pg_config: dict) -> bool: - """Validate PostgreSQL configuration parameters.""" - required_fields = ["host", "port", "database", "username", "password"] - for field in required_fields: - if field not in pg_config: - logger.error(f"Missing required PostgreSQL config field: {field}") - return False - - if not isinstance(pg_config["port"], int) or pg_config["port"] <= 0: - logger.error(f"Invalid PostgreSQL port: {pg_config['port']}") - return False - - return True - -def crop_region_by_class(frame, regions_dict, class_name): - """Crop a specific region from frame based on detected class.""" - if class_name not in regions_dict: - logger.warning(f"Class '{class_name}' not found in detected regions") - return None - - bbox = regions_dict[class_name]['bbox'] - x1, y1, x2, y2 = bbox - cropped = frame[y1:y2, x1:x2] - - if cropped.size == 0: - logger.warning(f"Empty crop for class '{class_name}' with bbox {bbox}") - return None - - return cropped - -def format_action_context(base_context, additional_context=None): - """Format action context with dynamic values.""" - context = {**base_context} - if additional_context: - context.update(additional_context) - return context - -def load_pipeline_node(node_config: dict, mpta_dir: str, redis_client, db_manager=None) -> dict: - # Recursively load a model node from configuration. - model_path = os.path.join(mpta_dir, node_config["modelFile"]) - if not os.path.exists(model_path): - logger.error(f"Model file {model_path} not found. Current directory: {os.getcwd()}") - logger.error(f"Directory content: {os.listdir(os.path.dirname(model_path))}") - raise FileNotFoundError(f"Model file {model_path} not found.") - logger.info(f"Loading model for node {node_config['modelId']} from {model_path}") - model = YOLO(model_path) - if torch.cuda.is_available(): - logger.info(f"CUDA available. Moving model {node_config['modelId']} to GPU") - model.to("cuda") - else: - logger.info(f"CUDA not available. Using CPU for model {node_config['modelId']}") - - # Prepare trigger class indices for optimization - trigger_classes = node_config.get("triggerClasses", []) - trigger_class_indices = None - if trigger_classes and hasattr(model, "names"): - # Convert class names to indices for the model - trigger_class_indices = [i for i, name in model.names.items() - if name in trigger_classes] - logger.debug(f"Converted trigger classes to indices: {trigger_class_indices}") - - node = { - "modelId": node_config["modelId"], - "modelFile": node_config["modelFile"], - "triggerClasses": trigger_classes, - "triggerClassIndices": trigger_class_indices, - "crop": node_config.get("crop", False), - "cropClass": node_config.get("cropClass"), - "minConfidence": node_config.get("minConfidence", None), - "multiClass": node_config.get("multiClass", False), - "expectedClasses": node_config.get("expectedClasses", []), - "parallel": node_config.get("parallel", False), - "actions": node_config.get("actions", []), - "parallelActions": node_config.get("parallelActions", []), - "model": model, - "branches": [], - "redis_client": redis_client, - "db_manager": db_manager - } - logger.debug(f"Configured node {node_config['modelId']} with trigger classes: {node['triggerClasses']}") - for child in node_config.get("branches", []): - logger.debug(f"Loading branch for parent node {node_config['modelId']}") - node["branches"].append(load_pipeline_node(child, mpta_dir, redis_client, db_manager)) - return node - -def load_pipeline_from_zip(zip_source: str, target_dir: str) -> dict: - logger.info(f"Attempting to load pipeline from {zip_source} to {target_dir}") - os.makedirs(target_dir, exist_ok=True) - zip_path = os.path.join(target_dir, "pipeline.mpta") - - # Parse the source; only local files are supported here. - parsed = urlparse(zip_source) - if parsed.scheme in ("", "file"): - local_path = parsed.path if parsed.scheme == "file" else zip_source - logger.debug(f"Checking if local file exists: {local_path}") - if os.path.exists(local_path): - try: - shutil.copy(local_path, zip_path) - logger.info(f"Copied local .mpta file from {local_path} to {zip_path}") - except Exception as e: - logger.error(f"Failed to copy local .mpta file from {local_path}: {str(e)}", exc_info=True) - return None - else: - logger.error(f"Local file {local_path} does not exist. Current directory: {os.getcwd()}") - # List all subdirectories of models directory to help debugging - if os.path.exists("models"): - logger.error(f"Content of models directory: {os.listdir('models')}") - for root, dirs, files in os.walk("models"): - logger.error(f"Directory {root} contains subdirs: {dirs} and files: {files}") - else: - logger.error("The models directory doesn't exist") - return None - else: - logger.error(f"HTTP download functionality has been moved. Use a local file path here. Received: {zip_source}") - return None - - try: - if not os.path.exists(zip_path): - logger.error(f"Zip file not found at expected location: {zip_path}") - return None - - logger.debug(f"Extracting .mpta file from {zip_path} to {target_dir}") - # Extract contents and track the directories created - extracted_dirs = [] - with zipfile.ZipFile(zip_path, "r") as zip_ref: - file_list = zip_ref.namelist() - logger.debug(f"Files in .mpta archive: {file_list}") - - # Extract and track the top-level directories - for file_path in file_list: - parts = file_path.split('/') - if len(parts) > 1: - top_dir = parts[0] - if top_dir and top_dir not in extracted_dirs: - extracted_dirs.append(top_dir) - - # Now extract the files - zip_ref.extractall(target_dir) - - logger.info(f"Successfully extracted .mpta file to {target_dir}") - logger.debug(f"Extracted directories: {extracted_dirs}") - - # Check what was actually created after extraction - actual_dirs = [d for d in os.listdir(target_dir) if os.path.isdir(os.path.join(target_dir, d))] - logger.debug(f"Actual directories created: {actual_dirs}") - except zipfile.BadZipFile as e: - logger.error(f"Bad zip file {zip_path}: {str(e)}", exc_info=True) - return None - except Exception as e: - logger.error(f"Failed to extract .mpta file {zip_path}: {str(e)}", exc_info=True) - return None - finally: - if os.path.exists(zip_path): - os.remove(zip_path) - logger.debug(f"Removed temporary zip file: {zip_path}") - - # Use the first extracted directory if it exists, otherwise use the expected name - pipeline_name = os.path.basename(zip_source) - pipeline_name = os.path.splitext(pipeline_name)[0] - - # Find the directory with pipeline.json - mpta_dir = None - # First try the expected directory name - expected_dir = os.path.join(target_dir, pipeline_name) - if os.path.exists(expected_dir) and os.path.exists(os.path.join(expected_dir, "pipeline.json")): - mpta_dir = expected_dir - logger.debug(f"Found pipeline.json in the expected directory: {mpta_dir}") - else: - # Look through all subdirectories for pipeline.json - for subdir in actual_dirs: - potential_dir = os.path.join(target_dir, subdir) - if os.path.exists(os.path.join(potential_dir, "pipeline.json")): - mpta_dir = potential_dir - logger.info(f"Found pipeline.json in directory: {mpta_dir} (different from expected: {expected_dir})") - break - - if not mpta_dir: - logger.error(f"Could not find pipeline.json in any extracted directory. Directory content: {os.listdir(target_dir)}") - return None - - pipeline_json_path = os.path.join(mpta_dir, "pipeline.json") - if not os.path.exists(pipeline_json_path): - logger.error(f"pipeline.json not found in the .mpta file. Files in directory: {os.listdir(mpta_dir)}") - return None - - try: - with open(pipeline_json_path, "r") as f: - pipeline_config = json.load(f) - logger.info(f"Successfully loaded pipeline configuration from {pipeline_json_path}") - logger.debug(f"Pipeline config: {json.dumps(pipeline_config, indent=2)}") - - # Establish Redis connection if configured - redis_client = None - if "redis" in pipeline_config: - redis_config = pipeline_config["redis"] - if not validate_redis_config(redis_config): - logger.error("Invalid Redis configuration, skipping Redis connection") - else: - try: - redis_client = redis.Redis( - host=redis_config["host"], - port=redis_config["port"], - password=redis_config.get("password"), - db=redis_config.get("db", 0), - decode_responses=True - ) - redis_client.ping() - logger.info(f"Successfully connected to Redis at {redis_config['host']}:{redis_config['port']}") - except redis.exceptions.ConnectionError as e: - logger.error(f"Failed to connect to Redis: {e}") - redis_client = None - - # Establish PostgreSQL connection if configured - db_manager = None - if "postgresql" in pipeline_config: - pg_config = pipeline_config["postgresql"] - if not validate_postgresql_config(pg_config): - logger.error("Invalid PostgreSQL configuration, skipping database connection") - else: - try: - db_manager = DatabaseManager(pg_config) - if db_manager.connect(): - logger.info(f"Successfully connected to PostgreSQL at {pg_config['host']}:{pg_config['port']}") - else: - logger.error("Failed to connect to PostgreSQL") - db_manager = None - except Exception as e: - logger.error(f"Error initializing PostgreSQL connection: {e}") - db_manager = None - - return load_pipeline_node(pipeline_config["pipeline"], mpta_dir, redis_client, db_manager) - except json.JSONDecodeError as e: - logger.error(f"Error parsing pipeline.json: {str(e)}", exc_info=True) - return None - except KeyError as e: - logger.error(f"Missing key in pipeline.json: {str(e)}", exc_info=True) - return None - except Exception as e: - logger.error(f"Error loading pipeline.json: {str(e)}", exc_info=True) - return None - -def execute_actions(node, frame, detection_result, regions_dict=None): - if not node["redis_client"] or not node["actions"]: - return - - # Create a dynamic context for this detection event - from datetime import datetime - action_context = { - **detection_result, - "timestamp_ms": int(time.time() * 1000), - "uuid": str(uuid.uuid4()), - "timestamp": datetime.now().strftime("%Y-%m-%dT%H-%M-%S"), - "filename": f"{uuid.uuid4()}.jpg" - } - - for action in node["actions"]: - try: - if action["type"] == "redis_save_image": - key = action["key"].format(**action_context) - - # Check if we need to crop a specific region - region_name = action.get("region") - image_to_save = frame - - if region_name and regions_dict: - cropped_image = crop_region_by_class(frame, regions_dict, region_name) - if cropped_image is not None: - image_to_save = cropped_image - logger.debug(f"Cropped region '{region_name}' for redis_save_image") - else: - logger.warning(f"Could not crop region '{region_name}', saving full frame instead") - - # Encode image with specified format and quality (default to JPEG) - img_format = action.get("format", "jpeg").lower() - quality = action.get("quality", 90) - - if img_format == "jpeg": - encode_params = [cv2.IMWRITE_JPEG_QUALITY, quality] - success, buffer = cv2.imencode('.jpg', image_to_save, encode_params) - elif img_format == "png": - success, buffer = cv2.imencode('.png', image_to_save) - else: - success, buffer = cv2.imencode('.jpg', image_to_save, [cv2.IMWRITE_JPEG_QUALITY, quality]) - - if not success: - logger.error(f"Failed to encode image for redis_save_image") - continue - - expire_seconds = action.get("expire_seconds") - if expire_seconds: - node["redis_client"].setex(key, expire_seconds, buffer.tobytes()) - logger.info(f"Saved image to Redis with key: {key} (expires in {expire_seconds}s)") - else: - node["redis_client"].set(key, buffer.tobytes()) - logger.info(f"Saved image to Redis with key: {key}") - action_context["image_key"] = key - elif action["type"] == "redis_publish": - channel = action["channel"] - try: - # Handle JSON message format by creating it programmatically - message_template = action["message"] - - # Check if the message is JSON-like (starts and ends with braces) - if message_template.strip().startswith('{') and message_template.strip().endswith('}'): - # Create JSON data programmatically to avoid formatting issues - json_data = {} - - # Add common fields - json_data["event"] = "frontal_detected" - json_data["display_id"] = action_context.get("display_id", "unknown") - json_data["session_id"] = action_context.get("session_id") - json_data["timestamp"] = action_context.get("timestamp", "") - json_data["image_key"] = action_context.get("image_key", "") - - # Convert to JSON string - message = json.dumps(json_data) - else: - # Use regular string formatting for non-JSON messages - message = message_template.format(**action_context) - - # Publish to Redis - if not node["redis_client"]: - logger.error("Redis client is None, cannot publish message") - continue - - # Test Redis connection - try: - node["redis_client"].ping() - logger.debug("Redis connection is active") - except Exception as ping_error: - logger.error(f"Redis connection test failed: {ping_error}") - continue - - result = node["redis_client"].publish(channel, message) - logger.info(f"Published message to Redis channel '{channel}': {message}") - logger.info(f"Redis publish result (subscribers count): {result}") - - # Additional debug info - if result == 0: - logger.warning(f"No subscribers listening to channel '{channel}'") - else: - logger.info(f"Message delivered to {result} subscriber(s)") - - except KeyError as e: - logger.error(f"Missing key in redis_publish message template: {e}") - logger.debug(f"Available context keys: {list(action_context.keys())}") - except Exception as e: - logger.error(f"Error in redis_publish action: {e}") - logger.debug(f"Message template: {action['message']}") - logger.debug(f"Available context keys: {list(action_context.keys())}") - import traceback - logger.debug(f"Full traceback: {traceback.format_exc()}") - except Exception as e: - logger.error(f"Error executing action {action['type']}: {e}") - -def execute_parallel_actions(node, frame, detection_result, regions_dict): - """Execute parallel actions after all required branches have completed.""" - if not node.get("parallelActions"): - return - - logger.debug("Executing parallel actions...") - branch_results = detection_result.get("branch_results", {}) - - for action in node["parallelActions"]: - try: - action_type = action.get("type") - logger.debug(f"Processing parallel action: {action_type}") - - if action_type == "postgresql_update_combined": - # Check if all required branches have completed - wait_for_branches = action.get("waitForBranches", []) - missing_branches = [branch for branch in wait_for_branches if branch not in branch_results] - - if missing_branches: - logger.warning(f"Cannot execute postgresql_update_combined: missing branch results for {missing_branches}") - continue - - logger.info(f"All required branches completed: {wait_for_branches}") - - # Execute the database update - execute_postgresql_update_combined(node, action, detection_result, branch_results) - else: - logger.warning(f"Unknown parallel action type: {action_type}") - - except Exception as e: - logger.error(f"Error executing parallel action {action.get('type', 'unknown')}: {e}") - import traceback - logger.debug(f"Full traceback: {traceback.format_exc()}") - -def execute_postgresql_update_combined(node, action, detection_result, branch_results): - """Execute a PostgreSQL update with combined branch results.""" - if not node.get("db_manager"): - logger.error("No database manager available for postgresql_update_combined action") - return - - try: - table = action["table"] - key_field = action["key_field"] - key_value_template = action["key_value"] - fields = action["fields"] - - # Create context for key value formatting - action_context = {**detection_result} - key_value = key_value_template.format(**action_context) - - logger.info(f"Executing database update: table={table}, {key_field}={key_value}") - - # Process field mappings - mapped_fields = {} - for db_field, value_template in fields.items(): - try: - mapped_value = resolve_field_mapping(value_template, branch_results, action_context) - if mapped_value is not None: - mapped_fields[db_field] = mapped_value - logger.debug(f"Mapped field: {db_field} = {mapped_value}") - else: - logger.warning(f"Could not resolve field mapping for {db_field}: {value_template}") - except Exception as e: - logger.error(f"Error mapping field {db_field} with template '{value_template}': {e}") - - if not mapped_fields: - logger.warning("No fields mapped successfully, skipping database update") - return - - # Execute the database update - success = node["db_manager"].execute_update(table, key_field, key_value, mapped_fields) - - if success: - logger.info(f"Successfully updated database: {table} with {len(mapped_fields)} fields") - else: - logger.error(f"Failed to update database: {table}") - - except KeyError as e: - logger.error(f"Missing required field in postgresql_update_combined action: {e}") - except Exception as e: - logger.error(f"Error in postgresql_update_combined action: {e}") - import traceback - logger.debug(f"Full traceback: {traceback.format_exc()}") - -def resolve_field_mapping(value_template, branch_results, action_context): - """Resolve field mapping templates like {car_brand_cls_v1.brand}.""" - try: - # Handle simple context variables first (non-branch references) - if not '.' in value_template: - return value_template.format(**action_context) - - # Handle branch result references like {model_id.field} - import re - branch_refs = re.findall(r'\{([^}]+\.[^}]+)\}', value_template) - - resolved_template = value_template - for ref in branch_refs: - try: - model_id, field_name = ref.split('.', 1) - - if model_id in branch_results: - branch_data = branch_results[model_id] - if field_name in branch_data: - field_value = branch_data[field_name] - resolved_template = resolved_template.replace(f'{{{ref}}}', str(field_value)) - logger.debug(f"Resolved {ref} to {field_value}") - else: - logger.warning(f"Field '{field_name}' not found in branch '{model_id}' results. Available fields: {list(branch_data.keys())}") - return None - else: - logger.warning(f"Branch '{model_id}' not found in results. Available branches: {list(branch_results.keys())}") - return None - except ValueError as e: - logger.error(f"Invalid branch reference format: {ref}") - return None - - # Format any remaining simple variables - try: - final_value = resolved_template.format(**action_context) - return final_value - except KeyError as e: - logger.warning(f"Could not resolve context variable in template: {e}") - return resolved_template - - except Exception as e: - logger.error(f"Error resolving field mapping '{value_template}': {e}") - return None - -def run_pipeline(frame, node: dict, return_bbox: bool=False, context=None): - """ - Enhanced pipeline that supports: - - Multi-class detection (detecting multiple classes simultaneously) - - Parallel branch processing - - Region-based actions and cropping - - Context passing for session/camera information - """ - try: - task = getattr(node["model"], "task", None) - - # ─── Classification stage ─────────────────────────────────── - if task == "classify": - results = node["model"].predict(frame, stream=False) - if not results: - return (None, None) if return_bbox else None - - r = results[0] - probs = r.probs - if probs is None: - return (None, None) if return_bbox else None - - top1_idx = int(probs.top1) - top1_conf = float(probs.top1conf) - class_name = node["model"].names[top1_idx] - - det = { - "class": class_name, - "confidence": top1_conf, - "id": None, - class_name: class_name # Add class name as key for backward compatibility - } - - # Add specific field mappings for database operations based on model type - model_id = node.get("modelId", "").lower() - if "brand" in model_id or "brand_cls" in model_id: - det["brand"] = class_name - elif "bodytype" in model_id or "body" in model_id: - det["body_type"] = class_name - elif "color" in model_id: - det["color"] = class_name - - execute_actions(node, frame, det) - return (det, None) if return_bbox else det - - # ─── Detection stage - Multi-class support ────────────────── - tk = node["triggerClassIndices"] - logger.debug(f"Running detection for node {node['modelId']} with trigger classes: {node.get('triggerClasses', [])} (indices: {tk})") - logger.debug(f"Node configuration: minConfidence={node['minConfidence']}, multiClass={node.get('multiClass', False)}") - - res = node["model"].track( - frame, - stream=False, - persist=True, - **({"classes": tk} if tk else {}) - )[0] - - # Collect all detections above confidence threshold - all_detections = [] - all_boxes = [] - regions_dict = {} - - logger.debug(f"Raw detection results from model: {len(res.boxes) if res.boxes is not None else 0} detections") - - for i, box in enumerate(res.boxes): - conf = float(box.cpu().conf[0]) - cid = int(box.cpu().cls[0]) - name = node["model"].names[cid] - - logger.debug(f"Detection {i}: class='{name}' (id={cid}), confidence={conf:.3f}, threshold={node['minConfidence']}") - - if conf < node["minConfidence"]: - logger.debug(f" -> REJECTED: confidence {conf:.3f} < threshold {node['minConfidence']}") - continue - - xy = box.cpu().xyxy[0] - x1, y1, x2, y2 = map(int, xy) - bbox = (x1, y1, x2, y2) - - detection = { - "class": name, - "confidence": conf, - "id": box.id.item() if hasattr(box, "id") else None, - "bbox": bbox - } - - all_detections.append(detection) - all_boxes.append(bbox) - - logger.debug(f" -> ACCEPTED: {name} with confidence {conf:.3f}, bbox={bbox}") - - # Store highest confidence detection for each class - if name not in regions_dict or conf > regions_dict[name]["confidence"]: - regions_dict[name] = { - "bbox": bbox, - "confidence": conf, - "detection": detection - } - logger.debug(f" -> Updated regions_dict['{name}'] with confidence {conf:.3f}") - - logger.info(f"Detection summary: {len(all_detections)} accepted detections from {len(res.boxes) if res.boxes is not None else 0} total") - logger.info(f"Detected classes: {list(regions_dict.keys())}") - - if not all_detections: - logger.warning("No detections above confidence threshold - returning null") - return (None, None) if return_bbox else None - - # ─── Multi-class validation ───────────────────────────────── - if node.get("multiClass", False) and node.get("expectedClasses"): - expected_classes = node["expectedClasses"] - detected_classes = list(regions_dict.keys()) - - logger.info(f"Multi-class validation: expected={expected_classes}, detected={detected_classes}") - - # Check if at least one expected class is detected (flexible mode) - matching_classes = [cls for cls in expected_classes if cls in detected_classes] - missing_classes = [cls for cls in expected_classes if cls not in detected_classes] - - logger.debug(f"Matching classes: {matching_classes}, Missing classes: {missing_classes}") - - if not matching_classes: - # No expected classes found at all - logger.warning(f"PIPELINE REJECTED: No expected classes detected. Expected: {expected_classes}, Detected: {detected_classes}") - return (None, None) if return_bbox else None - - if missing_classes: - logger.info(f"Partial multi-class detection: {matching_classes} found, {missing_classes} missing") - else: - logger.info(f"Complete multi-class detection success: {detected_classes}") - else: - logger.debug("No multi-class validation - proceeding with all detections") - - # ─── Execute actions with region information ──────────────── - detection_result = { - "detections": all_detections, - "regions": regions_dict, - **(context or {}) - } - - # ─── Create initial database record when Car+Frontal detected ──── - if node.get("db_manager") and node.get("multiClass", False): - # Only create database record if we have both Car and Frontal - has_car = "Car" in regions_dict - has_frontal = "Frontal" in regions_dict - - if has_car and has_frontal: - # Generate UUID session_id since client session is None for now - import uuid as uuid_lib - from datetime import datetime - generated_session_id = str(uuid_lib.uuid4()) - - # Insert initial detection record - display_id = detection_result.get("display_id", "unknown") - timestamp = datetime.now().strftime("%Y-%m-%dT%H-%M-%S") - - inserted_session_id = node["db_manager"].insert_initial_detection( - display_id=display_id, - captured_timestamp=timestamp, - session_id=generated_session_id - ) - - if inserted_session_id: - # Update detection_result with the generated session_id for actions and branches - detection_result["session_id"] = inserted_session_id - detection_result["timestamp"] = timestamp # Update with proper timestamp - logger.info(f"Created initial database record with session_id: {inserted_session_id}") - else: - logger.debug(f"Database record not created - missing required classes. Has Car: {has_car}, Has Frontal: {has_frontal}") - - execute_actions(node, frame, detection_result, regions_dict) - - # ─── Parallel branch processing ───────────────────────────── - if node["branches"]: - branch_results = {} - - # Filter branches that should be triggered - active_branches = [] - for br in node["branches"]: - trigger_classes = br.get("triggerClasses", []) - min_conf = br.get("minConfidence", 0) - - logger.debug(f"Evaluating branch {br['modelId']}: trigger_classes={trigger_classes}, min_conf={min_conf}") - - # Check if any detected class matches branch trigger - branch_triggered = False - for det_class in regions_dict: - det_confidence = regions_dict[det_class]["confidence"] - logger.debug(f" Checking detected class '{det_class}' (confidence={det_confidence:.3f}) against triggers {trigger_classes}") - - if (det_class in trigger_classes and det_confidence >= min_conf): - active_branches.append(br) - branch_triggered = True - logger.info(f"Branch {br['modelId']} activated by class '{det_class}' (conf={det_confidence:.3f} >= {min_conf})") - break - - if not branch_triggered: - logger.debug(f"Branch {br['modelId']} not triggered - no matching classes or insufficient confidence") - - if active_branches: - if node.get("parallel", False) or any(br.get("parallel", False) for br in active_branches): - # Run branches in parallel - with concurrent.futures.ThreadPoolExecutor(max_workers=len(active_branches)) as executor: - futures = {} - - for br in active_branches: - crop_class = br.get("cropClass", br.get("triggerClasses", [])[0] if br.get("triggerClasses") else None) - sub_frame = frame - - logger.info(f"Starting parallel branch: {br['modelId']}, crop_class: {crop_class}") - - if br.get("crop", False) and crop_class: - cropped = crop_region_by_class(frame, regions_dict, crop_class) - if cropped is not None: - sub_frame = cv2.resize(cropped, (224, 224)) - logger.debug(f"Successfully cropped {crop_class} region for {br['modelId']}") - else: - logger.warning(f"Failed to crop {crop_class} region for {br['modelId']}, skipping branch") - continue - - future = executor.submit(run_pipeline, sub_frame, br, True, context) - futures[future] = br - - # Collect results - for future in concurrent.futures.as_completed(futures): - br = futures[future] - try: - result, _ = future.result() - if result: - branch_results[br["modelId"]] = result - logger.info(f"Branch {br['modelId']} completed: {result}") - except Exception as e: - logger.error(f"Branch {br['modelId']} failed: {e}") - else: - # Run branches sequentially - for br in active_branches: - crop_class = br.get("cropClass", br.get("triggerClasses", [])[0] if br.get("triggerClasses") else None) - sub_frame = frame - - logger.info(f"Starting sequential branch: {br['modelId']}, crop_class: {crop_class}") - - if br.get("crop", False) and crop_class: - cropped = crop_region_by_class(frame, regions_dict, crop_class) - if cropped is not None: - sub_frame = cv2.resize(cropped, (224, 224)) - logger.debug(f"Successfully cropped {crop_class} region for {br['modelId']}") - else: - logger.warning(f"Failed to crop {crop_class} region for {br['modelId']}, skipping branch") - continue - - try: - result, _ = run_pipeline(sub_frame, br, True, context) - if result: - branch_results[br["modelId"]] = result - logger.info(f"Branch {br['modelId']} completed: {result}") - else: - logger.warning(f"Branch {br['modelId']} returned no result") - except Exception as e: - logger.error(f"Error in sequential branch {br['modelId']}: {e}") - import traceback - logger.debug(f"Branch error traceback: {traceback.format_exc()}") - - # Store branch results in detection_result for parallel actions - detection_result["branch_results"] = branch_results - - # ─── Execute Parallel Actions ─────────────────────────────── - if node.get("parallelActions") and "branch_results" in detection_result: - execute_parallel_actions(node, frame, detection_result, regions_dict) - - # ─── Return detection result ──────────────────────────────── - primary_detection = max(all_detections, key=lambda x: x["confidence"]) - primary_bbox = primary_detection["bbox"] - - # Add branch results to primary detection for compatibility - if "branch_results" in detection_result: - primary_detection["branch_results"] = detection_result["branch_results"] - - return (primary_detection, primary_bbox) if return_bbox else primary_detection - - except Exception as e: - logger.error(f"Error in node {node.get('modelId')}: {e}") - traceback.print_exc() - return (None, None) if return_bbox else None diff --git a/config.json b/config.json index 0d061f9..5b23cdf 100644 --- a/config.json +++ b/config.json @@ -1,9 +1,7 @@ { "poll_interval_ms": 100, - "max_streams": 20, + "max_streams": 999, "target_fps": 2, - "reconnect_interval_sec": 10, - "max_retries": -1, - "rtsp_buffer_size": 3, - "rtsp_tcp_transport": true + "reconnect_interval_sec": 5, + "max_retries": -1 } diff --git a/core/__init__.py b/core/__init__.py deleted file mode 100644 index e697cb2..0000000 --- a/core/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Core package for detector worker \ No newline at end of file diff --git a/core/communication/__init__.py b/core/communication/__init__.py deleted file mode 100644 index 73145a1..0000000 --- a/core/communication/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Communication module for WebSocket and HTTP handling \ No newline at end of file diff --git a/core/communication/messages.py b/core/communication/messages.py deleted file mode 100644 index 98cc9e5..0000000 --- a/core/communication/messages.py +++ /dev/null @@ -1,212 +0,0 @@ -""" -Message types, constants, and validation functions for WebSocket communication. -""" -import json -import logging -from typing import Dict, Any, Optional, Union -from .models import ( - IncomingMessage, OutgoingMessage, - SetSubscriptionListMessage, SetSessionIdMessage, SetProgressionStageMessage, - RequestStateMessage, PatchSessionResultMessage, - StateReportMessage, ImageDetectionMessage, PatchSessionMessage -) - -logger = logging.getLogger(__name__) - -# Message type constants -class MessageTypes: - """WebSocket message type constants.""" - - # Incoming from backend - SET_SUBSCRIPTION_LIST = "setSubscriptionList" - SET_SESSION_ID = "setSessionId" - SET_PROGRESSION_STAGE = "setProgressionStage" - REQUEST_STATE = "requestState" - PATCH_SESSION_RESULT = "patchSessionResult" - - # Outgoing to backend - STATE_REPORT = "stateReport" - IMAGE_DETECTION = "imageDetection" - PATCH_SESSION = "patchSession" - - -def parse_incoming_message(raw_message: str) -> Optional[IncomingMessage]: - """ - Parse incoming WebSocket message and validate against known types. - - Args: - raw_message: Raw JSON string from WebSocket - - Returns: - Parsed message object or None if invalid - """ - try: - data = json.loads(raw_message) - message_type = data.get("type") - - if not message_type: - logger.error("Message missing 'type' field") - return None - - # Route to appropriate message class - if message_type == MessageTypes.SET_SUBSCRIPTION_LIST: - return SetSubscriptionListMessage(**data) - elif message_type == MessageTypes.SET_SESSION_ID: - return SetSessionIdMessage(**data) - elif message_type == MessageTypes.SET_PROGRESSION_STAGE: - return SetProgressionStageMessage(**data) - elif message_type == MessageTypes.REQUEST_STATE: - return RequestStateMessage(**data) - elif message_type == MessageTypes.PATCH_SESSION_RESULT: - return PatchSessionResultMessage(**data) - else: - logger.warning(f"Unknown message type: {message_type}") - return None - - except json.JSONDecodeError as e: - logger.error(f"Failed to decode JSON message: {e}") - return None - except Exception as e: - logger.error(f"Failed to parse incoming message: {e}") - return None - - -def serialize_outgoing_message(message: OutgoingMessage) -> str: - """ - Serialize outgoing message to JSON string. - - Args: - message: Message object to serialize - - Returns: - JSON string representation - """ - try: - # For ImageDetectionMessage, we need to include None values for abandonment detection - from .models import ImageDetectionMessage - if isinstance(message, ImageDetectionMessage): - return message.model_dump_json(exclude_none=False) - else: - return message.model_dump_json(exclude_none=True) - except Exception as e: - logger.error(f"Failed to serialize outgoing message: {e}") - raise - - -def validate_subscription_identifier(identifier: str) -> bool: - """ - Validate subscription identifier format (displayId;cameraId). - - Args: - identifier: Subscription identifier to validate - - Returns: - True if valid format, False otherwise - """ - if not identifier or not isinstance(identifier, str): - return False - - parts = identifier.split(';') - if len(parts) != 2: - logger.error(f"Invalid subscription identifier format: {identifier}") - return False - - display_id, camera_id = parts - if not display_id or not camera_id: - logger.error(f"Empty display or camera ID in identifier: {identifier}") - return False - - return True - - -def extract_display_identifier(subscription_identifier: str) -> Optional[str]: - """ - Extract display identifier from subscription identifier. - - Args: - subscription_identifier: Full subscription identifier (displayId;cameraId) - - Returns: - Display identifier or None if invalid format - """ - if not validate_subscription_identifier(subscription_identifier): - return None - - return subscription_identifier.split(';')[0] - - -def create_state_report(cpu_usage: float, memory_usage: float, - gpu_usage: Optional[float] = None, - gpu_memory_usage: Optional[float] = None, - camera_connections: Optional[list] = None) -> StateReportMessage: - """ - Create a state report message with system metrics. - - Args: - cpu_usage: CPU usage percentage - memory_usage: Memory usage percentage - gpu_usage: GPU usage percentage (optional) - gpu_memory_usage: GPU memory usage in MB (optional) - camera_connections: List of active camera connections - - Returns: - StateReportMessage object - """ - return StateReportMessage( - cpuUsage=cpu_usage, - memoryUsage=memory_usage, - gpuUsage=gpu_usage, - gpuMemoryUsage=gpu_memory_usage, - cameraConnections=camera_connections or [] - ) - - -def create_image_detection(subscription_identifier: str, detection_data: Union[Dict[str, Any], None], - model_id: int, model_name: str) -> ImageDetectionMessage: - """ - Create an image detection message. - - Args: - subscription_identifier: Camera subscription identifier - detection_data: Detection results - Dict for data, {} for empty, None for abandonment - model_id: Model identifier - model_name: Model name - - Returns: - ImageDetectionMessage object - """ - from .models import DetectionData - from typing import Union - - # Handle three cases: - # 1. None = car abandonment (detection: null) - # 2. {} = empty detection (triggers session creation) - # 3. {...} = full detection data (updates session) - - data = DetectionData( - detection=detection_data, - modelId=model_id, - modelName=model_name - ) - - return ImageDetectionMessage( - subscriptionIdentifier=subscription_identifier, - data=data - ) - - -def create_patch_session(session_id: int, patch_data: Dict[str, Any]) -> PatchSessionMessage: - """ - Create a patch session message. - - Args: - session_id: Session ID to patch - patch_data: Partial session data to update - - Returns: - PatchSessionMessage object - """ - return PatchSessionMessage( - sessionId=session_id, - data=patch_data - ) \ No newline at end of file diff --git a/core/communication/models.py b/core/communication/models.py deleted file mode 100644 index 7214472..0000000 --- a/core/communication/models.py +++ /dev/null @@ -1,150 +0,0 @@ -""" -Message data structures for WebSocket communication. -Based on worker.md protocol specification. -""" -from typing import Dict, Any, List, Optional, Union, Literal -from pydantic import BaseModel, Field -from datetime import datetime - - -class SubscriptionObject(BaseModel): - """Individual camera subscription configuration.""" - subscriptionIdentifier: str = Field(..., description="Format: displayId;cameraId") - rtspUrl: Optional[str] = Field(None, description="RTSP stream URL") - snapshotUrl: Optional[str] = Field(None, description="HTTP snapshot URL") - snapshotInterval: Optional[int] = Field(None, description="Snapshot interval in milliseconds") - modelUrl: str = Field(..., description="Pre-signed URL to .mpta file") - modelId: int = Field(..., description="Unique model identifier") - modelName: str = Field(..., description="Human-readable model name") - cropX1: Optional[int] = Field(None, description="Crop region X1 coordinate") - cropY1: Optional[int] = Field(None, description="Crop region Y1 coordinate") - cropX2: Optional[int] = Field(None, description="Crop region X2 coordinate") - cropY2: Optional[int] = Field(None, description="Crop region Y2 coordinate") - - -class CameraConnection(BaseModel): - """Camera connection status for state reporting.""" - subscriptionIdentifier: str - modelId: int - modelName: str - online: bool - cropX1: Optional[int] = None - cropY1: Optional[int] = None - cropX2: Optional[int] = None - cropY2: Optional[int] = None - - -class DetectionData(BaseModel): - """ - Detection result data structure. - - Supports three cases: - 1. Empty detection: detection = {} (triggers session creation) - 2. Full detection: detection = {"carBrand": "Honda", ...} (updates session) - 3. Null detection: detection = None (car abandonment) - """ - model_config = { - "json_encoders": {type(None): lambda v: None}, - "arbitrary_types_allowed": True - } - - detection: Union[Dict[str, Any], None] = Field( - default_factory=dict, - description="Detection results: {} for empty, {...} for data, None/null for abandonment" - ) - modelId: int - modelName: str - - -# Incoming Messages from Backend to Worker - -class SetSubscriptionListMessage(BaseModel): - """Complete subscription list for declarative state management.""" - type: Literal["setSubscriptionList"] = "setSubscriptionList" - subscriptions: List[SubscriptionObject] - - -class SetSessionIdPayload(BaseModel): - """Session ID association payload.""" - displayIdentifier: str - sessionId: Optional[int] = None - - -class SetSessionIdMessage(BaseModel): - """Associate session ID with display.""" - type: Literal["setSessionId"] = "setSessionId" - payload: SetSessionIdPayload - - -class SetProgressionStagePayload(BaseModel): - """Progression stage payload.""" - displayIdentifier: str - progressionStage: Optional[str] = None - - -class SetProgressionStageMessage(BaseModel): - """Set progression stage for display.""" - type: Literal["setProgressionStage"] = "setProgressionStage" - payload: SetProgressionStagePayload - - -class RequestStateMessage(BaseModel): - """Request current worker state.""" - type: Literal["requestState"] = "requestState" - - -class PatchSessionResultPayload(BaseModel): - """Patch session result payload.""" - sessionId: int - success: bool - message: str - - -class PatchSessionResultMessage(BaseModel): - """Response to patch session request.""" - type: Literal["patchSessionResult"] = "patchSessionResult" - payload: PatchSessionResultPayload - - -# Outgoing Messages from Worker to Backend - -class StateReportMessage(BaseModel): - """Periodic heartbeat with system metrics.""" - type: Literal["stateReport"] = "stateReport" - cpuUsage: float - memoryUsage: float - gpuUsage: Optional[float] = None - gpuMemoryUsage: Optional[float] = None - cameraConnections: List[CameraConnection] - - -class ImageDetectionMessage(BaseModel): - """Detection event message.""" - type: Literal["imageDetection"] = "imageDetection" - subscriptionIdentifier: str - timestamp: str = Field(default_factory=lambda: datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ")) - data: DetectionData - - -class PatchSessionMessage(BaseModel): - """Request to modify session data.""" - type: Literal["patchSession"] = "patchSession" - sessionId: int - data: Dict[str, Any] = Field(..., description="Partial DisplayPersistentData structure") - - -# Union type for all incoming messages -IncomingMessage = Union[ - SetSubscriptionListMessage, - SetSessionIdMessage, - SetProgressionStageMessage, - RequestStateMessage, - PatchSessionResultMessage -] - -# Union type for all outgoing messages -OutgoingMessage = Union[ - StateReportMessage, - ImageDetectionMessage, - PatchSessionMessage -] \ No newline at end of file diff --git a/core/communication/state.py b/core/communication/state.py deleted file mode 100644 index 9016c07..0000000 --- a/core/communication/state.py +++ /dev/null @@ -1,234 +0,0 @@ -""" -Worker state management for system metrics and subscription tracking. -""" -import logging -import psutil -import threading -from typing import Dict, Set, Optional, List -from dataclasses import dataclass, field -from .models import CameraConnection, SubscriptionObject - -logger = logging.getLogger(__name__) - -# Try to import torch and pynvml for GPU monitoring -try: - import torch - TORCH_AVAILABLE = True -except ImportError: - TORCH_AVAILABLE = False - logger.warning("PyTorch not available, GPU metrics will not be collected") - -try: - import pynvml - PYNVML_AVAILABLE = True - pynvml.nvmlInit() - logger.info("NVIDIA ML Python (pynvml) initialized successfully") -except ImportError: - PYNVML_AVAILABLE = False - logger.debug("pynvml not available, falling back to PyTorch GPU monitoring") -except Exception as e: - PYNVML_AVAILABLE = False - logger.warning(f"Failed to initialize pynvml: {e}") - - -@dataclass -class WorkerState: - """Central state management for the detector worker.""" - - # Active subscriptions indexed by subscription identifier - subscriptions: Dict[str, SubscriptionObject] = field(default_factory=dict) - - # Session ID mapping: display_identifier -> session_id - session_ids: Dict[str, int] = field(default_factory=dict) - - # Progression stage mapping: display_identifier -> stage - progression_stages: Dict[str, str] = field(default_factory=dict) - - # Active camera connections for state reporting - camera_connections: List[CameraConnection] = field(default_factory=list) - - # Thread lock for state synchronization - _lock: threading.RLock = field(default_factory=threading.RLock) - - def set_subscriptions(self, new_subscriptions: List[SubscriptionObject]) -> None: - """ - Update active subscriptions with declarative list from backend. - - Args: - new_subscriptions: Complete list of desired subscriptions - """ - with self._lock: - # Convert to dict for easy lookup - new_sub_dict = {sub.subscriptionIdentifier: sub for sub in new_subscriptions} - - # Log changes for debugging - current_ids = set(self.subscriptions.keys()) - new_ids = set(new_sub_dict.keys()) - - added = new_ids - current_ids - removed = current_ids - new_ids - updated = current_ids & new_ids - - if added: - logger.info(f"[State Update] Adding subscriptions: {added}") - if removed: - logger.info(f"[State Update] Removing subscriptions: {removed}") - if updated: - logger.info(f"[State Update] Updating subscriptions: {updated}") - - # Replace entire subscription dict - self.subscriptions = new_sub_dict - - # Update camera connections for state reporting - self._update_camera_connections() - - def get_subscription(self, subscription_identifier: str) -> Optional[SubscriptionObject]: - """Get subscription by identifier.""" - with self._lock: - return self.subscriptions.get(subscription_identifier) - - def get_all_subscriptions(self) -> List[SubscriptionObject]: - """Get all active subscriptions.""" - with self._lock: - return list(self.subscriptions.values()) - - def set_session_id(self, display_identifier: str, session_id: Optional[int]) -> None: - """ - Set or clear session ID for a display. - - Args: - display_identifier: Display identifier - session_id: Session ID to set, or None to clear - """ - with self._lock: - if session_id is None: - self.session_ids.pop(display_identifier, None) - logger.info(f"[State Update] Cleared session ID for display {display_identifier}") - else: - self.session_ids[display_identifier] = session_id - logger.info(f"[State Update] Set session ID {session_id} for display {display_identifier}") - - def get_session_id(self, display_identifier: str) -> Optional[int]: - """Get session ID for display identifier.""" - with self._lock: - return self.session_ids.get(display_identifier) - - def get_session_id_for_subscription(self, subscription_identifier: str) -> Optional[int]: - """Get session ID for subscription by extracting display identifier.""" - from .messages import extract_display_identifier - - display_id = extract_display_identifier(subscription_identifier) - if display_id: - return self.get_session_id(display_id) - return None - - def set_progression_stage(self, display_identifier: str, stage: Optional[str]) -> None: - """ - Set or clear progression stage for a display. - - Args: - display_identifier: Display identifier - stage: Progression stage to set, or None to clear - """ - with self._lock: - if stage is None: - self.progression_stages.pop(display_identifier, None) - logger.info(f"[State Update] Cleared progression stage for display {display_identifier}") - else: - self.progression_stages[display_identifier] = stage - logger.info(f"[State Update] Set progression stage '{stage}' for display {display_identifier}") - - def get_progression_stage(self, display_identifier: str) -> Optional[str]: - """Get progression stage for display identifier.""" - with self._lock: - return self.progression_stages.get(display_identifier) - - def _update_camera_connections(self) -> None: - """Update camera connections list for state reporting.""" - connections = [] - - for sub in self.subscriptions.values(): - connection = CameraConnection( - subscriptionIdentifier=sub.subscriptionIdentifier, - modelId=sub.modelId, - modelName=sub.modelName, - online=True, # TODO: Add actual online status tracking - cropX1=sub.cropX1, - cropY1=sub.cropY1, - cropX2=sub.cropX2, - cropY2=sub.cropY2 - ) - connections.append(connection) - - self.camera_connections = connections - - def get_camera_connections(self) -> List[CameraConnection]: - """Get current camera connections for state reporting.""" - with self._lock: - return self.camera_connections.copy() - - -class SystemMetrics: - """System metrics collection for state reporting.""" - - @staticmethod - def get_cpu_usage() -> float: - """Get current CPU usage percentage.""" - try: - return psutil.cpu_percent(interval=0.1) - except Exception as e: - logger.error(f"Failed to get CPU usage: {e}") - return 0.0 - - @staticmethod - def get_memory_usage() -> float: - """Get current memory usage percentage.""" - try: - return psutil.virtual_memory().percent - except Exception as e: - logger.error(f"Failed to get memory usage: {e}") - return 0.0 - - @staticmethod - def get_gpu_usage() -> Optional[float]: - """Get current GPU usage percentage.""" - try: - # Prefer pynvml for accurate GPU utilization - if PYNVML_AVAILABLE: - handle = pynvml.nvmlDeviceGetHandleByIndex(0) # First GPU - utilization = pynvml.nvmlDeviceGetUtilizationRates(handle) - return float(utilization.gpu) - - # Fallback to PyTorch memory-based estimation - elif TORCH_AVAILABLE and torch.cuda.is_available(): - if hasattr(torch.cuda, 'utilization'): - return torch.cuda.utilization() - else: - # Estimate based on memory usage - allocated = torch.cuda.memory_allocated() - reserved = torch.cuda.memory_reserved() - if reserved > 0: - return (allocated / reserved) * 100 - - return None - except Exception as e: - logger.error(f"Failed to get GPU usage: {e}") - return None - - @staticmethod - def get_gpu_memory_usage() -> Optional[float]: - """Get current GPU memory usage in MB.""" - if not TORCH_AVAILABLE: - return None - - try: - if torch.cuda.is_available(): - return torch.cuda.memory_reserved() / (1024 ** 2) # Convert to MB - return None - except Exception as e: - logger.error(f"Failed to get GPU memory usage: {e}") - return None - - -# Global worker state instance -worker_state = WorkerState() \ No newline at end of file diff --git a/core/communication/websocket.py b/core/communication/websocket.py deleted file mode 100644 index e53096a..0000000 --- a/core/communication/websocket.py +++ /dev/null @@ -1,677 +0,0 @@ -""" -WebSocket message handling and protocol implementation. -""" -import asyncio -import json -import logging -import os -import cv2 -from datetime import datetime, timezone, timedelta -from pathlib import Path -from typing import Optional -from fastapi import WebSocket, WebSocketDisconnect -from websockets.exceptions import ConnectionClosedError - -from .messages import ( - parse_incoming_message, serialize_outgoing_message, - MessageTypes, create_state_report -) -from .models import ( - SetSubscriptionListMessage, SetSessionIdMessage, SetProgressionStageMessage, - RequestStateMessage, PatchSessionResultMessage -) -from .state import worker_state, SystemMetrics -from ..models import ModelManager -from ..streaming.manager import shared_stream_manager -from ..tracking.integration import TrackingPipelineIntegration - -logger = logging.getLogger(__name__) - -# Constants -HEARTBEAT_INTERVAL = 2.0 # seconds -WORKER_TIMEOUT_MS = 10000 - -# Global model manager instance -model_manager = ModelManager() - - -class WebSocketHandler: - """ - Handles WebSocket connection lifecycle and message processing. - """ - - def __init__(self, websocket: WebSocket): - self.websocket = websocket - self.connected = False - self._heartbeat_task: Optional[asyncio.Task] = None - self._message_task: Optional[asyncio.Task] = None - self._heartbeat_count = 0 - self._last_processed_models: set = set() # Cache of last processed model IDs - - async def handle_connection(self) -> None: - """ - Main connection handler that manages the WebSocket lifecycle. - Based on the original architecture from archive/app.py - """ - client_info = f"{self.websocket.client.host}:{self.websocket.client.port}" if self.websocket.client else "unknown" - logger.info(f"Starting WebSocket handler for {client_info}") - - stream_task = None - try: - logger.info(f"Accepting WebSocket connection from {client_info}") - await self.websocket.accept() - self.connected = True - logger.info(f"WebSocket connection accepted and established for {client_info}") - - # Send immediate heartbeat to show connection is alive - await self._send_immediate_heartbeat() - - # Start background tasks (matching original architecture) - stream_task = asyncio.create_task(self._process_streams()) - heartbeat_task = asyncio.create_task(self._send_heartbeat()) - message_task = asyncio.create_task(self._handle_messages()) - - logger.info(f"WebSocket background tasks started for {client_info} (stream + heartbeat + message handler)") - - # Wait for heartbeat and message tasks (stream runs independently) - await asyncio.gather(heartbeat_task, message_task) - - except Exception as e: - logger.error(f"Error in WebSocket connection for {client_info}: {e}", exc_info=True) - finally: - logger.info(f"Cleaning up connection for {client_info}") - # Cancel stream task - if stream_task and not stream_task.done(): - stream_task.cancel() - try: - await stream_task - except asyncio.CancelledError: - logger.debug(f"Stream task cancelled for {client_info}") - await self._cleanup() - - async def _send_immediate_heartbeat(self) -> None: - """Send immediate heartbeat on connection to show we're alive.""" - try: - cpu_usage = SystemMetrics.get_cpu_usage() - memory_usage = SystemMetrics.get_memory_usage() - gpu_usage = SystemMetrics.get_gpu_usage() - gpu_memory_usage = SystemMetrics.get_gpu_memory_usage() - camera_connections = worker_state.get_camera_connections() - - state_report = create_state_report( - cpu_usage=cpu_usage, - memory_usage=memory_usage, - gpu_usage=gpu_usage, - gpu_memory_usage=gpu_memory_usage, - camera_connections=camera_connections - ) - - await self._send_message(state_report) - logger.info(f"[TX → Backend] Initial stateReport: CPU {cpu_usage:.1f}%, Memory {memory_usage:.1f}%, " - f"GPU {gpu_usage or 'N/A'}, {len(camera_connections)} cameras") - - except Exception as e: - logger.error(f"Error sending immediate heartbeat: {e}") - - async def _send_heartbeat(self) -> None: - """Send periodic state reports as heartbeat.""" - while self.connected: - try: - # Collect system metrics - cpu_usage = SystemMetrics.get_cpu_usage() - memory_usage = SystemMetrics.get_memory_usage() - gpu_usage = SystemMetrics.get_gpu_usage() - gpu_memory_usage = SystemMetrics.get_gpu_memory_usage() - camera_connections = worker_state.get_camera_connections() - - # Create and send state report - state_report = create_state_report( - cpu_usage=cpu_usage, - memory_usage=memory_usage, - gpu_usage=gpu_usage, - gpu_memory_usage=gpu_memory_usage, - camera_connections=camera_connections - ) - - await self._send_message(state_report) - - # Only log full details every 10th heartbeat, otherwise just show a dot - self._heartbeat_count += 1 - if self._heartbeat_count % 10 == 0: - logger.info(f"[TX → Backend] Heartbeat #{self._heartbeat_count}: CPU {cpu_usage:.1f}%, Memory {memory_usage:.1f}%, " - f"GPU {gpu_usage or 'N/A'}, {len(camera_connections)} cameras") - else: - print(".", end="", flush=True) # Just show a dot to indicate heartbeat activity - - await asyncio.sleep(HEARTBEAT_INTERVAL) - - except Exception as e: - logger.error(f"Error sending heartbeat: {e}") - break - - async def _handle_messages(self) -> None: - """Handle incoming WebSocket messages.""" - while self.connected: - try: - raw_message = await self.websocket.receive_text() - logger.info(f"[RX ← Backend] {raw_message}") - - # Parse incoming message - message = parse_incoming_message(raw_message) - if not message: - logger.warning("Failed to parse incoming message") - continue - - # Route message to appropriate handler - await self._route_message(message) - - except (WebSocketDisconnect, ConnectionClosedError) as e: - logger.warning(f"WebSocket disconnected: {e}") - break - except json.JSONDecodeError: - logger.error("Received invalid JSON message") - except Exception as e: - logger.error(f"Error handling message: {e}") - break - - async def _route_message(self, message) -> None: - """Route parsed message to appropriate handler.""" - message_type = message.type - - try: - if message_type == MessageTypes.SET_SUBSCRIPTION_LIST: - await self._handle_set_subscription_list(message) - elif message_type == MessageTypes.SET_SESSION_ID: - await self._handle_set_session_id(message) - elif message_type == MessageTypes.SET_PROGRESSION_STAGE: - await self._handle_set_progression_stage(message) - elif message_type == MessageTypes.REQUEST_STATE: - await self._handle_request_state(message) - elif message_type == MessageTypes.PATCH_SESSION_RESULT: - await self._handle_patch_session_result(message) - else: - logger.warning(f"Unknown message type: {message_type}") - - except Exception as e: - logger.error(f"Error handling {message_type} message: {e}") - - async def _handle_set_subscription_list(self, message: SetSubscriptionListMessage) -> None: - """Handle setSubscriptionList message for declarative subscription management.""" - logger.info(f"[RX Processing] setSubscriptionList with {len(message.subscriptions)} subscriptions") - - # Update worker state with new subscriptions - worker_state.set_subscriptions(message.subscriptions) - - # Phase 2: Download and manage models - await self._ensure_models(message.subscriptions) - - # Phase 3 & 4: Integrate with streaming management and tracking - await self._update_stream_subscriptions(message.subscriptions) - - logger.info("Subscription list updated successfully") - - async def _ensure_models(self, subscriptions) -> None: - """Ensure all required models are downloaded and available.""" - # Extract unique model requirements - unique_models = {} - for subscription in subscriptions: - model_id = subscription.modelId - if model_id not in unique_models: - unique_models[model_id] = { - 'model_url': subscription.modelUrl, - 'model_name': subscription.modelName - } - - # Check if model set has changed to avoid redundant processing - current_model_ids = set(unique_models.keys()) - if current_model_ids == self._last_processed_models: - logger.debug(f"[Model Management] Model set unchanged {list(current_model_ids)}, skipping checks") - return - - logger.info(f"[Model Management] Processing {len(unique_models)} unique models: {list(unique_models.keys())}") - self._last_processed_models = current_model_ids - - # Check and download models concurrently - download_tasks = [] - for model_id, model_info in unique_models.items(): - task = asyncio.create_task( - self._ensure_single_model(model_id, model_info['model_url'], model_info['model_name']) - ) - download_tasks.append(task) - - # Wait for all downloads to complete - if download_tasks: - results = await asyncio.gather(*download_tasks, return_exceptions=True) - - # Log results - success_count = 0 - for i, result in enumerate(results): - model_id = list(unique_models.keys())[i] - if isinstance(result, Exception): - logger.error(f"[Model Management] Failed to ensure model {model_id}: {result}") - elif result: - success_count += 1 - logger.info(f"[Model Management] Model {model_id} ready for use") - else: - logger.error(f"[Model Management] Failed to ensure model {model_id}") - - logger.info(f"[Model Management] Successfully ensured {success_count}/{len(unique_models)} models") - - async def _update_stream_subscriptions(self, subscriptions) -> None: - """Update streaming subscriptions with tracking integration.""" - try: - # Convert subscriptions to the format expected by StreamManager - subscription_payloads = [] - for subscription in subscriptions: - payload = { - 'subscriptionIdentifier': subscription.subscriptionIdentifier, - 'rtspUrl': subscription.rtspUrl, - 'snapshotUrl': subscription.snapshotUrl, - 'snapshotInterval': subscription.snapshotInterval, - 'modelId': subscription.modelId, - 'modelUrl': subscription.modelUrl, - 'modelName': subscription.modelName - } - # Add crop coordinates if present - if hasattr(subscription, 'cropX1'): - payload.update({ - 'cropX1': subscription.cropX1, - 'cropY1': subscription.cropY1, - 'cropX2': subscription.cropX2, - 'cropY2': subscription.cropY2 - }) - subscription_payloads.append(payload) - - # Reconcile subscriptions with StreamManager - logger.info("[Streaming] Reconciling stream subscriptions with tracking") - reconcile_result = await self._reconcile_subscriptions_with_tracking(subscription_payloads) - - logger.info(f"[Streaming] Subscription reconciliation complete: " - f"added={reconcile_result.get('added', 0)}, " - f"removed={reconcile_result.get('removed', 0)}, " - f"failed={reconcile_result.get('failed', 0)}") - - except Exception as e: - logger.error(f"Error updating stream subscriptions: {e}", exc_info=True) - - async def _reconcile_subscriptions_with_tracking(self, target_subscriptions) -> dict: - """Reconcile subscriptions with tracking integration.""" - try: - # Create separate tracking integrations for each subscription (camera isolation) - tracking_integrations = {} - - for subscription_payload in target_subscriptions: - subscription_id = subscription_payload['subscriptionIdentifier'] - model_id = subscription_payload['modelId'] - - # Create separate tracking integration per subscription for camera isolation - # Get pipeline configuration for this model - pipeline_parser = model_manager.get_pipeline_config(model_id) - if pipeline_parser: - # Create tracking integration with message sender (separate instance per camera) - tracking_integration = TrackingPipelineIntegration( - pipeline_parser, model_manager, model_id, self._send_message - ) - - # Initialize tracking model - success = await tracking_integration.initialize_tracking_model() - if success: - tracking_integrations[subscription_id] = tracking_integration - logger.info(f"[Tracking] Created isolated tracking integration for subscription {subscription_id} (model {model_id})") - else: - logger.warning(f"[Tracking] Failed to initialize tracking for subscription {subscription_id} (model {model_id})") - else: - logger.warning(f"[Tracking] No pipeline config found for model {model_id} in subscription {subscription_id}") - - # Now reconcile with StreamManager, adding tracking integrations - current_subscription_ids = set() - for subscription_info in shared_stream_manager.get_all_subscriptions(): - current_subscription_ids.add(subscription_info.subscription_id) - - target_subscription_ids = {sub['subscriptionIdentifier'] for sub in target_subscriptions} - - # Find subscriptions to remove and add - to_remove = current_subscription_ids - target_subscription_ids - to_add = target_subscription_ids - current_subscription_ids - - # Remove old subscriptions - removed_count = 0 - for subscription_id in to_remove: - if shared_stream_manager.remove_subscription(subscription_id): - removed_count += 1 - logger.info(f"[Streaming] Removed subscription {subscription_id}") - - # Add new subscriptions with tracking - added_count = 0 - failed_count = 0 - for subscription_payload in target_subscriptions: - subscription_id = subscription_payload['subscriptionIdentifier'] - if subscription_id in to_add: - success = await self._add_subscription_with_tracking( - subscription_payload, tracking_integrations - ) - if success: - added_count += 1 - logger.info(f"[Streaming] Added subscription {subscription_id} with tracking") - else: - failed_count += 1 - logger.error(f"[Streaming] Failed to add subscription {subscription_id}") - - return { - 'removed': removed_count, - 'added': added_count, - 'failed': failed_count, - 'total_active': len(shared_stream_manager.get_all_subscriptions()) - } - - except Exception as e: - logger.error(f"Error in subscription reconciliation with tracking: {e}", exc_info=True) - return {'removed': 0, 'added': 0, 'failed': 0, 'total_active': 0} - - async def _add_subscription_with_tracking(self, payload, tracking_integrations) -> bool: - """Add a subscription with tracking integration.""" - try: - from ..streaming.manager import StreamConfig - - subscription_id = payload['subscriptionIdentifier'] - camera_id = subscription_id.split(';')[-1] - model_id = payload['modelId'] - - logger.info(f"[SUBSCRIPTION_MAPPING] subscription_id='{subscription_id}' → camera_id='{camera_id}'") - - # Get tracking integration for this subscription (camera-isolated) - tracking_integration = tracking_integrations.get(subscription_id) - - # Extract crop coordinates if present - crop_coords = None - if all(key in payload for key in ['cropX1', 'cropY1', 'cropX2', 'cropY2']): - crop_coords = ( - payload['cropX1'], - payload['cropY1'], - payload['cropX2'], - payload['cropY2'] - ) - - # Create stream configuration - stream_config = StreamConfig( - camera_id=camera_id, - rtsp_url=payload.get('rtspUrl'), - snapshot_url=payload.get('snapshotUrl'), - snapshot_interval=payload.get('snapshotInterval', 5000), - max_retries=3, - ) - - # Add subscription to StreamManager with tracking - success = shared_stream_manager.add_subscription( - subscription_id=subscription_id, - stream_config=stream_config, - crop_coords=crop_coords, - model_id=model_id, - model_url=payload.get('modelUrl'), - tracking_integration=tracking_integration - ) - - if success and tracking_integration: - logger.info(f"[Tracking] Subscription {subscription_id} configured with isolated tracking for model {model_id}") - - return success - - except Exception as e: - logger.error(f"Error adding subscription with tracking: {e}", exc_info=True) - return False - - async def _ensure_single_model(self, model_id: int, model_url: str, model_name: str) -> bool: - """Ensure a single model is downloaded and available.""" - try: - # Check if model is already available - if model_manager.is_model_downloaded(model_id): - logger.info(f"[Model Management] Model {model_id} ({model_name}) already available") - return True - - # Download and extract model in a thread pool to avoid blocking the event loop - logger.info(f"[Model Management] Downloading model {model_id} ({model_name}) from {model_url}") - - # Use asyncio.to_thread for CPU-bound operations (Python 3.9+) - # For compatibility, we'll use run_in_executor - loop = asyncio.get_event_loop() - model_path = await loop.run_in_executor( - None, - model_manager.ensure_model, - model_id, - model_url, - model_name - ) - - if model_path: - logger.info(f"[Model Management] Successfully prepared model {model_id} at {model_path}") - return True - else: - logger.error(f"[Model Management] Failed to prepare model {model_id}") - return False - - except Exception as e: - logger.error(f"[Model Management] Exception ensuring model {model_id}: {str(e)}", exc_info=True) - return False - - async def _save_snapshot(self, display_identifier: str, session_id: int) -> None: - """ - Save snapshot image to images folder after receiving sessionId. - - Args: - display_identifier: Display identifier to match with subscriptionIdentifier - session_id: Session ID to include in filename - """ - try: - # Find subscription that matches the displayIdentifier - matching_subscription = None - for subscription in worker_state.get_all_subscriptions(): - # Extract display ID from subscriptionIdentifier (format: displayId;cameraId) - from .messages import extract_display_identifier - sub_display_id = extract_display_identifier(subscription.subscriptionIdentifier) - if sub_display_id == display_identifier: - matching_subscription = subscription - break - - if not matching_subscription: - logger.error(f"[Snapshot Save] No subscription found for display {display_identifier}") - return - - if not matching_subscription.snapshotUrl: - logger.error(f"[Snapshot Save] No snapshotUrl found for display {display_identifier}") - return - - # Ensure images directory exists (relative path for Docker bind mount) - images_dir = Path("images") - images_dir.mkdir(exist_ok=True) - - # Generate filename with timestamp and session ID - timestamp = datetime.now(tz=timezone(timedelta(hours=7))).strftime("%Y%m%d_%H%M%S") - filename = f"{session_id}_{display_identifier}_{timestamp}.jpg" - filepath = images_dir / filename - - # Use existing HTTPSnapshotReader to fetch snapshot - logger.info(f"[Snapshot Save] Fetching snapshot from {matching_subscription.snapshotUrl}") - - # Run snapshot fetch in thread pool to avoid blocking async loop - loop = asyncio.get_event_loop() - frame = await loop.run_in_executor(None, self._fetch_snapshot_sync, matching_subscription.snapshotUrl) - - if frame is not None: - # Save the image using OpenCV - success = cv2.imwrite(str(filepath), frame) - if success: - logger.info(f"[Snapshot Save] Successfully saved snapshot to {filepath}") - else: - logger.error(f"[Snapshot Save] Failed to save image file {filepath}") - else: - logger.error(f"[Snapshot Save] Failed to fetch snapshot from {matching_subscription.snapshotUrl}") - - except Exception as e: - logger.error(f"[Snapshot Save] Error saving snapshot for display {display_identifier}: {e}", exc_info=True) - - def _fetch_snapshot_sync(self, snapshot_url: str): - """ - Synchronous snapshot fetching using existing HTTPSnapshotReader infrastructure. - - Args: - snapshot_url: URL to fetch snapshot from - - Returns: - np.ndarray or None: Fetched frame or None on error - """ - try: - from ..streaming.readers import HTTPSnapshotReader - - # Create temporary snapshot reader for single fetch - snapshot_reader = HTTPSnapshotReader( - camera_id="temp_snapshot", - snapshot_url=snapshot_url, - interval_ms=5000 # Not used for single fetch - ) - - # Use existing fetch_single_snapshot method - return snapshot_reader.fetch_single_snapshot() - - except Exception as e: - logger.error(f"Error in sync snapshot fetch: {e}") - return None - - async def _handle_set_session_id(self, message: SetSessionIdMessage) -> None: - """Handle setSessionId message.""" - display_identifier = message.payload.displayIdentifier - session_id = str(message.payload.sessionId) if message.payload.sessionId is not None else None - - logger.info(f"[RX Processing] setSessionId for display {display_identifier}: {session_id}") - - # Update worker state - worker_state.set_session_id(display_identifier, session_id) - - # Update tracking integrations with session ID - shared_stream_manager.set_session_id(display_identifier, session_id) - - async def _handle_set_progression_stage(self, message: SetProgressionStageMessage) -> None: - """Handle setProgressionStage message.""" - display_identifier = message.payload.displayIdentifier - stage = message.payload.progressionStage - - logger.info(f"[RX Processing] setProgressionStage for display {display_identifier}: {stage}") - - # Update worker state - worker_state.set_progression_stage(display_identifier, stage) - - # Update tracking integration for car abandonment detection - session_id = worker_state.get_session_id(display_identifier) - if session_id: - shared_stream_manager.set_progression_stage(session_id, stage) - - # Save snapshot image when progression stage is car_fueling - if stage == 'car_fueling' and session_id: - await self._save_snapshot(display_identifier, session_id) - - # If stage indicates session is cleared/finished, clear from tracking - if stage in ['finished', 'cleared', 'idle']: - # Get session ID for this display and clear it - if session_id: - shared_stream_manager.clear_session_id(session_id) - logger.info(f"[Tracking] Cleared session {session_id} due to progression stage: {stage}") - - async def _handle_request_state(self, message: RequestStateMessage) -> None: - """Handle requestState message by sending immediate state report.""" - logger.debug("[RX Processing] requestState - sending immediate state report") - - # Collect metrics and send state report - cpu_usage = SystemMetrics.get_cpu_usage() - memory_usage = SystemMetrics.get_memory_usage() - gpu_usage = SystemMetrics.get_gpu_usage() - gpu_memory_usage = SystemMetrics.get_gpu_memory_usage() - camera_connections = worker_state.get_camera_connections() - - state_report = create_state_report( - cpu_usage=cpu_usage, - memory_usage=memory_usage, - gpu_usage=gpu_usage, - gpu_memory_usage=gpu_memory_usage, - camera_connections=camera_connections - ) - - await self._send_message(state_report) - - async def _handle_patch_session_result(self, message: PatchSessionResultMessage) -> None: - """Handle patchSessionResult message.""" - payload = message.payload - logger.info(f"[RX Processing] patchSessionResult for session {payload.sessionId}: " - f"success={payload.success}, message='{payload.message}'") - - # TODO: Handle patch session result if needed - # For now, just log the response - - async def _send_message(self, message) -> None: - """Send message to backend via WebSocket.""" - if not self.connected: - logger.warning("Cannot send message: WebSocket not connected") - return - - try: - json_message = serialize_outgoing_message(message) - await self.websocket.send_text(json_message) - # Log non-heartbeat messages only (heartbeats are logged in their respective functions) - if not (hasattr(message, 'type') and message.type == 'stateReport'): - logger.info(f"[TX → Backend] {json_message}") - except Exception as e: - logger.error(f"Failed to send WebSocket message: {e}") - raise - - async def _process_streams(self) -> None: - """ - Stream processing task that handles frame processing and detection. - This is a placeholder for Phase 2 - currently just logs that it's running. - """ - logger.info("Stream processing task started") - try: - while self.connected: - # Get current subscriptions - subscriptions = worker_state.get_all_subscriptions() - - # TODO: Phase 2 - Add actual frame processing logic here - # This will include: - # - Frame reading from RTSP/HTTP streams - # - Model inference using loaded pipelines - # - Detection result sending via WebSocket - - # Sleep to prevent excessive CPU usage (similar to old poll_interval) - await asyncio.sleep(0.1) # 100ms polling interval - - except asyncio.CancelledError: - logger.info("Stream processing task cancelled") - except Exception as e: - logger.error(f"Error in stream processing: {e}", exc_info=True) - - async def _cleanup(self) -> None: - """Clean up resources when connection closes.""" - logger.info("Cleaning up WebSocket connection") - self.connected = False - - # Cancel background tasks - if self._heartbeat_task and not self._heartbeat_task.done(): - self._heartbeat_task.cancel() - if self._message_task and not self._message_task.done(): - self._message_task.cancel() - - # Clear worker state - worker_state.set_subscriptions([]) - worker_state.session_ids.clear() - worker_state.progression_stages.clear() - - logger.info("WebSocket connection cleanup completed") - - -# Factory function for FastAPI integration -async def websocket_endpoint(websocket: WebSocket) -> None: - """ - FastAPI WebSocket endpoint handler. - - Args: - websocket: FastAPI WebSocket connection - """ - handler = WebSocketHandler(websocket) - await handler.handle_connection() \ No newline at end of file diff --git a/core/detection/__init__.py b/core/detection/__init__.py deleted file mode 100644 index 2bcb75c..0000000 --- a/core/detection/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -""" -Detection module for the Python Detector Worker. - -This module provides the main detection pipeline orchestration and parallel branch processing -for advanced computer vision detection systems. -""" -from .pipeline import DetectionPipeline -from .branches import BranchProcessor - -__all__ = ['DetectionPipeline', 'BranchProcessor'] \ No newline at end of file diff --git a/core/detection/branches.py b/core/detection/branches.py deleted file mode 100644 index 247c5f8..0000000 --- a/core/detection/branches.py +++ /dev/null @@ -1,791 +0,0 @@ -""" -Parallel Branch Processing Module. -Handles concurrent execution of classification branches and result synchronization. -""" -import logging -import asyncio -import time -from typing import Dict, List, Optional, Any, Tuple -from concurrent.futures import ThreadPoolExecutor, as_completed -import numpy as np -import cv2 - -from ..models.inference import YOLOWrapper - -logger = logging.getLogger(__name__) - - -class BranchProcessor: - """ - Handles parallel processing of classification branches. - Manages branch synchronization and result collection. - """ - - def __init__(self, model_manager: Any, model_id: int): - """ - Initialize branch processor. - - Args: - model_manager: Model manager for loading models - model_id: The model ID to use for loading models - """ - self.model_manager = model_manager - self.model_id = model_id - - # Branch models cache - self.branch_models: Dict[str, YOLOWrapper] = {} - - # Thread pool for parallel execution - self.executor = ThreadPoolExecutor(max_workers=4) - - # Storage managers (set during initialization) - self.redis_manager = None - self.db_manager = None - - # Statistics - self.stats = { - 'branches_processed': 0, - 'parallel_executions': 0, - 'total_processing_time': 0.0, - 'models_loaded': 0 - } - - logger.info("BranchProcessor initialized") - - async def initialize(self, pipeline_config: Any, redis_manager: Any, db_manager: Any) -> bool: - """ - Initialize branch processor with pipeline configuration. - - Args: - pipeline_config: Pipeline configuration object - redis_manager: Redis manager instance - db_manager: Database manager instance - - Returns: - True if successful, False otherwise - """ - try: - self.redis_manager = redis_manager - self.db_manager = db_manager - - # Pre-load branch models if they exist - branches = getattr(pipeline_config, 'branches', []) - if branches: - await self._preload_branch_models(branches) - - logger.info(f"BranchProcessor initialized with {len(self.branch_models)} models") - return True - - except Exception as e: - logger.error(f"Error initializing branch processor: {e}", exc_info=True) - return False - - async def _preload_branch_models(self, branches: List[Any]) -> None: - """ - Pre-load all branch models for faster execution. - - Args: - branches: List of branch configurations - """ - for branch in branches: - try: - await self._load_branch_model(branch) - - # Recursively load nested branches - nested_branches = getattr(branch, 'branches', []) - if nested_branches: - await self._preload_branch_models(nested_branches) - - except Exception as e: - logger.error(f"Error preloading branch model {getattr(branch, 'model_id', 'unknown')}: {e}") - - async def _load_branch_model(self, branch_config: Any) -> Optional[YOLOWrapper]: - """ - Load a branch model if not already loaded. - - Args: - branch_config: Branch configuration object - - Returns: - Loaded YOLO model wrapper or None - """ - try: - model_id = getattr(branch_config, 'model_id', None) - model_file = getattr(branch_config, 'model_file', None) - - if not model_id or not model_file: - logger.warning(f"Invalid branch config: model_id={model_id}, model_file={model_file}") - return None - - # Check if model is already loaded - if model_id in self.branch_models: - logger.debug(f"Branch model {model_id} already loaded") - return self.branch_models[model_id] - - # Load model - logger.info(f"Loading branch model: {model_id} ({model_file})") - - # Load model using the proper model ID - model = self.model_manager.get_yolo_model(self.model_id, model_file) - - if model: - self.branch_models[model_id] = model - self.stats['models_loaded'] += 1 - logger.info(f"Branch model {model_id} loaded successfully") - return model - else: - logger.error(f"Failed to load branch model {model_id}") - return None - - except Exception as e: - logger.error(f"Error loading branch model {getattr(branch_config, 'model_id', 'unknown')}: {e}") - return None - - async def execute_branches(self, - frame: np.ndarray, - branches: List[Any], - detected_regions: Dict[str, Any], - detection_context: Dict[str, Any]) -> Dict[str, Any]: - """ - Execute all branches in parallel and collect results. - - Args: - frame: Input frame - branches: List of branch configurations - detected_regions: Dictionary of detected regions from main detection - detection_context: Detection context data - - Returns: - Dictionary with branch execution results - """ - start_time = time.time() - branch_results = {} - - try: - # Separate parallel and sequential branches - parallel_branches = [] - sequential_branches = [] - - for branch in branches: - if getattr(branch, 'parallel', False): - parallel_branches.append(branch) - else: - sequential_branches.append(branch) - - # Execute parallel branches concurrently - if parallel_branches: - logger.info(f"Executing {len(parallel_branches)} branches in parallel") - parallel_results = await self._execute_parallel_branches( - frame, parallel_branches, detected_regions, detection_context - ) - branch_results.update(parallel_results) - self.stats['parallel_executions'] += 1 - - # Execute sequential branches one by one - if sequential_branches: - logger.info(f"Executing {len(sequential_branches)} branches sequentially") - sequential_results = await self._execute_sequential_branches( - frame, sequential_branches, detected_regions, detection_context - ) - branch_results.update(sequential_results) - - # Update statistics - self.stats['branches_processed'] += len(branches) - processing_time = time.time() - start_time - self.stats['total_processing_time'] += processing_time - - logger.info(f"Branch execution completed in {processing_time:.3f}s with {len(branch_results)} results") - - except Exception as e: - logger.error(f"Error in branch execution: {e}", exc_info=True) - - return branch_results - - async def _execute_parallel_branches(self, - frame: np.ndarray, - branches: List[Any], - detected_regions: Dict[str, Any], - detection_context: Dict[str, Any]) -> Dict[str, Any]: - """ - Execute branches in parallel using ThreadPoolExecutor. - - Args: - frame: Input frame - branches: List of parallel branch configurations - detected_regions: Dictionary of detected regions - detection_context: Detection context data - - Returns: - Dictionary with parallel branch results - """ - results = {} - - # Submit all branches for parallel execution - future_to_branch = {} - - for branch in branches: - branch_id = getattr(branch, 'model_id', 'unknown') - logger.info(f"[PARALLEL SUBMIT] {branch_id}: Submitting branch to thread pool") - - future = self.executor.submit( - self._execute_single_branch_sync, - frame, branch, detected_regions, detection_context - ) - future_to_branch[future] = branch - - # Collect results as they complete - for future in as_completed(future_to_branch): - branch = future_to_branch[future] - branch_id = getattr(branch, 'model_id', 'unknown') - - try: - result = future.result() - results[branch_id] = result - logger.info(f"[PARALLEL COMPLETE] {branch_id}: Branch completed successfully") - except Exception as e: - logger.error(f"Error in parallel branch {branch_id}: {e}") - results[branch_id] = { - 'status': 'error', - 'message': str(e), - 'processing_time': 0.0 - } - - # Flatten nested branch results to top level for database access - flattened_results = {} - for branch_id, branch_result in results.items(): - # Add the branch result itself - flattened_results[branch_id] = branch_result - - # If this branch has nested branches, add them to the top level too - if isinstance(branch_result, dict) and 'nested_branches' in branch_result: - nested_branches = branch_result['nested_branches'] - for nested_branch_id, nested_result in nested_branches.items(): - flattened_results[nested_branch_id] = nested_result - logger.info(f"[FLATTEN] Added nested branch {nested_branch_id} to top-level results") - - return flattened_results - - async def _execute_sequential_branches(self, - frame: np.ndarray, - branches: List[Any], - detected_regions: Dict[str, Any], - detection_context: Dict[str, Any]) -> Dict[str, Any]: - """ - Execute branches sequentially. - - Args: - frame: Input frame - branches: List of sequential branch configurations - detected_regions: Dictionary of detected regions - detection_context: Detection context data - - Returns: - Dictionary with sequential branch results - """ - results = {} - - for branch in branches: - branch_id = getattr(branch, 'model_id', 'unknown') - - try: - result = await asyncio.get_event_loop().run_in_executor( - self.executor, - self._execute_single_branch_sync, - frame, branch, detected_regions, detection_context - ) - results[branch_id] = result - logger.debug(f"Sequential branch {branch_id} completed successfully") - except Exception as e: - logger.error(f"Error in sequential branch {branch_id}: {e}") - results[branch_id] = { - 'status': 'error', - 'message': str(e), - 'processing_time': 0.0 - } - - # Flatten nested branch results to top level for database access - flattened_results = {} - for branch_id, branch_result in results.items(): - # Add the branch result itself - flattened_results[branch_id] = branch_result - - # If this branch has nested branches, add them to the top level too - if isinstance(branch_result, dict) and 'nested_branches' in branch_result: - nested_branches = branch_result['nested_branches'] - for nested_branch_id, nested_result in nested_branches.items(): - flattened_results[nested_branch_id] = nested_result - logger.info(f"[FLATTEN] Added nested branch {nested_branch_id} to top-level results") - - return flattened_results - - def _execute_single_branch_sync(self, - frame: np.ndarray, - branch_config: Any, - detected_regions: Dict[str, Any], - detection_context: Dict[str, Any]) -> Dict[str, Any]: - """ - Synchronous execution of a single branch (for ThreadPoolExecutor). - - Args: - frame: Input frame - branch_config: Branch configuration object - detected_regions: Dictionary of detected regions - detection_context: Detection context data - - Returns: - Dictionary with branch execution result - """ - start_time = time.time() - branch_id = getattr(branch_config, 'model_id', 'unknown') - - logger.info(f"[BRANCH START] {branch_id}: Starting branch execution") - logger.debug(f"[BRANCH CONFIG] {branch_id}: crop={getattr(branch_config, 'crop', False)}, " - f"trigger_classes={getattr(branch_config, 'trigger_classes', [])}, " - f"min_confidence={getattr(branch_config, 'min_confidence', 0.6)}") - - # Check if branch should execute based on triggerClasses (execution conditions) - trigger_classes = getattr(branch_config, 'trigger_classes', []) - logger.info(f"[DETECTED REGIONS] {branch_id}: Available parent detections: {list(detected_regions.keys())}") - for region_name, region_data in detected_regions.items(): - logger.debug(f"[REGION DATA] {branch_id}: '{region_name}' -> bbox={region_data.get('bbox')}, conf={region_data.get('confidence')}") - - if trigger_classes: - # Check if any parent detection matches our trigger classes - should_execute = False - for trigger_class in trigger_classes: - if trigger_class in detected_regions: - should_execute = True - logger.info(f"[TRIGGER CHECK] {branch_id}: Found '{trigger_class}' in parent detections - branch will execute") - break - - if not should_execute: - logger.warning(f"[TRIGGER CHECK] {branch_id}: None of trigger classes {trigger_classes} found in parent detections {list(detected_regions.keys())} - skipping branch") - return { - 'status': 'skipped', - 'branch_id': branch_id, - 'message': f'No trigger classes {trigger_classes} found in parent detections', - 'processing_time': time.time() - start_time - } - - result = { - 'status': 'success', - 'branch_id': branch_id, - 'result': {}, - 'processing_time': 0.0, - 'timestamp': time.time() - } - - try: - # Get or load branch model - if branch_id not in self.branch_models: - logger.warning(f"Branch model {branch_id} not preloaded, loading now...") - # This should be rare since models are preloaded - return { - 'status': 'error', - 'message': f'Branch model {branch_id} not available', - 'processing_time': time.time() - start_time - } - - model = self.branch_models[branch_id] - - # Get configuration values first - min_confidence = getattr(branch_config, 'min_confidence', 0.6) - - # Prepare input frame for this branch - input_frame = frame - - # Handle cropping if required - use biggest bbox that passes min_confidence - if getattr(branch_config, 'crop', False): - crop_classes = getattr(branch_config, 'crop_class', []) - if isinstance(crop_classes, str): - crop_classes = [crop_classes] - - # Find the biggest bbox that passes min_confidence threshold - best_region = None - best_class = None - best_area = 0.0 - - for crop_class in crop_classes: - if crop_class in detected_regions: - region = detected_regions[crop_class] - confidence = region.get('confidence', 0.0) - - # Only use detections above min_confidence - if confidence >= min_confidence: - bbox = region['bbox'] - area = (bbox[2] - bbox[0]) * (bbox[3] - bbox[1]) # width * height - - # Choose biggest bbox among valid detections - if area > best_area: - best_region = region - best_class = crop_class - best_area = area - - if best_region: - bbox = best_region['bbox'] - x1, y1, x2, y2 = [int(coord) for coord in bbox] - cropped = frame[y1:y2, x1:x2] - if cropped.size > 0: - input_frame = cropped - confidence = best_region.get('confidence', 0.0) - logger.info(f"[CROP SUCCESS] {branch_id}: cropped '{best_class}' region (conf={confidence:.3f}, area={int(best_area)}) -> shape={cropped.shape}") - else: - logger.warning(f"Branch {branch_id}: empty crop, using full frame") - else: - logger.warning(f"Branch {branch_id}: no valid crop regions found (min_conf={min_confidence})") - - logger.info(f"[INFERENCE START] {branch_id}: Running inference on {'cropped' if input_frame is not frame else 'full'} frame " - f"({input_frame.shape[1]}x{input_frame.shape[0]}) with confidence={min_confidence}") - - - # Use .predict() method for both detection and classification models - inference_start = time.time() - detection_results = model.model.predict(input_frame, conf=min_confidence, verbose=False) - inference_time = time.time() - inference_start - logger.info(f"[INFERENCE DONE] {branch_id}: Predict completed in {inference_time:.3f}s using .predict() method") - - # Initialize branch_detections outside the conditional - branch_detections = [] - - # Process results using clean, unified logic - if detection_results and len(detection_results) > 0: - result_obj = detection_results[0] - - # Handle detection models (have .boxes attribute) - if hasattr(result_obj, 'boxes') and result_obj.boxes is not None: - logger.info(f"[RAW DETECTIONS] {branch_id}: Found {len(result_obj.boxes)} raw detections") - - for i, box in enumerate(result_obj.boxes): - class_id = int(box.cls[0]) - confidence = float(box.conf[0]) - bbox = box.xyxy[0].cpu().numpy().tolist() # [x1, y1, x2, y2] - class_name = model.model.names[class_id] - - logger.debug(f"[RAW DETECTION {i+1}] {branch_id}: '{class_name}', conf={confidence:.3f}") - - # All detections are included - no filtering by trigger_classes here - branch_detections.append({ - 'class_name': class_name, - 'confidence': confidence, - 'bbox': bbox - }) - - # Handle classification models (have .probs attribute) - elif hasattr(result_obj, 'probs') and result_obj.probs is not None: - logger.info(f"[RAW CLASSIFICATION] {branch_id}: Processing classification results") - - probs = result_obj.probs - top_indices = probs.top5 # Get top 5 predictions - top_conf = probs.top5conf.cpu().numpy() - - for idx, conf in zip(top_indices, top_conf): - if conf >= min_confidence: - class_name = model.model.names[int(idx)] - logger.debug(f"[CLASSIFICATION RESULT {len(branch_detections)+1}] {branch_id}: '{class_name}', conf={conf:.3f}") - - # For classification, use full input frame dimensions as bbox - branch_detections.append({ - 'class_name': class_name, - 'confidence': float(conf), - 'bbox': [0, 0, input_frame.shape[1], input_frame.shape[0]] - }) - else: - logger.warning(f"[UNKNOWN MODEL] {branch_id}: Model results have no .boxes or .probs") - - result['result'] = { - 'detections': branch_detections, - 'detection_count': len(branch_detections) - } - - logger.info(f"[FINAL RESULTS] {branch_id}: {len(branch_detections)} detections processed") - - # Extract best result for classification models - if branch_detections: - best_detection = max(branch_detections, key=lambda x: x['confidence']) - logger.info(f"[BEST DETECTION] {branch_id}: '{best_detection['class_name']}' with confidence {best_detection['confidence']:.3f}") - - # Add classification-style results for database operations - if 'brand' in branch_id.lower(): - result['result']['brand'] = best_detection['class_name'] - elif 'body' in branch_id.lower() or 'bodytype' in branch_id.lower(): - result['result']['body_type'] = best_detection['class_name'] - elif 'front_rear' in branch_id.lower(): - result['result']['front_rear'] = best_detection['confidence'] - - logger.info(f"[CLASSIFICATION RESULT] {branch_id}: Extracted classification fields") - else: - logger.warning(f"[NO RESULTS] {branch_id}: No detections found") - - # Execute branch actions if this branch found valid detections - actions_executed = [] - branch_actions = getattr(branch_config, 'actions', []) - if branch_actions and branch_detections: - logger.info(f"[BRANCH ACTIONS] {branch_id}: Executing {len(branch_actions)} actions") - - # Create detected_regions from THIS branch's detections for actions - branch_detected_regions = {} - for detection in branch_detections: - branch_detected_regions[detection['class_name']] = { - 'bbox': detection['bbox'], - 'confidence': detection['confidence'] - } - - for action in branch_actions: - try: - action_type = action.type.value # Access the enum value - logger.info(f"[ACTION EXECUTE] {branch_id}: Executing action '{action_type}'") - - if action_type == 'redis_save_image': - action_result = self._execute_redis_save_image_sync( - action, input_frame, branch_detected_regions, detection_context - ) - elif action_type == 'redis_publish': - action_result = self._execute_redis_publish_sync( - action, detection_context - ) - else: - logger.warning(f"[ACTION UNKNOWN] {branch_id}: Unknown action type '{action_type}'") - action_result = {'status': 'error', 'message': f'Unknown action type: {action_type}'} - - actions_executed.append({ - 'action_type': action_type, - 'result': action_result - }) - - logger.info(f"[ACTION COMPLETE] {branch_id}: Action '{action_type}' result: {action_result.get('status')}") - - except Exception as e: - action_type = getattr(action, 'type', None) - if action_type: - action_type = action_type.value if hasattr(action_type, 'value') else str(action_type) - logger.error(f"[ACTION ERROR] {branch_id}: Error executing action '{action_type}': {e}", exc_info=True) - actions_executed.append({ - 'action_type': action_type, - 'result': {'status': 'error', 'message': str(e)} - }) - - # Add actions executed to result - if actions_executed: - result['actions_executed'] = actions_executed - - # Handle nested branches ONLY if parent found valid detections - nested_branches = getattr(branch_config, 'branches', []) - if nested_branches: - # Check if parent branch found any valid detections - if not branch_detections: - logger.warning(f"[BRANCH SKIP] {branch_id}: Skipping {len(nested_branches)} nested branches - parent found no valid detections") - else: - logger.debug(f"Branch {branch_id}: executing {len(nested_branches)} nested branches") - - # Create detected_regions from THIS branch's detections for nested branches - # Nested branches should see their immediate parent's detections, not the root pipeline - nested_detected_regions = {} - for detection in branch_detections: - nested_detected_regions[detection['class_name']] = { - 'bbox': detection['bbox'], - 'confidence': detection['confidence'] - } - - logger.info(f"[NESTED REGIONS] {branch_id}: Passing {list(nested_detected_regions.keys())} to nested branches") - - # Note: For simplicity, nested branches are executed sequentially in this sync method - # In a full async implementation, these could also be parallelized - nested_results = {} - for nested_branch in nested_branches: - nested_result = self._execute_single_branch_sync( - input_frame, nested_branch, nested_detected_regions, detection_context - ) - nested_branch_id = getattr(nested_branch, 'model_id', 'unknown') - nested_results[nested_branch_id] = nested_result - - result['nested_branches'] = nested_results - - except Exception as e: - logger.error(f"[BRANCH ERROR] {branch_id}: Error in execution: {e}", exc_info=True) - result['status'] = 'error' - result['message'] = str(e) - - result['processing_time'] = time.time() - start_time - - # Summary log - logger.info(f"[BRANCH COMPLETE] {branch_id}: status={result['status']}, " - f"processing_time={result['processing_time']:.3f}s, " - f"result_keys={list(result['result'].keys()) if result['result'] else 'none'}") - - return result - - def _execute_redis_save_image_sync(self, - action: Dict, - frame: np.ndarray, - detected_regions: Dict[str, Any], - context: Dict[str, Any]) -> Dict[str, Any]: - """Execute redis_save_image action synchronously.""" - if not self.redis_manager: - return {'status': 'error', 'message': 'Redis not available'} - - try: - # Get image to save (cropped or full frame) - image_to_save = frame - region_name = action.params.get('region') - - bbox = None - if region_name and region_name in detected_regions: - # Crop the specified region - bbox = detected_regions[region_name]['bbox'] - elif region_name and region_name.lower() == 'frontal' and 'front_rear' in detected_regions: - # Special case: "frontal" region maps to "front_rear" detection - bbox = detected_regions['front_rear']['bbox'] - - if bbox is not None: - x1, y1, x2, y2 = [int(coord) for coord in bbox] - cropped = frame[y1:y2, x1:x2] - if cropped.size > 0: - image_to_save = cropped - logger.debug(f"Cropped region '{region_name}' for redis_save_image") - else: - logger.warning(f"Empty crop for region '{region_name}', using full frame") - - # Format key with context - key = action.params['key'].format(**context) - - # Convert image to bytes - import cv2 - image_format = action.params.get('format', 'jpeg') - quality = action.params.get('quality', 90) - - if image_format.lower() == 'jpeg': - encode_param = [cv2.IMWRITE_JPEG_QUALITY, quality] - _, image_bytes = cv2.imencode('.jpg', image_to_save, encode_param) - else: - _, image_bytes = cv2.imencode('.png', image_to_save) - - # Save to Redis synchronously using a sync Redis client - try: - import redis - import cv2 - - # Create a synchronous Redis client with same connection details - sync_redis = redis.Redis( - host=self.redis_manager.host, - port=self.redis_manager.port, - password=self.redis_manager.password, - db=self.redis_manager.db, - decode_responses=False, # We're storing binary data - socket_timeout=self.redis_manager.socket_timeout, - socket_connect_timeout=self.redis_manager.socket_connect_timeout - ) - - # Encode the image - if image_format.lower() == 'jpeg': - encode_param = [cv2.IMWRITE_JPEG_QUALITY, quality] - success, encoded_image = cv2.imencode('.jpg', image_to_save, encode_param) - else: - success, encoded_image = cv2.imencode('.png', image_to_save) - - if not success: - return {'status': 'error', 'message': 'Failed to encode image'} - - # Save to Redis with expiration - expire_seconds = action.params.get('expire_seconds', 600) - result = sync_redis.setex(key, expire_seconds, encoded_image.tobytes()) - - sync_redis.close() # Clean up connection - - if result: - # Add image_key to context for subsequent actions - context['image_key'] = key - return {'status': 'success', 'key': key} - else: - return {'status': 'error', 'message': 'Failed to save image to Redis'} - - except Exception as redis_error: - logger.error(f"Error calling Redis from sync context: {redis_error}") - return {'status': 'error', 'message': f'Redis operation failed: {redis_error}'} - - except Exception as e: - logger.error(f"Error in redis_save_image action: {e}", exc_info=True) - return {'status': 'error', 'message': str(e)} - - def _execute_redis_publish_sync(self, action: Dict, context: Dict[str, Any]) -> Dict[str, Any]: - """Execute redis_publish action synchronously.""" - if not self.redis_manager: - return {'status': 'error', 'message': 'Redis not available'} - - try: - channel = action.params['channel'] - message_template = action.params['message'] - - # Debug the message template - logger.debug(f"Message template: {repr(message_template)}") - logger.debug(f"Context keys: {list(context.keys())}") - - # Format message with context - handle JSON string formatting carefully - # The message template contains JSON which causes issues with .format() - # Use string replacement instead of format to avoid JSON brace conflicts - try: - # Ensure image_key is available for message formatting - if 'image_key' not in context: - context['image_key'] = '' # Default empty value if redis_save_image failed - - # Use string replacement to avoid JSON formatting issues - message = message_template - for key, value in context.items(): - placeholder = '{' + key + '}' - message = message.replace(placeholder, str(value)) - - logger.debug(f"Formatted message using replacement: {message}") - except Exception as e: - logger.error(f"Message formatting failed: {e}") - logger.error(f"Template: {repr(message_template)}") - logger.error(f"Context: {context}") - return {'status': 'error', 'message': f'Message formatting failed: {e}'} - - # Publish message synchronously using a sync Redis client - try: - import redis - - # Create a synchronous Redis client with same connection details - sync_redis = redis.Redis( - host=self.redis_manager.host, - port=self.redis_manager.port, - password=self.redis_manager.password, - db=self.redis_manager.db, - decode_responses=True, # For publishing text messages - socket_timeout=self.redis_manager.socket_timeout, - socket_connect_timeout=self.redis_manager.socket_connect_timeout - ) - - # Publish message - result = sync_redis.publish(channel, message) - sync_redis.close() # Clean up connection - - if result >= 0: # Redis publish returns number of subscribers - return {'status': 'success', 'subscribers': result, 'channel': channel} - else: - return {'status': 'error', 'message': 'Failed to publish message to Redis'} - - except Exception as redis_error: - logger.error(f"Error calling Redis from sync context: {redis_error}") - return {'status': 'error', 'message': f'Redis operation failed: {redis_error}'} - - except Exception as e: - logger.error(f"Error in redis_publish action: {e}", exc_info=True) - return {'status': 'error', 'message': str(e)} - - def get_statistics(self) -> Dict[str, Any]: - """Get branch processor statistics.""" - return { - **self.stats, - 'loaded_models': list(self.branch_models.keys()), - 'model_count': len(self.branch_models) - } - - def cleanup(self): - """Cleanup resources.""" - if self.executor: - self.executor.shutdown(wait=False) - - # Clear model cache - self.branch_models.clear() - - logger.info("BranchProcessor cleaned up") \ No newline at end of file diff --git a/core/detection/pipeline.py b/core/detection/pipeline.py deleted file mode 100644 index d395f3a..0000000 --- a/core/detection/pipeline.py +++ /dev/null @@ -1,1169 +0,0 @@ -""" -Detection Pipeline Module. -Main detection pipeline orchestration that coordinates detection flow and execution. -""" -import asyncio -import logging -import time -import uuid -from datetime import datetime -from typing import Dict, List, Optional, Any -from concurrent.futures import ThreadPoolExecutor -import numpy as np - -from ..models.inference import YOLOWrapper -from ..models.pipeline import PipelineParser -from .branches import BranchProcessor -from ..storage.redis import RedisManager -from ..storage.database import DatabaseManager -from ..storage.license_plate import LicensePlateManager - -logger = logging.getLogger(__name__) - - -class DetectionPipeline: - """ - Main detection pipeline that orchestrates the complete detection flow. - Handles detection execution, branch coordination, and result aggregation. - """ - - def __init__(self, pipeline_parser: PipelineParser, model_manager: Any, model_id: int, message_sender=None): - """ - Initialize detection pipeline. - - Args: - pipeline_parser: Pipeline parser with loaded configuration - model_manager: Model manager for loading models - model_id: The model ID to use for loading models - message_sender: Optional callback function for sending WebSocket messages - """ - self.pipeline_parser = pipeline_parser - self.model_manager = model_manager - self.model_id = model_id - self.message_sender = message_sender - - # Initialize components - self.branch_processor = BranchProcessor(model_manager, model_id) - self.redis_manager = None - self.db_manager = None - self.license_plate_manager = None - - # Main detection model - self.detection_model: Optional[YOLOWrapper] = None - self.detection_model_id = None - - # Thread pool for parallel processing - self.executor = ThreadPoolExecutor(max_workers=4) - - # Pipeline configuration - self.pipeline_config = pipeline_parser.pipeline_config - - # SessionId to subscriptionIdentifier mapping - self.session_to_subscription = {} - - # SessionId to processing results mapping (for combining with license plate results) - self.session_processing_results = {} - - # Field mappings from parallelActions (e.g., {"car_brand": "{car_brand_cls_v3.brand}"}) - self.field_mappings = {} - self._parse_field_mappings() - - # Statistics - self.stats = { - 'detections_processed': 0, - 'branches_executed': 0, - 'actions_executed': 0, - 'total_processing_time': 0.0 - } - - logger.info("DetectionPipeline initialized") - - def _parse_field_mappings(self): - """ - Parse field mappings from parallelActions.postgresql_update_combined.fields. - Extracts mappings like {"car_brand": "{car_brand_cls_v3.brand}"} for dynamic field resolution. - """ - try: - if not self.pipeline_config or not hasattr(self.pipeline_config, 'parallel_actions'): - return - - for action in self.pipeline_config.parallel_actions: - if action.type.value == 'postgresql_update_combined': - fields = action.params.get('fields', {}) - self.field_mappings = fields - logger.info(f"[FIELD MAPPINGS] Parsed from pipeline config: {self.field_mappings}") - break - - except Exception as e: - logger.error(f"Error parsing field mappings: {e}", exc_info=True) - - async def initialize(self) -> bool: - """ - Initialize all pipeline components including models, Redis, and database. - - Returns: - True if successful, False otherwise - """ - try: - # Initialize Redis connection - if self.pipeline_parser.redis_config: - self.redis_manager = RedisManager(self.pipeline_parser.redis_config.__dict__) - if not await self.redis_manager.initialize(): - logger.error("Failed to initialize Redis connection") - return False - logger.info("Redis connection initialized") - - # Initialize database connection - if self.pipeline_parser.postgresql_config: - self.db_manager = DatabaseManager(self.pipeline_parser.postgresql_config.__dict__) - if not self.db_manager.connect(): - logger.error("Failed to initialize database connection") - return False - # Create required tables - if not self.db_manager.create_car_frontal_info_table(): - logger.warning("Failed to create car_frontal_info table") - logger.info("Database connection initialized") - - # Initialize license plate manager (using same Redis config as main Redis manager) - if self.pipeline_parser.redis_config: - self.license_plate_manager = LicensePlateManager(self.pipeline_parser.redis_config.__dict__) - if not await self.license_plate_manager.initialize(self._on_license_plate_result): - logger.error("Failed to initialize license plate manager") - return False - logger.info("License plate manager initialized") - - - # Initialize main detection model - if not await self._initialize_detection_model(): - logger.error("Failed to initialize detection model") - return False - - # Initialize branch processor - if not await self.branch_processor.initialize( - self.pipeline_config, - self.redis_manager, - self.db_manager - ): - logger.error("Failed to initialize branch processor") - return False - - logger.info("Detection pipeline initialization completed successfully") - return True - - except Exception as e: - logger.error(f"Error initializing detection pipeline: {e}", exc_info=True) - return False - - async def _initialize_detection_model(self) -> bool: - """ - Load and initialize the main detection model. - - Returns: - True if successful, False otherwise - """ - try: - if not self.pipeline_config: - logger.warning("No pipeline configuration found") - return False - - model_file = getattr(self.pipeline_config, 'model_file', None) - model_id = getattr(self.pipeline_config, 'model_id', None) - - if not model_file: - logger.warning("No detection model file specified") - return False - - # Load detection model - logger.info(f"Loading detection model: {model_id} ({model_file})") - self.detection_model = self.model_manager.get_yolo_model(self.model_id, model_file) - if not self.detection_model: - logger.error(f"Failed to load detection model {model_file} from model {self.model_id}") - return False - - self.detection_model_id = model_id - logger.info(f"Detection model {model_id} loaded successfully") - return True - - except Exception as e: - logger.error(f"Error initializing detection model: {e}", exc_info=True) - return False - - def _extract_fields_from_branches(self, branch_results: Dict[str, Any]) -> Dict[str, Any]: - """ - Extract fields dynamically from branch results using field mappings. - - Args: - branch_results: Dictionary of branch execution results - - Returns: - Dictionary with extracted field values (e.g., {"car_brand": "Honda", "body_type": "Sedan"}) - """ - extracted = {} - - try: - for db_field_name, template in self.field_mappings.items(): - # Parse template like "{car_brand_cls_v3.brand}" -> branch_id="car_brand_cls_v3", field="brand" - if template.startswith('{') and template.endswith('}'): - var_name = template[1:-1] - if '.' in var_name: - branch_id, field_name = var_name.split('.', 1) - - # Look up value in branch_results - if branch_id in branch_results: - branch_data = branch_results[branch_id] - if isinstance(branch_data, dict) and 'result' in branch_data: - result_data = branch_data['result'] - if isinstance(result_data, dict) and field_name in result_data: - extracted[field_name] = result_data[field_name] - logger.debug(f"[DYNAMIC EXTRACT] {field_name}={result_data[field_name]} from branch {branch_id}") - else: - logger.debug(f"[DYNAMIC EXTRACT] Field '{field_name}' not found in branch {branch_id}") - else: - logger.debug(f"[DYNAMIC EXTRACT] Branch '{branch_id}' not in results") - - except Exception as e: - logger.error(f"Error extracting fields from branches: {e}", exc_info=True) - - return extracted - - async def _on_license_plate_result(self, session_id: str, license_data: Dict[str, Any]): - """ - Callback for handling license plate results from LPR service. - - Args: - session_id: Session identifier - license_data: License plate data including text and confidence - """ - try: - license_text = license_data.get('license_plate_text', '') - confidence = license_data.get('confidence', 0.0) - - logger.info(f"[LICENSE PLATE CALLBACK] Session {session_id}: " - f"text='{license_text}', confidence={confidence:.3f}") - - # Find matching subscriptionIdentifier for this sessionId - subscription_id = self.session_to_subscription.get(session_id) - - if not subscription_id: - logger.warning(f"[LICENSE PLATE] No subscription found for sessionId '{session_id}' (type: {type(session_id)}), cannot send imageDetection") - logger.warning(f"[LICENSE PLATE DEBUG] Current session mappings: {dict(self.session_to_subscription)}") - - # Try to find by type conversion in case of type mismatch - # Try as integer if session_id is string - if isinstance(session_id, str) and session_id.isdigit(): - session_id_int = int(session_id) - subscription_id = self.session_to_subscription.get(session_id_int) - if subscription_id: - logger.info(f"[LICENSE PLATE] Found subscription using int conversion: '{session_id}' -> {session_id_int} -> '{subscription_id}'") - else: - logger.error(f"[LICENSE PLATE] Failed to find subscription with int conversion") - return - # Try as string if session_id is integer - elif isinstance(session_id, int): - session_id_str = str(session_id) - subscription_id = self.session_to_subscription.get(session_id_str) - if subscription_id: - logger.info(f"[LICENSE PLATE] Found subscription using string conversion: {session_id} -> '{session_id_str}' -> '{subscription_id}'") - else: - logger.error(f"[LICENSE PLATE] Failed to find subscription with string conversion") - return - else: - logger.error(f"[LICENSE PLATE] Failed to find subscription with any type conversion") - return - - # Send imageDetection message with license plate data combined with processing results - await self._send_license_plate_message(subscription_id, license_text, confidence, session_id) - - # Update database with license plate information if database manager is available - if self.db_manager and license_text: - success = self.db_manager.execute_update( - table='car_frontal_info', - key_field='session_id', - key_value=session_id, - fields={ - 'license_character': license_text, - 'license_type': 'LPR_detected' # Mark as detected by LPR service - } - ) - if success: - logger.info(f"[LICENSE PLATE] Updated database for session {session_id}") - else: - logger.warning(f"[LICENSE PLATE] Failed to update database for session {session_id}") - - except Exception as e: - logger.error(f"Error in license plate result callback: {e}", exc_info=True) - - - async def _send_license_plate_message(self, subscription_id: str, license_text: str, confidence: float, session_id: str = None): - """ - Send imageDetection message with license plate data plus any available processing results. - - Args: - subscription_id: Subscription identifier to send message to - license_text: License plate text - confidence: License plate confidence score - session_id: Session identifier for looking up processing results - """ - try: - if not self.message_sender: - logger.warning("No message sender configured, cannot send imageDetection") - return - - # Import here to avoid circular imports - from ..communication.models import ImageDetectionMessage, DetectionData - - # Get processing results for this session from stored results - car_brand = None - body_type = None - - # Find session_id from session mappings (we need session_id as key) - session_id_for_lookup = None - - # Try direct lookup first (if session_id is already the right type) - if session_id in self.session_processing_results: - session_id_for_lookup = session_id - else: - # Try to find by type conversion - for stored_session_id in self.session_processing_results.keys(): - if str(stored_session_id) == str(session_id): - session_id_for_lookup = stored_session_id - break - - if session_id_for_lookup and session_id_for_lookup in self.session_processing_results: - branch_results = self.session_processing_results[session_id_for_lookup] - logger.info(f"[LICENSE PLATE] Retrieved processing results for session {session_id_for_lookup}") - - # Extract fields dynamically using field mappings from pipeline config - extracted_fields = self._extract_fields_from_branches(branch_results) - car_brand = extracted_fields.get('brand') - body_type = extracted_fields.get('body_type') - - logger.info(f"[LICENSE PLATE] Extracted fields: brand={car_brand}, body_type={body_type}") - - # Clean up stored results after use - del self.session_processing_results[session_id_for_lookup] - logger.debug(f"[LICENSE PLATE] Cleaned up stored results for session {session_id_for_lookup}") - else: - logger.warning(f"[LICENSE PLATE] No processing results found for session {session_id}") - - # Create detection data with combined information - detection_data_obj = DetectionData( - detection={ - "carBrand": car_brand, - "carModel": None, - "bodyType": body_type, - "licensePlateText": license_text, - "licensePlateConfidence": confidence - }, - modelId=self.model_id, - modelName=self.pipeline_parser.pipeline_config.model_id if self.pipeline_parser.pipeline_config else "detection_model" - ) - - # Create imageDetection message - detection_message = ImageDetectionMessage( - subscriptionIdentifier=subscription_id, - data=detection_data_obj - ) - - # Send message - await self.message_sender(detection_message) - logger.info(f"[COMBINED MESSAGE] Sent imageDetection with brand='{car_brand}', bodyType='{body_type}', license='{license_text}' to '{subscription_id}'") - - except Exception as e: - logger.error(f"Error sending license plate imageDetection message: {e}", exc_info=True) - - async def _send_initial_detection_message(self, subscription_id: str): - """ - Send initial imageDetection message when vehicle is first detected. - - Args: - subscription_id: Subscription identifier to send message to - """ - try: - if not self.message_sender: - logger.warning("No message sender configured, cannot send imageDetection") - return - - # Import here to avoid circular imports - from ..communication.models import ImageDetectionMessage, DetectionData - - # Create detection data with all fields as None (vehicle just detected, no classification yet) - detection_data_obj = DetectionData( - detection={ - "carBrand": None, - "carModel": None, - "bodyType": None, - "licensePlateText": None, - "licensePlateConfidence": None - }, - modelId=self.model_id, - modelName=self.pipeline_parser.pipeline_config.model_id if self.pipeline_parser.pipeline_config else "detection_model" - ) - - # Create imageDetection message - detection_message = ImageDetectionMessage( - subscriptionIdentifier=subscription_id, - data=detection_data_obj - ) - - # Send message - await self.message_sender(detection_message) - logger.info(f"[INITIAL DETECTION] Sent imageDetection for vehicle detection to '{subscription_id}'") - - except Exception as e: - logger.error(f"Error sending initial detection imageDetection message: {e}", exc_info=True) - - async def execute_detection_phase(self, - frame: np.ndarray, - display_id: str, - subscription_id: str) -> Dict[str, Any]: - """ - Execute only the detection phase - run main detection and send imageDetection message. - This is the first phase that runs when a vehicle is validated. - - Args: - frame: Input frame to process - display_id: Display identifier - subscription_id: Subscription identifier - - Returns: - Dictionary with detection phase results - """ - start_time = time.time() - result = { - 'status': 'success', - 'detections': [], - 'message_sent': False, - 'processing_time': 0.0, - 'timestamp': datetime.now().isoformat() - } - - try: - # Run main detection model - if not self.detection_model: - result['status'] = 'error' - result['message'] = 'Detection model not available' - return result - - # Create detection context - detection_context = { - 'display_id': display_id, - 'subscription_id': subscription_id, - 'timestamp': datetime.now().strftime("%Y-%m-%dT%H-%M-%S"), - 'timestamp_ms': int(time.time() * 1000) - } - - # Run inference on single snapshot using .predict() method - detection_results = self.detection_model.model.predict( - frame, - conf=getattr(self.pipeline_config, 'min_confidence', 0.6), - verbose=False - ) - - # Process detection results using clean logic - valid_detections = [] - detected_regions = {} - - if detection_results and len(detection_results) > 0: - result_obj = detection_results[0] - trigger_classes = getattr(self.pipeline_config, 'trigger_classes', []) - - # Handle .predict() results which have .boxes for detection models - if hasattr(result_obj, 'boxes') and result_obj.boxes is not None: - logger.info(f"[DETECTION PHASE] Found {len(result_obj.boxes)} raw detections from {getattr(self.pipeline_config, 'model_id', 'unknown')}") - - for i, box in enumerate(result_obj.boxes): - class_id = int(box.cls[0]) - confidence = float(box.conf[0]) - bbox = box.xyxy[0].cpu().numpy().tolist() # [x1, y1, x2, y2] - class_name = self.detection_model.model.names[class_id] - - logger.info(f"[DETECTION PHASE {i+1}] {class_name}: bbox={bbox}, conf={confidence:.3f}") - - # Check if detection matches trigger classes - if trigger_classes and class_name not in trigger_classes: - logger.debug(f"[DETECTION PHASE] Filtered '{class_name}' - not in trigger_classes {trigger_classes}") - continue - - logger.info(f"[DETECTION PHASE] Accepted '{class_name}' - matches trigger_classes") - - # Store detection info - detection_info = { - 'class_name': class_name, - 'confidence': confidence, - 'bbox': bbox - } - valid_detections.append(detection_info) - - # Store region for processing phase - detected_regions[class_name] = { - 'bbox': bbox, - 'confidence': confidence - } - else: - logger.warning("[DETECTION PHASE] No boxes found in detection results") - - # Store detected_regions in result for processing phase - result['detected_regions'] = detected_regions - - result['detections'] = valid_detections - - # If we have valid detections, create session and send initial imageDetection - if valid_detections: - logger.info(f"Found {len(valid_detections)} valid detections, storing session mapping") - - # Store mapping from display_id to subscriptionIdentifier (for detection phase) - # Note: We'll store session_id mapping later in processing phase - self.session_to_subscription[display_id] = subscription_id - logger.info(f"[SESSION MAPPING] Stored mapping: displayId '{display_id}' -> subscriptionIdentifier '{subscription_id}'") - - # Send initial imageDetection message with empty detection data - await self._send_initial_detection_message(subscription_id) - - logger.info(f"Detection phase completed - {len(valid_detections)} detections found for {display_id}") - result['message_sent'] = True - else: - logger.debug("No valid detections found in detection phase") - - except Exception as e: - logger.error(f"Error in detection phase: {e}", exc_info=True) - result['status'] = 'error' - result['message'] = str(e) - - result['processing_time'] = time.time() - start_time - return result - - async def execute_processing_phase(self, - frame: np.ndarray, - display_id: str, - session_id: str, - subscription_id: str, - detected_regions: Dict[str, Any] = None) -> Dict[str, Any]: - """ - Execute the processing phase - run branches and database operations after receiving sessionId. - This is the second phase that runs after backend sends setSessionId. - - Args: - frame: Input frame to process - display_id: Display identifier - session_id: Session ID from backend - subscription_id: Subscription identifier - detected_regions: Pre-detected regions from detection phase - - Returns: - Dictionary with processing phase results - """ - start_time = time.time() - result = { - 'status': 'success', - 'branch_results': {}, - 'actions_executed': [], - 'session_id': session_id, - 'processing_time': 0.0, - 'timestamp': datetime.now().isoformat() - } - - try: - # Create enhanced detection context with session_id - detection_context = { - 'display_id': display_id, - 'session_id': session_id, - 'subscription_id': subscription_id, - 'timestamp': datetime.now().strftime("%Y-%m-%dT%H-%M-%S"), - 'timestamp_ms': int(time.time() * 1000), - 'uuid': str(uuid.uuid4()), - 'filename': f"{uuid.uuid4()}.jpg" - } - - # If no detected_regions provided, re-run detection to get them - if not detected_regions: - # Use .predict() method for detection - detection_results = self.detection_model.model.predict( - frame, - conf=getattr(self.pipeline_config, 'min_confidence', 0.6), - verbose=False - ) - - detected_regions = {} - if detection_results and len(detection_results) > 0: - result_obj = detection_results[0] - if hasattr(result_obj, 'boxes') and result_obj.boxes is not None: - for box in result_obj.boxes: - class_id = int(box.cls[0]) - confidence = float(box.conf[0]) - bbox = box.xyxy[0].cpu().numpy().tolist() # [x1, y1, x2, y2] - class_name = self.detection_model.model.names[class_id] - - detected_regions[class_name] = { - 'bbox': bbox, - 'confidence': confidence - } - - # Store session mapping for license plate callback - if session_id: - self.session_to_subscription[session_id] = subscription_id - logger.info(f"[SESSION MAPPING] Stored mapping: sessionId '{session_id}' -> subscriptionIdentifier '{subscription_id}'") - - # Initialize database record with session_id - if session_id and self.db_manager: - success = self.db_manager.insert_initial_detection( - display_id=display_id, - captured_timestamp=detection_context['timestamp'], - session_id=session_id - ) - if success: - logger.info(f"Created initial database record with session {session_id}") - else: - logger.warning(f"Failed to create initial database record for session {session_id}") - - # Execute branches in parallel - if hasattr(self.pipeline_config, 'branches') and self.pipeline_config.branches: - branch_results = await self.branch_processor.execute_branches( - frame=frame, - branches=self.pipeline_config.branches, - detected_regions=detected_regions, - detection_context=detection_context - ) - result['branch_results'] = branch_results - logger.info(f"Executed {len(branch_results)} branches for session {session_id}") - - # Execute immediate actions (non-parallel) - immediate_actions = getattr(self.pipeline_config, 'actions', []) - if immediate_actions: - executed_actions = await self._execute_immediate_actions( - actions=immediate_actions, - frame=frame, - detected_regions=detected_regions, - detection_context=detection_context - ) - result['actions_executed'].extend(executed_actions) - - # Execute parallel actions (after all branches complete) - parallel_actions = getattr(self.pipeline_config, 'parallel_actions', []) - if parallel_actions: - # Add branch results to context - enhanced_context = {**detection_context} - if result['branch_results']: - enhanced_context['branch_results'] = result['branch_results'] - - executed_parallel_actions = await self._execute_parallel_actions( - actions=parallel_actions, - frame=frame, - detected_regions=detected_regions, - context=enhanced_context - ) - result['actions_executed'].extend(executed_parallel_actions) - - # Store processing results for later combination with license plate data - if result['branch_results'] and session_id: - self.session_processing_results[session_id] = result['branch_results'] - logger.info(f"[PROCESSING RESULTS] Stored results for session {session_id} for later combination") - - logger.info(f"Processing phase completed for session {session_id}: " - f"{len(result['branch_results'])} branches, {len(result['actions_executed'])} actions") - - except Exception as e: - logger.error(f"Error in processing phase: {e}", exc_info=True) - result['status'] = 'error' - result['message'] = str(e) - - result['processing_time'] = time.time() - start_time - return result - - - async def execute_detection(self, - frame: np.ndarray, - display_id: str, - session_id: Optional[str] = None, - subscription_id: Optional[str] = None) -> Dict[str, Any]: - """ - Execute the main detection pipeline on a frame. - - Args: - frame: Input frame to process - display_id: Display identifier - session_id: Optional session ID - subscription_id: Optional subscription identifier - - Returns: - Dictionary with detection results - """ - start_time = time.time() - result = { - 'status': 'success', - 'detections': [], - 'branch_results': {}, - 'actions_executed': [], - 'session_id': session_id, - 'processing_time': 0.0, - 'timestamp': datetime.now().isoformat() - } - - try: - # Update stats - self.stats['detections_processed'] += 1 - - # Run main detection model - if not self.detection_model: - result['status'] = 'error' - result['message'] = 'Detection model not available' - return result - - # Create detection context - detection_context = { - 'display_id': display_id, - 'session_id': session_id, - 'subscription_id': subscription_id, - 'timestamp': datetime.now().strftime("%Y-%m-%dT%H-%M-%S"), - 'timestamp_ms': int(time.time() * 1000), - 'uuid': str(uuid.uuid4()), - 'filename': f"{uuid.uuid4()}.jpg" - } - - - # Run inference on single snapshot using .predict() method - detection_results = self.detection_model.model.predict( - frame, - conf=getattr(self.pipeline_config, 'min_confidence', 0.6), - verbose=False - ) - - # Process detection results - detected_regions = {} - valid_detections = [] - - if detection_results and len(detection_results) > 0: - result_obj = detection_results[0] - trigger_classes = getattr(self.pipeline_config, 'trigger_classes', []) - - # Handle .predict() results which have .boxes for detection models - if hasattr(result_obj, 'boxes') and result_obj.boxes is not None: - logger.info(f"[PIPELINE RAW] Found {len(result_obj.boxes)} raw detections from {getattr(self.pipeline_config, 'model_id', 'unknown')}") - - for i, box in enumerate(result_obj.boxes): - class_id = int(box.cls[0]) - confidence = float(box.conf[0]) - bbox = box.xyxy[0].cpu().numpy().tolist() # [x1, y1, x2, y2] - class_name = self.detection_model.model.names[class_id] - - logger.info(f"[PIPELINE RAW {i+1}] {class_name}: bbox={bbox}, conf={confidence:.3f}") - - # Check if detection matches trigger classes - if trigger_classes and class_name not in trigger_classes: - continue - - # Store detection info - detection_info = { - 'class_name': class_name, - 'confidence': confidence, - 'bbox': bbox - } - valid_detections.append(detection_info) - - # Store region for cropping - detected_regions[class_name] = { - 'bbox': bbox, - 'confidence': confidence - } - logger.info(f"[PIPELINE DETECTION] {class_name}: bbox={bbox}, conf={confidence:.3f}") - - result['detections'] = valid_detections - - # If we have valid detections, proceed with branches and actions - if valid_detections: - logger.info(f"Found {len(valid_detections)} valid detections for pipeline processing") - - # Initialize database record if session_id is provided - if session_id and self.db_manager: - success = self.db_manager.insert_initial_detection( - display_id=display_id, - captured_timestamp=detection_context['timestamp'], - session_id=session_id - ) - if not success: - logger.warning(f"Failed to create initial database record for session {session_id}") - - # Execute branches in parallel - if hasattr(self.pipeline_config, 'branches') and self.pipeline_config.branches: - branch_results = await self.branch_processor.execute_branches( - frame=frame, - branches=self.pipeline_config.branches, - detected_regions=detected_regions, - detection_context=detection_context - ) - result['branch_results'] = branch_results - self.stats['branches_executed'] += len(branch_results) - - # Execute immediate actions (non-parallel) - immediate_actions = getattr(self.pipeline_config, 'actions', []) - if immediate_actions: - executed_actions = await self._execute_immediate_actions( - actions=immediate_actions, - frame=frame, - detected_regions=detected_regions, - detection_context=detection_context - ) - result['actions_executed'].extend(executed_actions) - - # Execute parallel actions (after all branches complete) - parallel_actions = getattr(self.pipeline_config, 'parallel_actions', []) - if parallel_actions: - # Add branch results to context - enhanced_context = {**detection_context} - if result['branch_results']: - enhanced_context['branch_results'] = result['branch_results'] - - executed_parallel_actions = await self._execute_parallel_actions( - actions=parallel_actions, - frame=frame, - detected_regions=detected_regions, - context=enhanced_context - ) - result['actions_executed'].extend(executed_parallel_actions) - - self.stats['actions_executed'] += len(result['actions_executed']) - else: - logger.debug("No valid detections found for pipeline processing") - - except Exception as e: - logger.error(f"Error in detection pipeline execution: {e}", exc_info=True) - result['status'] = 'error' - result['message'] = str(e) - - # Update timing - processing_time = time.time() - start_time - result['processing_time'] = processing_time - self.stats['total_processing_time'] += processing_time - - return result - - async def _execute_immediate_actions(self, - actions: List[Dict], - frame: np.ndarray, - detected_regions: Dict[str, Any], - detection_context: Dict[str, Any]) -> List[Dict]: - """ - Execute immediate actions (non-parallel). - - Args: - actions: List of action configurations - frame: Input frame - detected_regions: Dictionary of detected regions - detection_context: Detection context data - - Returns: - List of executed action results - """ - executed_actions = [] - - for action in actions: - try: - action_type = action.type.value - logger.debug(f"Executing immediate action: {action_type}") - - if action_type == 'redis_save_image': - result = await self._execute_redis_save_image( - action, frame, detected_regions, detection_context - ) - elif action_type == 'redis_publish': - result = await self._execute_redis_publish( - action, detection_context - ) - else: - logger.warning(f"Unknown immediate action type: {action_type}") - result = {'status': 'error', 'message': f'Unknown action type: {action_type}'} - - executed_actions.append({ - 'action_type': action_type, - 'result': result - }) - - except Exception as e: - logger.error(f"Error executing immediate action {action_type}: {e}", exc_info=True) - executed_actions.append({ - 'action_type': action.type.value, - 'result': {'status': 'error', 'message': str(e)} - }) - - return executed_actions - - async def _execute_parallel_actions(self, - actions: List[Dict], - frame: np.ndarray, - detected_regions: Dict[str, Any], - context: Dict[str, Any]) -> List[Dict]: - """ - Execute parallel actions (after branches complete). - - Args: - actions: List of parallel action configurations - frame: Input frame - detected_regions: Dictionary of detected regions - context: Enhanced context with branch results - - Returns: - List of executed action results - """ - executed_actions = [] - - for action in actions: - try: - action_type = action.type.value - logger.debug(f"Executing parallel action: {action_type}") - - if action_type == 'postgresql_update_combined': - result = await self._execute_postgresql_update_combined(action, context) - - # Update session state with processing results after database update - if result.get('status') == 'success': - await self._update_session_with_processing_results(context) - else: - logger.warning(f"Unknown parallel action type: {action_type}") - result = {'status': 'error', 'message': f'Unknown action type: {action_type}'} - - executed_actions.append({ - 'action_type': action_type, - 'result': result - }) - - except Exception as e: - logger.error(f"Error executing parallel action {action_type}: {e}", exc_info=True) - executed_actions.append({ - 'action_type': action.type.value, - 'result': {'status': 'error', 'message': str(e)} - }) - - return executed_actions - - async def _execute_redis_save_image(self, - action: Dict, - frame: np.ndarray, - detected_regions: Dict[str, Any], - context: Dict[str, Any]) -> Dict[str, Any]: - """Execute redis_save_image action.""" - if not self.redis_manager: - return {'status': 'error', 'message': 'Redis not available'} - - try: - # Get image to save (cropped or full frame) - image_to_save = frame - region_name = action.params.get('region') - - if region_name and region_name in detected_regions: - # Crop the specified region - bbox = detected_regions[region_name]['bbox'] - x1, y1, x2, y2 = [int(coord) for coord in bbox] - cropped = frame[y1:y2, x1:x2] - if cropped.size > 0: - image_to_save = cropped - logger.debug(f"Cropped region '{region_name}' for redis_save_image") - else: - logger.warning(f"Empty crop for region '{region_name}', using full frame") - - # Format key with context - key = action.params['key'].format(**context) - - # Save image to Redis - result = await self.redis_manager.save_image( - key=key, - image=image_to_save, - expire_seconds=action.params.get('expire_seconds'), - image_format=action.params.get('format', 'jpeg'), - quality=action.params.get('quality', 90) - ) - - if result: - # Add image_key to context for subsequent actions - context['image_key'] = key - return {'status': 'success', 'key': key} - else: - return {'status': 'error', 'message': 'Failed to save image to Redis'} - - except Exception as e: - logger.error(f"Error in redis_save_image action: {e}", exc_info=True) - return {'status': 'error', 'message': str(e)} - - async def _execute_redis_publish(self, action: Dict, context: Dict[str, Any]) -> Dict[str, Any]: - """Execute redis_publish action.""" - if not self.redis_manager: - return {'status': 'error', 'message': 'Redis not available'} - - try: - channel = action.params['channel'] - message_template = action.params['message'] - - # Format message with context - message = message_template.format(**context) - - # Publish message - result = await self.redis_manager.publish_message(channel, message) - - if result >= 0: # Redis publish returns number of subscribers - return {'status': 'success', 'subscribers': result, 'channel': channel} - else: - return {'status': 'error', 'message': 'Failed to publish message to Redis'} - - except Exception as e: - logger.error(f"Error in redis_publish action: {e}", exc_info=True) - return {'status': 'error', 'message': str(e)} - - async def _execute_postgresql_update_combined(self, - action: Dict, - context: Dict[str, Any]) -> Dict[str, Any]: - """Execute postgresql_update_combined action.""" - if not self.db_manager: - return {'status': 'error', 'message': 'Database not available'} - - try: - # Wait for required branches if specified - wait_for_branches = action.params.get('waitForBranches', []) - branch_results = context.get('branch_results', {}) - - # Check if all required branches have completed - for branch_id in wait_for_branches: - if branch_id not in branch_results: - logger.warning(f"Branch {branch_id} result not available for database update") - return {'status': 'error', 'message': f'Missing branch result: {branch_id}'} - - # Prepare fields for database update - table = action.params.get('table', 'car_frontal_info') - key_field = action.params.get('key_field', 'session_id') - key_value = action.params.get('key_value', '{session_id}').format(**context) - field_mappings = action.params.get('fields', {}) - - # Resolve field values using branch results - resolved_fields = {} - for field_name, field_template in field_mappings.items(): - try: - # Replace template variables with actual values from branch results - resolved_value = self._resolve_field_template(field_template, branch_results, context) - resolved_fields[field_name] = resolved_value - except Exception as e: - logger.warning(f"Failed to resolve field {field_name}: {e}") - resolved_fields[field_name] = None - - # Execute database update - success = self.db_manager.execute_update( - table=table, - key_field=key_field, - key_value=key_value, - fields=resolved_fields - ) - - if success: - return {'status': 'success', 'table': table, 'key': f'{key_field}={key_value}', 'fields': resolved_fields} - else: - return {'status': 'error', 'message': 'Database update failed'} - - except Exception as e: - logger.error(f"Error in postgresql_update_combined action: {e}", exc_info=True) - return {'status': 'error', 'message': str(e)} - - def _resolve_field_template(self, template: str, branch_results: Dict, context: Dict) -> str: - """ - Resolve field template using branch results and context. - - Args: - template: Template string like "{car_brand_cls_v3.brand}" - branch_results: Dictionary of branch execution results - context: Detection context - - Returns: - Resolved field value - """ - try: - # Handle simple context variables first - if template.startswith('{') and template.endswith('}'): - var_name = template[1:-1] - - # Check for branch result reference (e.g., "car_brand_cls_v3.brand") - if '.' in var_name: - branch_id, field_name = var_name.split('.', 1) - if branch_id in branch_results: - branch_data = branch_results[branch_id] - # Look for the field in branch results - if isinstance(branch_data, dict) and 'result' in branch_data: - result_data = branch_data['result'] - if isinstance(result_data, dict) and field_name in result_data: - return str(result_data[field_name]) - logger.warning(f"Field {field_name} not found in branch {branch_id} results") - return None - else: - logger.warning(f"Branch {branch_id} not found in results") - return None - - # Simple context variable - elif var_name in context: - return str(context[var_name]) - - logger.warning(f"Template variable {var_name} not found in context or branch results") - return None - - # Return template as-is if not a template variable - return template - - except Exception as e: - logger.error(f"Error resolving field template {template}: {e}") - return None - - async def _update_session_with_processing_results(self, context: Dict[str, Any]): - """ - Update session state with processing results from branch execution. - - Args: - context: Detection context containing branch results and session info - """ - try: - branch_results = context.get('branch_results', {}) - session_id = context.get('session_id', '') - subscription_id = context.get('subscription_id', '') - - if not session_id: - logger.warning("No session_id in context for processing results") - return - - # Extract fields dynamically using field mappings from pipeline config - extracted_fields = self._extract_fields_from_branches(branch_results) - car_brand = extracted_fields.get('brand') - body_type = extracted_fields.get('body_type') - - logger.info(f"[PROCESSING RESULTS] Completed for session {session_id}: " - f"brand={car_brand}, bodyType={body_type}") - - except Exception as e: - logger.error(f"Error updating session with processing results: {e}", exc_info=True) - - def get_statistics(self) -> Dict[str, Any]: - """Get detection pipeline statistics.""" - branch_stats = self.branch_processor.get_statistics() if self.branch_processor else {} - license_stats = self.license_plate_manager.get_statistics() if self.license_plate_manager else {} - - return { - 'pipeline': self.stats, - 'branches': branch_stats, - 'license_plate': license_stats, - 'redis_available': self.redis_manager is not None, - 'database_available': self.db_manager is not None, - 'detection_model_loaded': self.detection_model is not None - } - - def cleanup(self): - """Cleanup resources.""" - if self.executor: - self.executor.shutdown(wait=False) - - if self.redis_manager: - self.redis_manager.cleanup() - - if self.db_manager: - self.db_manager.disconnect() - - if self.branch_processor: - self.branch_processor.cleanup() - - if self.license_plate_manager: - # Schedule cleanup task and track it to prevent warnings - cleanup_task = asyncio.create_task(self.license_plate_manager.close()) - cleanup_task.add_done_callback(lambda _: None) # Suppress "Task exception was never retrieved" - - logger.info("Detection pipeline cleaned up") \ No newline at end of file diff --git a/core/models/__init__.py b/core/models/__init__.py deleted file mode 100644 index c817eb2..0000000 --- a/core/models/__init__.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -Models Module - MPTA management, pipeline configuration, and YOLO inference -""" - -from .manager import ModelManager -from .pipeline import ( - PipelineParser, - PipelineConfig, - TrackingConfig, - ModelBranch, - Action, - ActionType, - RedisConfig, - PostgreSQLConfig -) -from .inference import ( - YOLOWrapper, - ModelInferenceManager, - Detection, - InferenceResult -) - -__all__ = [ - # Manager - 'ModelManager', - - # Pipeline - 'PipelineParser', - 'PipelineConfig', - 'TrackingConfig', - 'ModelBranch', - 'Action', - 'ActionType', - 'RedisConfig', - 'PostgreSQLConfig', - - # Inference - 'YOLOWrapper', - 'ModelInferenceManager', - 'Detection', - 'InferenceResult', -] \ No newline at end of file diff --git a/core/models/inference.py b/core/models/inference.py deleted file mode 100644 index f96c0e8..0000000 --- a/core/models/inference.py +++ /dev/null @@ -1,447 +0,0 @@ -""" -YOLO Model Inference Wrapper - Handles model loading and inference optimization -""" - -import logging -import torch -import numpy as np -from pathlib import Path -from typing import Dict, List, Optional, Any, Tuple, Union -from threading import Lock -from dataclasses import dataclass -import cv2 - -logger = logging.getLogger(__name__) - - -@dataclass -class Detection: - """Represents a single detection result""" - bbox: List[float] # [x1, y1, x2, y2] - confidence: float - class_id: int - class_name: str - track_id: Optional[int] = None - - -@dataclass -class InferenceResult: - """Result from model inference""" - detections: List[Detection] - image_shape: Tuple[int, int] # (height, width) - inference_time: float - model_id: str - - -class YOLOWrapper: - """Wrapper for YOLO models with caching and optimization""" - - # Class-level model cache shared across all instances - _model_cache: Dict[str, Any] = {} - _cache_lock = Lock() - - def __init__(self, model_path: Path, model_id: str, device: Optional[str] = None): - """ - Initialize YOLO wrapper - - Args: - model_path: Path to the .pt model file - model_id: Unique identifier for the model - device: Device to run inference on ('cuda', 'cpu', or None for auto) - """ - self.model_path = model_path - self.model_id = model_id - - # Auto-detect device if not specified - if device is None: - self.device = 'cuda' if torch.cuda.is_available() else 'cpu' - else: - self.device = device - - self.model = None - self._class_names = [] - - - self._load_model() - - logger.info(f"Initialized YOLO wrapper for {model_id} on {self.device}") - - def _load_model(self) -> None: - """Load the YOLO model with caching""" - cache_key = str(self.model_path) - - with self._cache_lock: - # Check if model is already cached - if cache_key in self._model_cache: - logger.info(f"Loading model {self.model_id} from cache") - self.model = self._model_cache[cache_key] - self._extract_class_names() - return - - # Load model - try: - from ultralytics import YOLO - - logger.info(f"Loading YOLO model from {self.model_path}") - self.model = YOLO(str(self.model_path)) - - # Move model to device - if self.device == 'cuda' and torch.cuda.is_available(): - self.model.to('cuda') - logger.info(f"Model {self.model_id} moved to GPU") - - # Cache the model - self._model_cache[cache_key] = self.model - self._extract_class_names() - - logger.info(f"Successfully loaded model {self.model_id}") - - except ImportError: - logger.error("Ultralytics YOLO not installed. Install with: pip install ultralytics") - raise - except Exception as e: - logger.error(f"Failed to load YOLO model {self.model_id}: {str(e)}", exc_info=True) - raise - - def _extract_class_names(self) -> None: - """Extract class names from the model""" - try: - if hasattr(self.model, 'names'): - self._class_names = self.model.names - elif hasattr(self.model, 'model') and hasattr(self.model.model, 'names'): - self._class_names = self.model.model.names - else: - logger.warning(f"Could not extract class names from model {self.model_id}") - self._class_names = {} - except Exception as e: - logger.error(f"Failed to extract class names: {str(e)}") - self._class_names = {} - - - def infer( - self, - image: np.ndarray, - confidence_threshold: float = 0.5, - trigger_classes: Optional[List[str]] = None, - iou_threshold: float = 0.45 - ) -> InferenceResult: - """ - Run inference on an image - - Args: - image: Input image as numpy array (BGR format) - confidence_threshold: Minimum confidence for detections - trigger_classes: List of class names to filter (None = all classes) - iou_threshold: IoU threshold for NMS - - Returns: - InferenceResult containing detections - """ - if self.model is None: - raise RuntimeError(f"Model {self.model_id} not loaded") - - try: - import time - start_time = time.time() - - # Run inference - results = self.model( - image, - conf=confidence_threshold, - iou=iou_threshold, - verbose=False - ) - - inference_time = time.time() - start_time - - # Parse results - detections = self._parse_results(results[0], trigger_classes) - - return InferenceResult( - detections=detections, - image_shape=(image.shape[0], image.shape[1]), - inference_time=inference_time, - model_id=self.model_id - ) - - except Exception as e: - logger.error(f"Inference failed for model {self.model_id}: {str(e)}", exc_info=True) - raise - - def _parse_results( - self, - result: Any, - trigger_classes: Optional[List[str]] = None - ) -> List[Detection]: - """ - Parse YOLO results into Detection objects - - Args: - result: YOLO result object - trigger_classes: Optional list of class names to filter - - Returns: - List of Detection objects - """ - detections = [] - - try: - if result.boxes is None: - return detections - - boxes = result.boxes - for i in range(len(boxes)): - # Get box coordinates - box = boxes.xyxy[i].cpu().numpy() - x1, y1, x2, y2 = box - - # Get confidence and class - conf = float(boxes.conf[i]) - cls_id = int(boxes.cls[i]) - - # Get class name - class_name = self._class_names.get(cls_id, f"class_{cls_id}") - - # Filter by trigger classes if specified - if trigger_classes and class_name not in trigger_classes: - continue - - # Get track ID if available - track_id = None - if hasattr(boxes, 'id') and boxes.id is not None: - track_id = int(boxes.id[i]) - - detection = Detection( - bbox=[float(x1), float(y1), float(x2), float(y2)], - confidence=conf, - class_id=cls_id, - class_name=class_name, - track_id=track_id - ) - detections.append(detection) - - except Exception as e: - logger.error(f"Failed to parse results: {str(e)}", exc_info=True) - - return detections - - - def track( - self, - image: np.ndarray, - confidence_threshold: float = 0.5, - trigger_classes: Optional[List[str]] = None, - persist: bool = True, - camera_id: Optional[str] = None - ) -> InferenceResult: - """ - Run detection (tracking will be handled by external tracker) - - Args: - image: Input image as numpy array (BGR format) - confidence_threshold: Minimum confidence for detections - trigger_classes: List of class names to filter - persist: Ignored - tracking handled externally - camera_id: Ignored - tracking handled externally - - Returns: - InferenceResult containing detections (no track IDs from YOLO) - """ - # Just do detection - no YOLO tracking - return self.infer(image, confidence_threshold, trigger_classes) - - def predict_classification( - self, - image: np.ndarray, - top_k: int = 1 - ) -> Dict[str, float]: - """ - Run classification on an image - - Args: - image: Input image as numpy array (BGR format) - top_k: Number of top predictions to return - - Returns: - Dictionary of class_name -> confidence scores - """ - if self.model is None: - raise RuntimeError(f"Model {self.model_id} not loaded") - - try: - # Run inference - results = self.model(image, verbose=False) - - # For classification models, extract probabilities - if hasattr(results[0], 'probs'): - probs = results[0].probs - top_indices = probs.top5[:top_k] - top_conf = probs.top5conf[:top_k].cpu().numpy() - - predictions = {} - for idx, conf in zip(top_indices, top_conf): - class_name = self._class_names.get(int(idx), f"class_{idx}") - predictions[class_name] = float(conf) - - return predictions - else: - logger.warning(f"Model {self.model_id} does not support classification") - return {} - - except Exception as e: - logger.error(f"Classification failed for model {self.model_id}: {str(e)}", exc_info=True) - raise - - def crop_detection( - self, - image: np.ndarray, - detection: Detection, - padding: int = 0 - ) -> np.ndarray: - """ - Crop image to detection bounding box - - Args: - image: Original image - detection: Detection to crop - padding: Additional padding around the box - - Returns: - Cropped image region - """ - h, w = image.shape[:2] - x1, y1, x2, y2 = detection.bbox - - # Add padding and clip to image boundaries - x1 = max(0, int(x1) - padding) - y1 = max(0, int(y1) - padding) - x2 = min(w, int(x2) + padding) - y2 = min(h, int(y2) + padding) - - return image[y1:y2, x1:x2] - - def get_class_names(self) -> Dict[int, str]: - """Get the class names dictionary""" - return self._class_names.copy() - - def get_num_classes(self) -> int: - """Get the number of classes the model can detect""" - return len(self._class_names) - - - def clear_cache(self) -> None: - """Clear the model cache""" - with self._cache_lock: - cache_key = str(self.model_path) - if cache_key in self._model_cache: - del self._model_cache[cache_key] - logger.info(f"Cleared cache for model {self.model_id}") - - @classmethod - def clear_all_cache(cls) -> None: - """Clear all cached models""" - with cls._cache_lock: - cls._model_cache.clear() - logger.info("Cleared all model cache") - - def warmup(self, image_size: Tuple[int, int] = (640, 640)) -> None: - """ - Warmup the model with a dummy inference - - Args: - image_size: Size of dummy image (height, width) - """ - try: - dummy_image = np.zeros((image_size[0], image_size[1], 3), dtype=np.uint8) - self.infer(dummy_image, confidence_threshold=0.5) - logger.info(f"Model {self.model_id} warmed up") - except Exception as e: - logger.warning(f"Failed to warmup model {self.model_id}: {str(e)}") - - -class ModelInferenceManager: - """Manages multiple YOLO models for a pipeline""" - - def __init__(self, model_dir: Path): - """ - Initialize the inference manager - - Args: - model_dir: Directory containing model files - """ - self.model_dir = model_dir - self.models: Dict[str, YOLOWrapper] = {} - self._lock = Lock() - - logger.info(f"Initialized ModelInferenceManager with model directory: {model_dir}") - - def load_model( - self, - model_id: str, - model_file: str, - device: Optional[str] = None - ) -> YOLOWrapper: - """ - Load a model for inference - - Args: - model_id: Unique identifier for the model - model_file: Filename of the model - device: Device to run on - - Returns: - YOLOWrapper instance - """ - with self._lock: - # Check if already loaded - if model_id in self.models: - logger.debug(f"Model {model_id} already loaded") - return self.models[model_id] - - # Load the model - model_path = self.model_dir / model_file - if not model_path.exists(): - raise FileNotFoundError(f"Model file not found: {model_path}") - - wrapper = YOLOWrapper(model_path, model_id, device) - self.models[model_id] = wrapper - - return wrapper - - def get_model(self, model_id: str) -> Optional[YOLOWrapper]: - """ - Get a loaded model - - Args: - model_id: Model identifier - - Returns: - YOLOWrapper instance or None if not loaded - """ - return self.models.get(model_id) - - def unload_model(self, model_id: str) -> bool: - """ - Unload a model to free memory - - Args: - model_id: Model identifier - - Returns: - True if unloaded, False if not found - """ - with self._lock: - if model_id in self.models: - self.models[model_id].clear_cache() - del self.models[model_id] - logger.info(f"Unloaded model {model_id}") - return True - return False - - def unload_all(self) -> None: - """Unload all models""" - with self._lock: - for model_id in list(self.models.keys()): - self.models[model_id].clear_cache() - self.models.clear() - logger.info("Unloaded all models") \ No newline at end of file diff --git a/core/models/manager.py b/core/models/manager.py deleted file mode 100644 index d40c48f..0000000 --- a/core/models/manager.py +++ /dev/null @@ -1,439 +0,0 @@ -""" -Model Manager Module - Handles MPTA download, extraction, and model loading -""" - -import os -import logging -import zipfile -import json -import hashlib -import requests -from pathlib import Path -from typing import Dict, Optional, Any, Set -from threading import Lock -from urllib.parse import urlparse, parse_qs - -logger = logging.getLogger(__name__) - - -class ModelManager: - """Manages MPTA model downloads, extraction, and caching""" - - def __init__(self, models_dir: str = "models"): - """ - Initialize the Model Manager - - Args: - models_dir: Base directory for storing models - """ - self.models_dir = Path(models_dir) - self.models_dir.mkdir(parents=True, exist_ok=True) - - # Track downloaded models to avoid duplicates - self._downloaded_models: Set[int] = set() - self._model_paths: Dict[int, Path] = {} - self._download_lock = Lock() - - # Scan existing models - self._scan_existing_models() - - logger.info(f"ModelManager initialized with models directory: {self.models_dir}") - logger.info(f"Found existing models: {list(self._downloaded_models)}") - - def _scan_existing_models(self) -> None: - """Scan the models directory for existing downloaded models""" - if not self.models_dir.exists(): - return - - for model_dir in self.models_dir.iterdir(): - if model_dir.is_dir() and model_dir.name.isdigit(): - model_id = int(model_dir.name) - # Check if extraction was successful by looking for pipeline.json - extracted_dirs = list(model_dir.glob("*/pipeline.json")) - if extracted_dirs: - self._downloaded_models.add(model_id) - # Store path to the extracted model directory - self._model_paths[model_id] = extracted_dirs[0].parent - logger.debug(f"Found existing model {model_id} at {extracted_dirs[0].parent}") - - def get_model_path(self, model_id: int) -> Optional[Path]: - """ - Get the path to an extracted model directory - - Args: - model_id: The model ID - - Returns: - Path to the extracted model directory or None if not found - """ - return self._model_paths.get(model_id) - - def is_model_downloaded(self, model_id: int) -> bool: - """ - Check if a model has already been downloaded and extracted - - Args: - model_id: The model ID to check - - Returns: - True if the model is already available - """ - return model_id in self._downloaded_models - - def ensure_model(self, model_id: int, model_url: str, model_name: str = None) -> Optional[Path]: - """ - Ensure a model is downloaded and extracted, downloading if necessary - - Args: - model_id: The model ID - model_url: URL to download the MPTA file from - model_name: Optional model name for logging - - Returns: - Path to the extracted model directory or None if failed - """ - # Check if already downloaded - if self.is_model_downloaded(model_id): - logger.info(f"Model {model_id} already available at {self._model_paths[model_id]}") - return self._model_paths[model_id] - - # Download and extract with lock to prevent concurrent downloads of same model - with self._download_lock: - # Double-check after acquiring lock - if self.is_model_downloaded(model_id): - return self._model_paths[model_id] - - logger.info(f"Model {model_id} not found locally, downloading from {model_url}") - - # Create model directory - model_dir = self.models_dir / str(model_id) - model_dir.mkdir(parents=True, exist_ok=True) - - # Extract filename from URL - mpta_filename = self._extract_filename_from_url(model_url, model_name, model_id) - mpta_path = model_dir / mpta_filename - - # Download MPTA file - if not self._download_mpta(model_url, mpta_path): - logger.error(f"Failed to download model {model_id}") - return None - - # Extract MPTA file - extracted_path = self._extract_mpta(mpta_path, model_dir) - if not extracted_path: - logger.error(f"Failed to extract model {model_id}") - return None - - # Mark as downloaded and store path - self._downloaded_models.add(model_id) - self._model_paths[model_id] = extracted_path - - logger.info(f"Successfully prepared model {model_id} at {extracted_path}") - return extracted_path - - def _extract_filename_from_url(self, url: str, model_name: str = None, model_id: int = None) -> str: - """ - Extract a suitable filename from the URL - - Args: - url: The URL to extract filename from - model_name: Optional model name - model_id: Optional model ID - - Returns: - A suitable filename for the MPTA file - """ - parsed = urlparse(url) - path = parsed.path - - # Try to get filename from path - if path: - filename = os.path.basename(path) - if filename and filename.endswith('.mpta'): - return filename - - # Fallback to constructed name - if model_name: - return f"{model_name}-{model_id}.mpta" - else: - return f"model-{model_id}.mpta" - - def _download_mpta(self, url: str, dest_path: Path) -> bool: - """ - Download an MPTA file from a URL - - Args: - url: URL to download from - dest_path: Destination path for the file - - Returns: - True if successful, False otherwise - """ - try: - logger.info(f"Starting download of model from {url}") - logger.debug(f"Download destination: {dest_path}") - - response = requests.get(url, stream=True, timeout=300) - if response.status_code != 200: - logger.error(f"Failed to download MPTA file (status {response.status_code})") - return False - - file_size = int(response.headers.get('content-length', 0)) - logger.info(f"Model file size: {file_size/1024/1024:.2f} MB") - - downloaded = 0 - last_log_percent = 0 - - with open(dest_path, 'wb') as f: - for chunk in response.iter_content(chunk_size=8192): - if chunk: - f.write(chunk) - downloaded += len(chunk) - - # Log progress every 10% - if file_size > 0: - percent = int(downloaded * 100 / file_size) - if percent >= last_log_percent + 10: - logger.debug(f"Download progress: {percent}%") - last_log_percent = percent - - logger.info(f"Successfully downloaded MPTA file to {dest_path}") - return True - - except requests.RequestException as e: - logger.error(f"Network error downloading MPTA: {str(e)}", exc_info=True) - # Clean up partial download - if dest_path.exists(): - dest_path.unlink() - return False - except Exception as e: - logger.error(f"Unexpected error downloading MPTA: {str(e)}", exc_info=True) - # Clean up partial download - if dest_path.exists(): - dest_path.unlink() - return False - - def _extract_mpta(self, mpta_path: Path, target_dir: Path) -> Optional[Path]: - """ - Extract an MPTA (ZIP) file to the target directory - - Args: - mpta_path: Path to the MPTA file - target_dir: Directory to extract to - - Returns: - Path to the extracted model directory containing pipeline.json, or None if failed - """ - try: - if not mpta_path.exists(): - logger.error(f"MPTA file not found: {mpta_path}") - return None - - logger.info(f"Extracting MPTA file from {mpta_path} to {target_dir}") - - with zipfile.ZipFile(mpta_path, 'r') as zip_ref: - # Get list of files - file_list = zip_ref.namelist() - logger.debug(f"Files in MPTA archive: {len(file_list)} files") - - # Extract all files - zip_ref.extractall(target_dir) - - logger.info(f"Successfully extracted MPTA file to {target_dir}") - - # Find the directory containing pipeline.json - pipeline_files = list(target_dir.glob("*/pipeline.json")) - if not pipeline_files: - # Check if pipeline.json is in root - if (target_dir / "pipeline.json").exists(): - logger.info(f"Found pipeline.json in root of {target_dir}") - return target_dir - logger.error(f"No pipeline.json found after extraction in {target_dir}") - return None - - # Return the directory containing pipeline.json - extracted_dir = pipeline_files[0].parent - logger.info(f"Extracted model to {extracted_dir}") - - # Keep the MPTA file for reference but could delete if space is a concern - # mpta_path.unlink() - # logger.debug(f"Removed MPTA file after extraction: {mpta_path}") - - return extracted_dir - - except zipfile.BadZipFile as e: - logger.error(f"Invalid ZIP/MPTA file {mpta_path}: {str(e)}", exc_info=True) - return None - except Exception as e: - logger.error(f"Failed to extract MPTA file {mpta_path}: {str(e)}", exc_info=True) - return None - - def load_pipeline_config(self, model_id: int) -> Optional[Dict[str, Any]]: - """ - Load the pipeline.json configuration for a model - - Args: - model_id: The model ID - - Returns: - The pipeline configuration dictionary or None if not found - """ - model_path = self.get_model_path(model_id) - if not model_path: - logger.error(f"Model {model_id} not found") - return None - - pipeline_path = model_path / "pipeline.json" - if not pipeline_path.exists(): - logger.error(f"pipeline.json not found for model {model_id}") - return None - - try: - with open(pipeline_path, 'r') as f: - config = json.load(f) - logger.debug(f"Loaded pipeline config for model {model_id}") - return config - except json.JSONDecodeError as e: - logger.error(f"Invalid JSON in pipeline.json for model {model_id}: {str(e)}") - return None - except Exception as e: - logger.error(f"Failed to load pipeline.json for model {model_id}: {str(e)}") - return None - - def get_model_file_path(self, model_id: int, filename: str) -> Optional[Path]: - """ - Get the full path to a model file (e.g., .pt file) - - Args: - model_id: The model ID - filename: The filename within the model directory - - Returns: - Full path to the model file or None if not found - """ - model_path = self.get_model_path(model_id) - if not model_path: - return None - - file_path = model_path / filename - if not file_path.exists(): - logger.error(f"Model file {filename} not found in model {model_id}") - return None - - return file_path - - def cleanup_model(self, model_id: int) -> bool: - """ - Remove a downloaded model to free up space - - Args: - model_id: The model ID to remove - - Returns: - True if successful, False otherwise - """ - if model_id not in self._downloaded_models: - logger.warning(f"Model {model_id} not in downloaded models") - return False - - try: - model_dir = self.models_dir / str(model_id) - if model_dir.exists(): - import shutil - shutil.rmtree(model_dir) - logger.info(f"Removed model directory: {model_dir}") - - self._downloaded_models.discard(model_id) - self._model_paths.pop(model_id, None) - return True - - except Exception as e: - logger.error(f"Failed to cleanup model {model_id}: {str(e)}") - return False - - def get_all_downloaded_models(self) -> Set[int]: - """ - Get a set of all downloaded model IDs - - Returns: - Set of model IDs that are currently downloaded - """ - return self._downloaded_models.copy() - - def get_pipeline_config(self, model_id: int) -> Optional[Any]: - """ - Get the pipeline configuration for a model. - - Args: - model_id: The model ID - - Returns: - PipelineConfig object if found, None otherwise - """ - try: - if model_id not in self._downloaded_models: - logger.warning(f"Model {model_id} not downloaded") - return None - - model_path = self._model_paths.get(model_id) - if not model_path: - logger.warning(f"Model path not found for model {model_id}") - return None - - # Import here to avoid circular imports - from .pipeline import PipelineParser - - # Load pipeline.json - pipeline_file = model_path / "pipeline.json" - if not pipeline_file.exists(): - logger.warning(f"No pipeline.json found for model {model_id}") - return None - - # Create PipelineParser object and parse the configuration - pipeline_parser = PipelineParser() - success = pipeline_parser.parse(pipeline_file) - - if success: - return pipeline_parser - else: - logger.error(f"Failed to parse pipeline.json for model {model_id}") - return None - - except Exception as e: - logger.error(f"Error getting pipeline config for model {model_id}: {e}", exc_info=True) - return None - - def get_yolo_model(self, model_id: int, model_filename: str) -> Optional[Any]: - """ - Create a YOLOWrapper instance for a specific model file. - - Args: - model_id: The model ID - model_filename: The .pt model filename - - Returns: - YOLOWrapper instance if successful, None otherwise - """ - try: - # Get the model file path - model_file_path = self.get_model_file_path(model_id, model_filename) - if not model_file_path or not model_file_path.exists(): - logger.error(f"Model file {model_filename} not found for model {model_id}") - return None - - # Import here to avoid circular imports - from .inference import YOLOWrapper - - # Create YOLOWrapper instance - yolo_model = YOLOWrapper( - model_path=model_file_path, - model_id=f"{model_id}_{model_filename}", - device=None # Auto-detect device - ) - - logger.info(f"Created YOLOWrapper for model {model_id}: {model_filename}") - return yolo_model - - except Exception as e: - logger.error(f"Error creating YOLO model for {model_id}:{model_filename}: {e}", exc_info=True) - return None \ No newline at end of file diff --git a/core/models/pipeline.py b/core/models/pipeline.py deleted file mode 100644 index de5667b..0000000 --- a/core/models/pipeline.py +++ /dev/null @@ -1,357 +0,0 @@ -""" -Pipeline Configuration Parser - Handles pipeline.json parsing and validation -""" - -import json -import logging -from pathlib import Path -from typing import Dict, List, Any, Optional, Set -from dataclasses import dataclass, field -from enum import Enum - -logger = logging.getLogger(__name__) - - -class ActionType(Enum): - """Supported action types in pipeline""" - REDIS_SAVE_IMAGE = "redis_save_image" - REDIS_PUBLISH = "redis_publish" - POSTGRESQL_UPDATE = "postgresql_update" - POSTGRESQL_UPDATE_COMBINED = "postgresql_update_combined" - POSTGRESQL_INSERT = "postgresql_insert" - - -@dataclass -class RedisConfig: - """Redis connection configuration""" - host: str - port: int = 6379 - password: Optional[str] = None - db: int = 0 - - @classmethod - def from_dict(cls, data: Dict[str, Any]) -> 'RedisConfig': - return cls( - host=data['host'], - port=data.get('port', 6379), - password=data.get('password'), - db=data.get('db', 0) - ) - - -@dataclass -class PostgreSQLConfig: - """PostgreSQL connection configuration""" - host: str - port: int - database: str - username: str - password: str - - @classmethod - def from_dict(cls, data: Dict[str, Any]) -> 'PostgreSQLConfig': - return cls( - host=data['host'], - port=data.get('port', 5432), - database=data['database'], - username=data['username'], - password=data['password'] - ) - - -@dataclass -class Action: - """Represents an action in the pipeline""" - type: ActionType - params: Dict[str, Any] = field(default_factory=dict) - - @classmethod - def from_dict(cls, data: Dict[str, Any]) -> 'Action': - action_type = ActionType(data['type']) - params = {k: v for k, v in data.items() if k != 'type'} - return cls(type=action_type, params=params) - - -@dataclass -class ModelBranch: - """Represents a branch in the pipeline with its own model""" - model_id: str - model_file: str - trigger_classes: List[str] - min_confidence: float = 0.5 - crop: bool = False - crop_class: Optional[Any] = None # Can be string or list - parallel: bool = False - actions: List[Action] = field(default_factory=list) - branches: List['ModelBranch'] = field(default_factory=list) - - @classmethod - def from_dict(cls, data: Dict[str, Any]) -> 'ModelBranch': - actions = [Action.from_dict(a) for a in data.get('actions', [])] - branches = [cls.from_dict(b) for b in data.get('branches', [])] - - return cls( - model_id=data['modelId'], - model_file=data['modelFile'], - trigger_classes=data.get('triggerClasses', []), - min_confidence=data.get('minConfidence', 0.5), - crop=data.get('crop', False), - crop_class=data.get('cropClass'), - parallel=data.get('parallel', False), - actions=actions, - branches=branches - ) - - -@dataclass -class TrackingConfig: - """Configuration for the tracking phase""" - model_id: str - model_file: str - trigger_classes: List[str] - min_confidence: float = 0.6 - - @classmethod - def from_dict(cls, data: Dict[str, Any]) -> 'TrackingConfig': - return cls( - model_id=data['modelId'], - model_file=data['modelFile'], - trigger_classes=data.get('triggerClasses', []), - min_confidence=data.get('minConfidence', 0.6) - ) - - -@dataclass -class PipelineConfig: - """Main pipeline configuration""" - model_id: str - model_file: str - trigger_classes: List[str] - min_confidence: float = 0.5 - crop: bool = False - branches: List[ModelBranch] = field(default_factory=list) - parallel_actions: List[Action] = field(default_factory=list) - - @classmethod - def from_dict(cls, data: Dict[str, Any]) -> 'PipelineConfig': - branches = [ModelBranch.from_dict(b) for b in data.get('branches', [])] - parallel_actions = [Action.from_dict(a) for a in data.get('parallelActions', [])] - - return cls( - model_id=data['modelId'], - model_file=data['modelFile'], - trigger_classes=data.get('triggerClasses', []), - min_confidence=data.get('minConfidence', 0.5), - crop=data.get('crop', False), - branches=branches, - parallel_actions=parallel_actions - ) - - -class PipelineParser: - """Parser for pipeline.json configuration files""" - - def __init__(self): - self.redis_config: Optional[RedisConfig] = None - self.postgresql_config: Optional[PostgreSQLConfig] = None - self.tracking_config: Optional[TrackingConfig] = None - self.pipeline_config: Optional[PipelineConfig] = None - self._model_dependencies: Set[str] = set() - - def parse(self, config_path: Path) -> bool: - """ - Parse a pipeline.json configuration file - - Args: - config_path: Path to the pipeline.json file - - Returns: - True if parsing was successful, False otherwise - """ - try: - if not config_path.exists(): - logger.error(f"Pipeline config not found: {config_path}") - return False - - with open(config_path, 'r') as f: - data = json.load(f) - - return self.parse_dict(data) - - except json.JSONDecodeError as e: - logger.error(f"Invalid JSON in pipeline config: {str(e)}") - return False - except Exception as e: - logger.error(f"Failed to parse pipeline config: {str(e)}", exc_info=True) - return False - - def parse_dict(self, data: Dict[str, Any]) -> bool: - """ - Parse a pipeline configuration from a dictionary - - Args: - data: The configuration dictionary - - Returns: - True if parsing was successful, False otherwise - """ - try: - # Parse Redis configuration - if 'redis' in data: - self.redis_config = RedisConfig.from_dict(data['redis']) - logger.debug(f"Parsed Redis config: {self.redis_config.host}:{self.redis_config.port}") - - # Parse PostgreSQL configuration - if 'postgresql' in data: - self.postgresql_config = PostgreSQLConfig.from_dict(data['postgresql']) - logger.debug(f"Parsed PostgreSQL config: {self.postgresql_config.host}:{self.postgresql_config.port}/{self.postgresql_config.database}") - - # Parse tracking configuration - if 'tracking' in data: - self.tracking_config = TrackingConfig.from_dict(data['tracking']) - self._model_dependencies.add(self.tracking_config.model_file) - logger.debug(f"Parsed tracking config: {self.tracking_config.model_id}") - - # Parse main pipeline configuration - if 'pipeline' in data: - self.pipeline_config = PipelineConfig.from_dict(data['pipeline']) - self._collect_model_dependencies(self.pipeline_config) - logger.debug(f"Parsed pipeline config: {self.pipeline_config.model_id}") - - logger.info(f"Successfully parsed pipeline configuration") - logger.debug(f"Model dependencies: {self._model_dependencies}") - return True - - except KeyError as e: - logger.error(f"Missing required field in pipeline config: {str(e)}") - return False - except Exception as e: - logger.error(f"Failed to parse pipeline config: {str(e)}", exc_info=True) - return False - - def _collect_model_dependencies(self, config: Any) -> None: - """ - Recursively collect all model file dependencies - - Args: - config: Pipeline or branch configuration - """ - if hasattr(config, 'model_file'): - self._model_dependencies.add(config.model_file) - - if hasattr(config, 'branches'): - for branch in config.branches: - self._collect_model_dependencies(branch) - - def get_model_dependencies(self) -> Set[str]: - """ - Get all model file dependencies from the pipeline - - Returns: - Set of model filenames required by the pipeline - """ - return self._model_dependencies.copy() - - def validate(self) -> bool: - """ - Validate the parsed configuration - - Returns: - True if configuration is valid, False otherwise - """ - if not self.pipeline_config: - logger.error("No pipeline configuration found") - return False - - # Check that all required model files are specified - if not self.pipeline_config.model_file: - logger.error("Main pipeline model file not specified") - return False - - # Validate action configurations - if not self._validate_actions(self.pipeline_config): - return False - - # Validate parallel actions - for action in self.pipeline_config.parallel_actions: - if action.type == ActionType.POSTGRESQL_UPDATE_COMBINED: - wait_for = action.params.get('waitForBranches', []) - if wait_for: - # Check that referenced branches exist - branch_ids = self._get_all_branch_ids(self.pipeline_config) - for branch_id in wait_for: - if branch_id not in branch_ids: - logger.error(f"Referenced branch '{branch_id}' in waitForBranches not found") - return False - - logger.info("Pipeline configuration validated successfully") - return True - - def _validate_actions(self, config: Any) -> bool: - """ - Validate actions in a pipeline or branch configuration - - Args: - config: Pipeline or branch configuration - - Returns: - True if valid, False otherwise - """ - if hasattr(config, 'actions'): - for action in config.actions: - # Validate Redis actions need Redis config - if action.type in [ActionType.REDIS_SAVE_IMAGE, ActionType.REDIS_PUBLISH]: - if not self.redis_config: - logger.error(f"Action {action.type} requires Redis configuration") - return False - - # Validate PostgreSQL actions need PostgreSQL config - if action.type in [ActionType.POSTGRESQL_UPDATE, ActionType.POSTGRESQL_UPDATE_COMBINED, ActionType.POSTGRESQL_INSERT]: - if not self.postgresql_config: - logger.error(f"Action {action.type} requires PostgreSQL configuration") - return False - - # Recursively validate branches - if hasattr(config, 'branches'): - for branch in config.branches: - if not self._validate_actions(branch): - return False - - return True - - def _get_all_branch_ids(self, config: Any, branch_ids: Set[str] = None) -> Set[str]: - """ - Recursively collect all branch model IDs - - Args: - config: Pipeline or branch configuration - branch_ids: Set to collect IDs into - - Returns: - Set of all branch model IDs - """ - if branch_ids is None: - branch_ids = set() - - if hasattr(config, 'branches'): - for branch in config.branches: - branch_ids.add(branch.model_id) - self._get_all_branch_ids(branch, branch_ids) - - return branch_ids - - def get_redis_config(self) -> Optional[RedisConfig]: - """Get the Redis configuration""" - return self.redis_config - - def get_postgresql_config(self) -> Optional[PostgreSQLConfig]: - """Get the PostgreSQL configuration""" - return self.postgresql_config - - def get_tracking_config(self) -> Optional[TrackingConfig]: - """Get the tracking configuration""" - return self.tracking_config - - def get_pipeline_config(self) -> Optional[PipelineConfig]: - """Get the main pipeline configuration""" - return self.pipeline_config \ No newline at end of file diff --git a/core/monitoring/__init__.py b/core/monitoring/__init__.py deleted file mode 100644 index 2ad32ed..0000000 --- a/core/monitoring/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -""" -Comprehensive health monitoring system for detector worker. -Tracks stream health, thread responsiveness, and system performance. -""" - -from .health import HealthMonitor, HealthStatus, HealthCheck -from .stream_health import StreamHealthTracker -from .thread_health import ThreadHealthMonitor -from .recovery import RecoveryManager - -__all__ = [ - 'HealthMonitor', - 'HealthStatus', - 'HealthCheck', - 'StreamHealthTracker', - 'ThreadHealthMonitor', - 'RecoveryManager' -] \ No newline at end of file diff --git a/core/monitoring/health.py b/core/monitoring/health.py deleted file mode 100644 index be094f3..0000000 --- a/core/monitoring/health.py +++ /dev/null @@ -1,456 +0,0 @@ -""" -Core health monitoring system for comprehensive stream and system health tracking. -Provides centralized health status, alerting, and recovery coordination. -""" -import time -import threading -import logging -import psutil -from typing import Dict, List, Optional, Any, Callable -from dataclasses import dataclass, field -from enum import Enum -from collections import defaultdict, deque - - -logger = logging.getLogger(__name__) - - -class HealthStatus(Enum): - """Health status levels.""" - HEALTHY = "healthy" - WARNING = "warning" - CRITICAL = "critical" - UNKNOWN = "unknown" - - -@dataclass -class HealthCheck: - """Individual health check result.""" - name: str - status: HealthStatus - message: str - timestamp: float = field(default_factory=time.time) - details: Dict[str, Any] = field(default_factory=dict) - recovery_action: Optional[str] = None - - -@dataclass -class HealthMetrics: - """Health metrics for a component.""" - component_id: str - last_update: float - frame_count: int = 0 - error_count: int = 0 - warning_count: int = 0 - restart_count: int = 0 - avg_frame_interval: float = 0.0 - last_frame_time: Optional[float] = None - thread_alive: bool = True - connection_healthy: bool = True - memory_usage_mb: float = 0.0 - cpu_usage_percent: float = 0.0 - - -class HealthMonitor: - """Comprehensive health monitoring system.""" - - def __init__(self, check_interval: float = 30.0): - """ - Initialize health monitor. - - Args: - check_interval: Interval between health checks in seconds - """ - self.check_interval = check_interval - self.running = False - self.monitor_thread = None - self._lock = threading.RLock() - - # Health data storage - self.health_checks: Dict[str, HealthCheck] = {} - self.metrics: Dict[str, HealthMetrics] = {} - self.alert_history: deque = deque(maxlen=1000) - self.recovery_actions: deque = deque(maxlen=500) - - # Thresholds (configurable) - self.thresholds = { - 'frame_stale_warning_seconds': 120, # 2 minutes - 'frame_stale_critical_seconds': 300, # 5 minutes - 'thread_unresponsive_seconds': 60, # 1 minute - 'memory_warning_mb': 500, # 500MB per stream - 'memory_critical_mb': 1000, # 1GB per stream - 'cpu_warning_percent': 80, # 80% CPU - 'cpu_critical_percent': 95, # 95% CPU - 'error_rate_warning': 0.1, # 10% error rate - 'error_rate_critical': 0.3, # 30% error rate - 'restart_threshold': 3 # Max restarts per hour - } - - # Health check functions - self.health_checkers: List[Callable[[], List[HealthCheck]]] = [] - self.recovery_callbacks: Dict[str, Callable[[str, HealthCheck], bool]] = {} - - # System monitoring - self.process = psutil.Process() - self.system_start_time = time.time() - - def start(self): - """Start health monitoring.""" - if self.running: - logger.warning("Health monitor already running") - return - - self.running = True - self.monitor_thread = threading.Thread(target=self._monitor_loop, daemon=True) - self.monitor_thread.start() - logger.info(f"Health monitor started (check interval: {self.check_interval}s)") - - def stop(self): - """Stop health monitoring.""" - self.running = False - if self.monitor_thread: - self.monitor_thread.join(timeout=5.0) - logger.info("Health monitor stopped") - - def register_health_checker(self, checker: Callable[[], List[HealthCheck]]): - """Register a health check function.""" - self.health_checkers.append(checker) - logger.debug(f"Registered health checker: {checker.__name__}") - - def register_recovery_callback(self, component: str, callback: Callable[[str, HealthCheck], bool]): - """Register a recovery callback for a component.""" - self.recovery_callbacks[component] = callback - logger.debug(f"Registered recovery callback for {component}") - - def update_metrics(self, component_id: str, **kwargs): - """Update metrics for a component.""" - with self._lock: - if component_id not in self.metrics: - self.metrics[component_id] = HealthMetrics( - component_id=component_id, - last_update=time.time() - ) - - metrics = self.metrics[component_id] - metrics.last_update = time.time() - - # Update provided metrics - for key, value in kwargs.items(): - if hasattr(metrics, key): - setattr(metrics, key, value) - - def report_frame_received(self, component_id: str): - """Report that a frame was received for a component.""" - current_time = time.time() - with self._lock: - if component_id not in self.metrics: - self.metrics[component_id] = HealthMetrics( - component_id=component_id, - last_update=current_time - ) - - metrics = self.metrics[component_id] - - # Update frame metrics - if metrics.last_frame_time: - interval = current_time - metrics.last_frame_time - # Moving average of frame intervals - if metrics.avg_frame_interval == 0: - metrics.avg_frame_interval = interval - else: - metrics.avg_frame_interval = (metrics.avg_frame_interval * 0.9) + (interval * 0.1) - - metrics.last_frame_time = current_time - metrics.frame_count += 1 - metrics.last_update = current_time - - def report_error(self, component_id: str, error_type: str = "general"): - """Report an error for a component.""" - with self._lock: - if component_id not in self.metrics: - self.metrics[component_id] = HealthMetrics( - component_id=component_id, - last_update=time.time() - ) - - self.metrics[component_id].error_count += 1 - self.metrics[component_id].last_update = time.time() - - logger.debug(f"Error reported for {component_id}: {error_type}") - - def report_warning(self, component_id: str, warning_type: str = "general"): - """Report a warning for a component.""" - with self._lock: - if component_id not in self.metrics: - self.metrics[component_id] = HealthMetrics( - component_id=component_id, - last_update=time.time() - ) - - self.metrics[component_id].warning_count += 1 - self.metrics[component_id].last_update = time.time() - - logger.debug(f"Warning reported for {component_id}: {warning_type}") - - def report_restart(self, component_id: str): - """Report that a component was restarted.""" - with self._lock: - if component_id not in self.metrics: - self.metrics[component_id] = HealthMetrics( - component_id=component_id, - last_update=time.time() - ) - - self.metrics[component_id].restart_count += 1 - self.metrics[component_id].last_update = time.time() - - # Log recovery action - recovery_action = { - 'timestamp': time.time(), - 'component': component_id, - 'action': 'restart', - 'reason': 'manual_restart' - } - - with self._lock: - self.recovery_actions.append(recovery_action) - - logger.info(f"Restart reported for {component_id}") - - def get_health_status(self, component_id: Optional[str] = None) -> Dict[str, Any]: - """Get comprehensive health status.""" - with self._lock: - if component_id: - # Get health for specific component - return self._get_component_health(component_id) - else: - # Get overall health status - return self._get_overall_health() - - def _get_component_health(self, component_id: str) -> Dict[str, Any]: - """Get health status for a specific component.""" - if component_id not in self.metrics: - return { - 'component_id': component_id, - 'status': HealthStatus.UNKNOWN.value, - 'message': 'No metrics available', - 'metrics': {} - } - - metrics = self.metrics[component_id] - current_time = time.time() - - # Determine health status - status = HealthStatus.HEALTHY - issues = [] - - # Check frame freshness - if metrics.last_frame_time: - frame_age = current_time - metrics.last_frame_time - if frame_age > self.thresholds['frame_stale_critical_seconds']: - status = HealthStatus.CRITICAL - issues.append(f"Frames stale for {frame_age:.1f}s") - elif frame_age > self.thresholds['frame_stale_warning_seconds']: - if status == HealthStatus.HEALTHY: - status = HealthStatus.WARNING - issues.append(f"Frames aging ({frame_age:.1f}s)") - - # Check error rates - if metrics.frame_count > 0: - error_rate = metrics.error_count / metrics.frame_count - if error_rate > self.thresholds['error_rate_critical']: - status = HealthStatus.CRITICAL - issues.append(f"High error rate ({error_rate:.1%})") - elif error_rate > self.thresholds['error_rate_warning']: - if status == HealthStatus.HEALTHY: - status = HealthStatus.WARNING - issues.append(f"Elevated error rate ({error_rate:.1%})") - - # Check restart frequency - restart_rate = metrics.restart_count / max(1, (current_time - self.system_start_time) / 3600) - if restart_rate > self.thresholds['restart_threshold']: - status = HealthStatus.CRITICAL - issues.append(f"Frequent restarts ({restart_rate:.1f}/hour)") - - # Check thread health - if not metrics.thread_alive: - status = HealthStatus.CRITICAL - issues.append("Thread not alive") - - # Check connection health - if not metrics.connection_healthy: - if status == HealthStatus.HEALTHY: - status = HealthStatus.WARNING - issues.append("Connection unhealthy") - - return { - 'component_id': component_id, - 'status': status.value, - 'message': '; '.join(issues) if issues else 'All checks passing', - 'metrics': { - 'frame_count': metrics.frame_count, - 'error_count': metrics.error_count, - 'warning_count': metrics.warning_count, - 'restart_count': metrics.restart_count, - 'avg_frame_interval': metrics.avg_frame_interval, - 'last_frame_age': current_time - metrics.last_frame_time if metrics.last_frame_time else None, - 'thread_alive': metrics.thread_alive, - 'connection_healthy': metrics.connection_healthy, - 'memory_usage_mb': metrics.memory_usage_mb, - 'cpu_usage_percent': metrics.cpu_usage_percent, - 'uptime_seconds': current_time - self.system_start_time - }, - 'last_update': metrics.last_update - } - - def _get_overall_health(self) -> Dict[str, Any]: - """Get overall system health status.""" - current_time = time.time() - components = {} - overall_status = HealthStatus.HEALTHY - - # Get health for all components - for component_id in self.metrics.keys(): - component_health = self._get_component_health(component_id) - components[component_id] = component_health - - # Determine overall status - component_status = HealthStatus(component_health['status']) - if component_status == HealthStatus.CRITICAL: - overall_status = HealthStatus.CRITICAL - elif component_status == HealthStatus.WARNING and overall_status == HealthStatus.HEALTHY: - overall_status = HealthStatus.WARNING - - # System metrics - try: - system_memory = self.process.memory_info() - system_cpu = self.process.cpu_percent() - except Exception: - system_memory = None - system_cpu = 0.0 - - return { - 'overall_status': overall_status.value, - 'timestamp': current_time, - 'uptime_seconds': current_time - self.system_start_time, - 'total_components': len(self.metrics), - 'components': components, - 'system_metrics': { - 'memory_mb': system_memory.rss / (1024 * 1024) if system_memory else 0, - 'cpu_percent': system_cpu, - 'process_id': self.process.pid - }, - 'recent_alerts': list(self.alert_history)[-10:], # Last 10 alerts - 'recent_recoveries': list(self.recovery_actions)[-10:] # Last 10 recovery actions - } - - def _monitor_loop(self): - """Main health monitoring loop.""" - logger.info("Health monitor loop started") - - while self.running: - try: - start_time = time.time() - - # Run all registered health checks - all_checks = [] - for checker in self.health_checkers: - try: - checks = checker() - all_checks.extend(checks) - except Exception as e: - logger.error(f"Error in health checker {checker.__name__}: {e}") - - # Process health checks and trigger recovery if needed - for check in all_checks: - self._process_health_check(check) - - # Update system metrics - self._update_system_metrics() - - # Sleep until next check - elapsed = time.time() - start_time - sleep_time = max(0, self.check_interval - elapsed) - if sleep_time > 0: - time.sleep(sleep_time) - - except Exception as e: - logger.error(f"Error in health monitor loop: {e}") - time.sleep(5.0) # Fallback sleep - - logger.info("Health monitor loop ended") - - def _process_health_check(self, check: HealthCheck): - """Process a health check result and trigger recovery if needed.""" - with self._lock: - # Store health check - self.health_checks[check.name] = check - - # Log alerts for non-healthy status - if check.status != HealthStatus.HEALTHY: - alert = { - 'timestamp': check.timestamp, - 'component': check.name, - 'status': check.status.value, - 'message': check.message, - 'details': check.details - } - self.alert_history.append(alert) - - logger.warning(f"Health alert [{check.status.value.upper()}] {check.name}: {check.message}") - - # Trigger recovery if critical and recovery action available - if check.status == HealthStatus.CRITICAL and check.recovery_action: - self._trigger_recovery(check.name, check) - - def _trigger_recovery(self, component: str, check: HealthCheck): - """Trigger recovery action for a component.""" - if component in self.recovery_callbacks: - try: - logger.info(f"Triggering recovery for {component}: {check.recovery_action}") - - success = self.recovery_callbacks[component](component, check) - - recovery_action = { - 'timestamp': time.time(), - 'component': component, - 'action': check.recovery_action, - 'reason': check.message, - 'success': success - } - - with self._lock: - self.recovery_actions.append(recovery_action) - - if success: - logger.info(f"Recovery successful for {component}") - else: - logger.error(f"Recovery failed for {component}") - - except Exception as e: - logger.error(f"Error in recovery callback for {component}: {e}") - - def _update_system_metrics(self): - """Update system-level metrics.""" - try: - # Update process metrics for all components - current_time = time.time() - - with self._lock: - for component_id, metrics in self.metrics.items(): - # Update CPU and memory if available - try: - # This is a simplified approach - in practice you'd want - # per-thread or per-component resource tracking - metrics.cpu_usage_percent = self.process.cpu_percent() / len(self.metrics) - memory_info = self.process.memory_info() - metrics.memory_usage_mb = memory_info.rss / (1024 * 1024) / len(self.metrics) - except Exception: - pass - - except Exception as e: - logger.error(f"Error updating system metrics: {e}") - - -# Global health monitor instance -health_monitor = HealthMonitor() \ No newline at end of file diff --git a/core/monitoring/recovery.py b/core/monitoring/recovery.py deleted file mode 100644 index 4ea16dc..0000000 --- a/core/monitoring/recovery.py +++ /dev/null @@ -1,385 +0,0 @@ -""" -Recovery manager for automatic handling of health issues. -Provides circuit breaker patterns, automatic restarts, and graceful degradation. -""" -import time -import logging -import threading -from typing import Dict, List, Optional, Any, Callable -from dataclasses import dataclass -from enum import Enum -from collections import defaultdict, deque - -from .health import HealthCheck, HealthStatus, health_monitor - - -logger = logging.getLogger(__name__) - - -class RecoveryAction(Enum): - """Types of recovery actions.""" - RESTART_STREAM = "restart_stream" - RESTART_THREAD = "restart_thread" - CLEAR_BUFFER = "clear_buffer" - RECONNECT = "reconnect" - THROTTLE = "throttle" - DISABLE = "disable" - - -@dataclass -class RecoveryAttempt: - """Record of a recovery attempt.""" - timestamp: float - component: str - action: RecoveryAction - reason: str - success: bool - details: Dict[str, Any] = None - - -@dataclass -class RecoveryState: - """Recovery state for a component - simplified without circuit breaker.""" - failure_count: int = 0 - success_count: int = 0 - last_failure_time: Optional[float] = None - last_success_time: Optional[float] = None - - -class RecoveryManager: - """Manages automatic recovery actions for health issues.""" - - def __init__(self): - self.recovery_handlers: Dict[str, Callable[[str, HealthCheck], bool]] = {} - self.recovery_states: Dict[str, RecoveryState] = {} - self.recovery_history: deque = deque(maxlen=1000) - self._lock = threading.RLock() - - # Configuration - simplified without circuit breaker - self.recovery_cooldown = 30 # 30 seconds between recovery attempts - self.max_attempts_per_hour = 20 # Still limit to prevent spam, but much higher - - # Track recovery attempts per component - self.recovery_attempts: Dict[str, deque] = defaultdict(lambda: deque(maxlen=50)) - - # Register with health monitor - health_monitor.register_recovery_callback("stream", self._handle_stream_recovery) - health_monitor.register_recovery_callback("thread", self._handle_thread_recovery) - health_monitor.register_recovery_callback("buffer", self._handle_buffer_recovery) - - def register_recovery_handler(self, action: RecoveryAction, handler: Callable[[str, Dict[str, Any]], bool]): - """ - Register a recovery handler for a specific action. - - Args: - action: Type of recovery action - handler: Function that performs the recovery - """ - self.recovery_handlers[action.value] = handler - logger.info(f"Registered recovery handler for {action.value}") - - def can_attempt_recovery(self, component: str) -> bool: - """ - Check if recovery can be attempted for a component. - - Args: - component: Component identifier - - Returns: - True if recovery can be attempted (always allow with minimal throttling) - """ - with self._lock: - current_time = time.time() - - # Check recovery attempt rate limiting (much more permissive) - recent_attempts = [ - attempt for attempt in self.recovery_attempts[component] - if current_time - attempt <= 3600 # Last hour - ] - - # Only block if truly excessive attempts - if len(recent_attempts) >= self.max_attempts_per_hour: - logger.warning(f"Recovery rate limit exceeded for {component} " - f"({len(recent_attempts)} attempts in last hour)") - return False - - # Check cooldown period (shorter cooldown) - if recent_attempts: - last_attempt = max(recent_attempts) - if current_time - last_attempt < self.recovery_cooldown: - logger.debug(f"Recovery cooldown active for {component} " - f"(last attempt {current_time - last_attempt:.1f}s ago)") - return False - - return True - - def attempt_recovery(self, component: str, action: RecoveryAction, reason: str, - details: Optional[Dict[str, Any]] = None) -> bool: - """ - Attempt recovery for a component. - - Args: - component: Component identifier - action: Recovery action to perform - reason: Reason for recovery - details: Additional details - - Returns: - True if recovery was successful - """ - if not self.can_attempt_recovery(component): - return False - - current_time = time.time() - - logger.info(f"Attempting recovery for {component}: {action.value} ({reason})") - - try: - # Record recovery attempt - with self._lock: - self.recovery_attempts[component].append(current_time) - - # Perform recovery action - success = self._execute_recovery_action(component, action, details or {}) - - # Record recovery result - attempt = RecoveryAttempt( - timestamp=current_time, - component=component, - action=action, - reason=reason, - success=success, - details=details - ) - - with self._lock: - self.recovery_history.append(attempt) - - # Update recovery state - self._update_recovery_state(component, success) - - if success: - logger.info(f"Recovery successful for {component}: {action.value}") - else: - logger.error(f"Recovery failed for {component}: {action.value}") - - return success - - except Exception as e: - logger.error(f"Error during recovery for {component}: {e}") - self._update_recovery_state(component, False) - return False - - def _execute_recovery_action(self, component: str, action: RecoveryAction, - details: Dict[str, Any]) -> bool: - """Execute a specific recovery action.""" - handler_key = action.value - - if handler_key not in self.recovery_handlers: - logger.error(f"No recovery handler registered for action: {handler_key}") - return False - - try: - handler = self.recovery_handlers[handler_key] - return handler(component, details) - - except Exception as e: - logger.error(f"Error executing recovery action {handler_key} for {component}: {e}") - return False - - def _update_recovery_state(self, component: str, success: bool): - """Update recovery state based on recovery result.""" - current_time = time.time() - - with self._lock: - if component not in self.recovery_states: - self.recovery_states[component] = RecoveryState() - - state = self.recovery_states[component] - - if success: - state.success_count += 1 - state.last_success_time = current_time - # Reset failure count on success - state.failure_count = max(0, state.failure_count - 1) - logger.debug(f"Recovery success for {component} (total successes: {state.success_count})") - else: - state.failure_count += 1 - state.last_failure_time = current_time - logger.debug(f"Recovery failure for {component} (total failures: {state.failure_count})") - - def _handle_stream_recovery(self, component: str, health_check: HealthCheck) -> bool: - """Handle recovery for stream-related issues.""" - if "frames" in health_check.name: - # Frame-related issue - restart stream - return self.attempt_recovery( - component, - RecoveryAction.RESTART_STREAM, - health_check.message, - health_check.details - ) - elif "connection" in health_check.name: - # Connection issue - reconnect - return self.attempt_recovery( - component, - RecoveryAction.RECONNECT, - health_check.message, - health_check.details - ) - elif "errors" in health_check.name: - # High error rate - throttle or restart - return self.attempt_recovery( - component, - RecoveryAction.THROTTLE, - health_check.message, - health_check.details - ) - else: - # Generic stream issue - restart - return self.attempt_recovery( - component, - RecoveryAction.RESTART_STREAM, - health_check.message, - health_check.details - ) - - def _handle_thread_recovery(self, component: str, health_check: HealthCheck) -> bool: - """Handle recovery for thread-related issues.""" - if "deadlock" in health_check.name: - # Deadlock detected - restart thread - return self.attempt_recovery( - component, - RecoveryAction.RESTART_THREAD, - health_check.message, - health_check.details - ) - elif "responsive" in health_check.name: - # Thread unresponsive - restart - return self.attempt_recovery( - component, - RecoveryAction.RESTART_THREAD, - health_check.message, - health_check.details - ) - else: - # Generic thread issue - restart - return self.attempt_recovery( - component, - RecoveryAction.RESTART_THREAD, - health_check.message, - health_check.details - ) - - def _handle_buffer_recovery(self, component: str, health_check: HealthCheck) -> bool: - """Handle recovery for buffer-related issues.""" - # Buffer issues - clear buffer - return self.attempt_recovery( - component, - RecoveryAction.CLEAR_BUFFER, - health_check.message, - health_check.details - ) - - def get_recovery_stats(self) -> Dict[str, Any]: - """Get recovery statistics.""" - current_time = time.time() - - with self._lock: - # Calculate stats from history - recent_recoveries = [ - attempt for attempt in self.recovery_history - if current_time - attempt.timestamp <= 3600 # Last hour - ] - - stats_by_component = defaultdict(lambda: { - 'attempts': 0, - 'successes': 0, - 'failures': 0, - 'last_attempt': None, - 'last_success': None - }) - - for attempt in recent_recoveries: - stats = stats_by_component[attempt.component] - stats['attempts'] += 1 - - if attempt.success: - stats['successes'] += 1 - if not stats['last_success'] or attempt.timestamp > stats['last_success']: - stats['last_success'] = attempt.timestamp - else: - stats['failures'] += 1 - - if not stats['last_attempt'] or attempt.timestamp > stats['last_attempt']: - stats['last_attempt'] = attempt.timestamp - - return { - 'total_recoveries_last_hour': len(recent_recoveries), - 'recovery_by_component': dict(stats_by_component), - 'recovery_states': { - component: { - 'failure_count': state.failure_count, - 'success_count': state.success_count, - 'last_failure_time': state.last_failure_time, - 'last_success_time': state.last_success_time - } - for component, state in self.recovery_states.items() - }, - 'recent_history': [ - { - 'timestamp': attempt.timestamp, - 'component': attempt.component, - 'action': attempt.action.value, - 'reason': attempt.reason, - 'success': attempt.success - } - for attempt in list(self.recovery_history)[-10:] # Last 10 attempts - ] - } - - def force_recovery(self, component: str, action: RecoveryAction, reason: str = "manual") -> bool: - """ - Force recovery for a component, bypassing rate limiting. - - Args: - component: Component identifier - action: Recovery action to perform - reason: Reason for forced recovery - - Returns: - True if recovery was successful - """ - logger.info(f"Forcing recovery for {component}: {action.value} ({reason})") - - current_time = time.time() - - try: - # Execute recovery action directly - success = self._execute_recovery_action(component, action, {}) - - # Record forced recovery - attempt = RecoveryAttempt( - timestamp=current_time, - component=component, - action=action, - reason=f"forced: {reason}", - success=success, - details={'forced': True} - ) - - with self._lock: - self.recovery_history.append(attempt) - self.recovery_attempts[component].append(current_time) - - # Update recovery state - self._update_recovery_state(component, success) - - return success - - except Exception as e: - logger.error(f"Error during forced recovery for {component}: {e}") - return False - - -# Global recovery manager instance -recovery_manager = RecoveryManager() \ No newline at end of file diff --git a/core/monitoring/stream_health.py b/core/monitoring/stream_health.py deleted file mode 100644 index 770dfe4..0000000 --- a/core/monitoring/stream_health.py +++ /dev/null @@ -1,351 +0,0 @@ -""" -Stream-specific health monitoring for video streams. -Tracks frame production, connection health, and stream-specific metrics. -""" -import time -import logging -import threading -import requests -from typing import Dict, Optional, List, Any -from collections import deque -from dataclasses import dataclass - -from .health import HealthCheck, HealthStatus, health_monitor - - -logger = logging.getLogger(__name__) - - -@dataclass -class StreamMetrics: - """Metrics for an individual stream.""" - camera_id: str - stream_type: str # 'rtsp', 'http_snapshot' - start_time: float - last_frame_time: Optional[float] = None - frame_count: int = 0 - error_count: int = 0 - reconnect_count: int = 0 - bytes_received: int = 0 - frames_per_second: float = 0.0 - connection_attempts: int = 0 - last_connection_test: Optional[float] = None - connection_healthy: bool = True - last_error: Optional[str] = None - last_error_time: Optional[float] = None - - -class StreamHealthTracker: - """Tracks health for individual video streams.""" - - def __init__(self): - self.streams: Dict[str, StreamMetrics] = {} - self._lock = threading.RLock() - - # Configuration - self.connection_test_interval = 300 # Test connection every 5 minutes - self.frame_timeout_warning = 120 # Warn if no frames for 2 minutes - self.frame_timeout_critical = 300 # Critical if no frames for 5 minutes - self.error_rate_threshold = 0.1 # 10% error rate threshold - - # Register with health monitor - health_monitor.register_health_checker(self._perform_health_checks) - - def register_stream(self, camera_id: str, stream_type: str, source_url: Optional[str] = None): - """Register a new stream for monitoring.""" - with self._lock: - if camera_id not in self.streams: - self.streams[camera_id] = StreamMetrics( - camera_id=camera_id, - stream_type=stream_type, - start_time=time.time() - ) - logger.info(f"Registered stream for monitoring: {camera_id} ({stream_type})") - - # Update health monitor metrics - health_monitor.update_metrics( - camera_id, - thread_alive=True, - connection_healthy=True - ) - - def unregister_stream(self, camera_id: str): - """Unregister a stream from monitoring.""" - with self._lock: - if camera_id in self.streams: - del self.streams[camera_id] - logger.info(f"Unregistered stream from monitoring: {camera_id}") - - def report_frame_received(self, camera_id: str, frame_size_bytes: int = 0): - """Report that a frame was received.""" - current_time = time.time() - - with self._lock: - if camera_id not in self.streams: - logger.warning(f"Frame received for unregistered stream: {camera_id}") - return - - stream = self.streams[camera_id] - - # Update frame metrics - if stream.last_frame_time: - interval = current_time - stream.last_frame_time - # Calculate FPS as moving average - if stream.frames_per_second == 0: - stream.frames_per_second = 1.0 / interval if interval > 0 else 0 - else: - new_fps = 1.0 / interval if interval > 0 else 0 - stream.frames_per_second = (stream.frames_per_second * 0.9) + (new_fps * 0.1) - - stream.last_frame_time = current_time - stream.frame_count += 1 - stream.bytes_received += frame_size_bytes - - # Report to health monitor - health_monitor.report_frame_received(camera_id) - health_monitor.update_metrics( - camera_id, - frame_count=stream.frame_count, - avg_frame_interval=1.0 / stream.frames_per_second if stream.frames_per_second > 0 else 0, - last_frame_time=current_time - ) - - def report_error(self, camera_id: str, error_message: str): - """Report an error for a stream.""" - current_time = time.time() - - with self._lock: - if camera_id not in self.streams: - logger.warning(f"Error reported for unregistered stream: {camera_id}") - return - - stream = self.streams[camera_id] - stream.error_count += 1 - stream.last_error = error_message - stream.last_error_time = current_time - - # Report to health monitor - health_monitor.report_error(camera_id, "stream_error") - health_monitor.update_metrics( - camera_id, - error_count=stream.error_count - ) - - logger.debug(f"Error reported for stream {camera_id}: {error_message}") - - def report_reconnect(self, camera_id: str, reason: str = "unknown"): - """Report that a stream reconnected.""" - current_time = time.time() - - with self._lock: - if camera_id not in self.streams: - logger.warning(f"Reconnect reported for unregistered stream: {camera_id}") - return - - stream = self.streams[camera_id] - stream.reconnect_count += 1 - - # Report to health monitor - health_monitor.report_restart(camera_id) - health_monitor.update_metrics( - camera_id, - restart_count=stream.reconnect_count - ) - - logger.info(f"Reconnect reported for stream {camera_id}: {reason}") - - def report_connection_attempt(self, camera_id: str, success: bool): - """Report a connection attempt.""" - with self._lock: - if camera_id not in self.streams: - return - - stream = self.streams[camera_id] - stream.connection_attempts += 1 - stream.connection_healthy = success - - # Report to health monitor - health_monitor.update_metrics( - camera_id, - connection_healthy=success - ) - - def test_http_connection(self, camera_id: str, url: str) -> bool: - """Test HTTP connection health for snapshot streams.""" - try: - # Quick HEAD request to test connectivity - response = requests.head(url, timeout=5, verify=False) - success = response.status_code in [200, 404] # 404 might be normal for some cameras - - self.report_connection_attempt(camera_id, success) - - if success: - logger.debug(f"Connection test passed for {camera_id}") - else: - logger.warning(f"Connection test failed for {camera_id}: HTTP {response.status_code}") - - return success - - except Exception as e: - logger.warning(f"Connection test failed for {camera_id}: {e}") - self.report_connection_attempt(camera_id, False) - return False - - def get_stream_metrics(self, camera_id: str) -> Optional[Dict[str, Any]]: - """Get metrics for a specific stream.""" - with self._lock: - if camera_id not in self.streams: - return None - - stream = self.streams[camera_id] - current_time = time.time() - - # Calculate derived metrics - uptime = current_time - stream.start_time - frame_age = current_time - stream.last_frame_time if stream.last_frame_time else None - error_rate = stream.error_count / max(1, stream.frame_count) - - return { - 'camera_id': camera_id, - 'stream_type': stream.stream_type, - 'uptime_seconds': uptime, - 'frame_count': stream.frame_count, - 'frames_per_second': stream.frames_per_second, - 'bytes_received': stream.bytes_received, - 'error_count': stream.error_count, - 'error_rate': error_rate, - 'reconnect_count': stream.reconnect_count, - 'connection_attempts': stream.connection_attempts, - 'connection_healthy': stream.connection_healthy, - 'last_frame_age_seconds': frame_age, - 'last_error': stream.last_error, - 'last_error_time': stream.last_error_time - } - - def get_all_metrics(self) -> Dict[str, Dict[str, Any]]: - """Get metrics for all streams.""" - with self._lock: - return { - camera_id: self.get_stream_metrics(camera_id) - for camera_id in self.streams.keys() - } - - def _perform_health_checks(self) -> List[HealthCheck]: - """Perform health checks for all streams.""" - checks = [] - current_time = time.time() - - with self._lock: - for camera_id, stream in self.streams.items(): - checks.extend(self._check_stream_health(camera_id, stream, current_time)) - - return checks - - def _check_stream_health(self, camera_id: str, stream: StreamMetrics, current_time: float) -> List[HealthCheck]: - """Perform health checks for a single stream.""" - checks = [] - - # Check frame freshness - if stream.last_frame_time: - frame_age = current_time - stream.last_frame_time - - if frame_age > self.frame_timeout_critical: - checks.append(HealthCheck( - name=f"stream_{camera_id}_frames", - status=HealthStatus.CRITICAL, - message=f"No frames for {frame_age:.1f}s (critical threshold: {self.frame_timeout_critical}s)", - details={ - 'frame_age': frame_age, - 'threshold': self.frame_timeout_critical, - 'last_frame_time': stream.last_frame_time - }, - recovery_action="restart_stream" - )) - elif frame_age > self.frame_timeout_warning: - checks.append(HealthCheck( - name=f"stream_{camera_id}_frames", - status=HealthStatus.WARNING, - message=f"Frames aging: {frame_age:.1f}s (warning threshold: {self.frame_timeout_warning}s)", - details={ - 'frame_age': frame_age, - 'threshold': self.frame_timeout_warning, - 'last_frame_time': stream.last_frame_time - } - )) - else: - # No frames received yet - startup_time = current_time - stream.start_time - if startup_time > 60: # Allow 1 minute for initial connection - checks.append(HealthCheck( - name=f"stream_{camera_id}_startup", - status=HealthStatus.CRITICAL, - message=f"No frames received since startup {startup_time:.1f}s ago", - details={ - 'startup_time': startup_time, - 'start_time': stream.start_time - }, - recovery_action="restart_stream" - )) - - # Check error rate - if stream.frame_count > 10: # Need sufficient samples - error_rate = stream.error_count / stream.frame_count - if error_rate > self.error_rate_threshold: - checks.append(HealthCheck( - name=f"stream_{camera_id}_errors", - status=HealthStatus.WARNING, - message=f"High error rate: {error_rate:.1%} ({stream.error_count}/{stream.frame_count})", - details={ - 'error_rate': error_rate, - 'error_count': stream.error_count, - 'frame_count': stream.frame_count, - 'last_error': stream.last_error - } - )) - - # Check connection health - if not stream.connection_healthy: - checks.append(HealthCheck( - name=f"stream_{camera_id}_connection", - status=HealthStatus.WARNING, - message="Connection unhealthy (last test failed)", - details={ - 'connection_attempts': stream.connection_attempts, - 'last_connection_test': stream.last_connection_test - } - )) - - # Check excessive reconnects - uptime_hours = (current_time - stream.start_time) / 3600 - if uptime_hours > 1 and stream.reconnect_count > 5: # More than 5 reconnects per hour - reconnect_rate = stream.reconnect_count / uptime_hours - checks.append(HealthCheck( - name=f"stream_{camera_id}_stability", - status=HealthStatus.WARNING, - message=f"Frequent reconnects: {reconnect_rate:.1f}/hour ({stream.reconnect_count} total)", - details={ - 'reconnect_rate': reconnect_rate, - 'reconnect_count': stream.reconnect_count, - 'uptime_hours': uptime_hours - } - )) - - # Check frame rate health - if stream.last_frame_time and stream.frames_per_second > 0: - expected_fps = 6.0 # Expected FPS for streams - if stream.frames_per_second < expected_fps * 0.5: # Less than 50% of expected - checks.append(HealthCheck( - name=f"stream_{camera_id}_framerate", - status=HealthStatus.WARNING, - message=f"Low frame rate: {stream.frames_per_second:.1f} fps (expected: ~{expected_fps} fps)", - details={ - 'current_fps': stream.frames_per_second, - 'expected_fps': expected_fps - } - )) - - return checks - - -# Global stream health tracker instance -stream_health_tracker = StreamHealthTracker() \ No newline at end of file diff --git a/core/monitoring/thread_health.py b/core/monitoring/thread_health.py deleted file mode 100644 index a29625b..0000000 --- a/core/monitoring/thread_health.py +++ /dev/null @@ -1,381 +0,0 @@ -""" -Thread health monitoring for detecting unresponsive and deadlocked threads. -Provides thread liveness detection and responsiveness testing. -""" -import time -import threading -import logging -import signal -import traceback -from typing import Dict, List, Optional, Any, Callable -from dataclasses import dataclass -from collections import defaultdict - -from .health import HealthCheck, HealthStatus, health_monitor - - -logger = logging.getLogger(__name__) - - -@dataclass -class ThreadInfo: - """Information about a monitored thread.""" - thread_id: int - thread_name: str - start_time: float - last_heartbeat: float - heartbeat_count: int = 0 - is_responsive: bool = True - last_activity: Optional[str] = None - stack_traces: List[str] = None - - -class ThreadHealthMonitor: - """Monitors thread health and responsiveness.""" - - def __init__(self): - self.monitored_threads: Dict[int, ThreadInfo] = {} - self.heartbeat_callbacks: Dict[int, Callable[[], bool]] = {} - self._lock = threading.RLock() - - # Configuration - self.heartbeat_timeout = 60.0 # 1 minute without heartbeat = unresponsive - self.responsiveness_test_interval = 30.0 # Test responsiveness every 30 seconds - self.stack_trace_count = 5 # Keep last 5 stack traces for analysis - - # Register with health monitor - health_monitor.register_health_checker(self._perform_health_checks) - - # Enable periodic responsiveness testing - self.test_thread = threading.Thread(target=self._responsiveness_test_loop, daemon=True) - self.test_thread.start() - - def register_thread(self, thread: threading.Thread, heartbeat_callback: Optional[Callable[[], bool]] = None): - """ - Register a thread for monitoring. - - Args: - thread: Thread to monitor - heartbeat_callback: Optional callback to test thread responsiveness - """ - with self._lock: - thread_info = ThreadInfo( - thread_id=thread.ident, - thread_name=thread.name, - start_time=time.time(), - last_heartbeat=time.time() - ) - - self.monitored_threads[thread.ident] = thread_info - - if heartbeat_callback: - self.heartbeat_callbacks[thread.ident] = heartbeat_callback - - logger.info(f"Registered thread for monitoring: {thread.name} (ID: {thread.ident})") - - def unregister_thread(self, thread_id: int): - """Unregister a thread from monitoring.""" - with self._lock: - if thread_id in self.monitored_threads: - thread_name = self.monitored_threads[thread_id].thread_name - del self.monitored_threads[thread_id] - - if thread_id in self.heartbeat_callbacks: - del self.heartbeat_callbacks[thread_id] - - logger.info(f"Unregistered thread from monitoring: {thread_name} (ID: {thread_id})") - - def heartbeat(self, thread_id: Optional[int] = None, activity: Optional[str] = None): - """ - Report thread heartbeat. - - Args: - thread_id: Thread ID (uses current thread if None) - activity: Description of current activity - """ - if thread_id is None: - thread_id = threading.current_thread().ident - - current_time = time.time() - - with self._lock: - if thread_id in self.monitored_threads: - thread_info = self.monitored_threads[thread_id] - thread_info.last_heartbeat = current_time - thread_info.heartbeat_count += 1 - thread_info.is_responsive = True - - if activity: - thread_info.last_activity = activity - - # Report to health monitor - health_monitor.update_metrics( - f"thread_{thread_info.thread_name}", - thread_alive=True, - last_frame_time=current_time - ) - - def get_thread_info(self, thread_id: int) -> Optional[Dict[str, Any]]: - """Get information about a monitored thread.""" - with self._lock: - if thread_id not in self.monitored_threads: - return None - - thread_info = self.monitored_threads[thread_id] - current_time = time.time() - - return { - 'thread_id': thread_id, - 'thread_name': thread_info.thread_name, - 'uptime_seconds': current_time - thread_info.start_time, - 'last_heartbeat_age': current_time - thread_info.last_heartbeat, - 'heartbeat_count': thread_info.heartbeat_count, - 'is_responsive': thread_info.is_responsive, - 'last_activity': thread_info.last_activity, - 'stack_traces': thread_info.stack_traces or [] - } - - def get_all_thread_info(self) -> Dict[int, Dict[str, Any]]: - """Get information about all monitored threads.""" - with self._lock: - return { - thread_id: self.get_thread_info(thread_id) - for thread_id in self.monitored_threads.keys() - } - - def test_thread_responsiveness(self, thread_id: int) -> bool: - """ - Test if a thread is responsive by calling its heartbeat callback. - - Args: - thread_id: ID of thread to test - - Returns: - True if thread responds within timeout - """ - if thread_id not in self.heartbeat_callbacks: - return True # Can't test if no callback provided - - try: - # Call the heartbeat callback with a timeout - callback = self.heartbeat_callbacks[thread_id] - - # This is a simple approach - in practice you might want to use - # threading.Timer or asyncio for more sophisticated timeout handling - start_time = time.time() - result = callback() - response_time = time.time() - start_time - - with self._lock: - if thread_id in self.monitored_threads: - self.monitored_threads[thread_id].is_responsive = result - - if response_time > 5.0: # Slow response - logger.warning(f"Thread {thread_id} slow response: {response_time:.1f}s") - - return result - - except Exception as e: - logger.error(f"Error testing thread {thread_id} responsiveness: {e}") - with self._lock: - if thread_id in self.monitored_threads: - self.monitored_threads[thread_id].is_responsive = False - return False - - def capture_stack_trace(self, thread_id: int) -> Optional[str]: - """ - Capture stack trace for a thread. - - Args: - thread_id: ID of thread to capture - - Returns: - Stack trace string or None if not available - """ - try: - # Get all frames for all threads - frames = dict(threading._current_frames()) - - if thread_id not in frames: - return None - - # Format stack trace - frame = frames[thread_id] - stack_trace = ''.join(traceback.format_stack(frame)) - - # Store in thread info - with self._lock: - if thread_id in self.monitored_threads: - thread_info = self.monitored_threads[thread_id] - if thread_info.stack_traces is None: - thread_info.stack_traces = [] - - thread_info.stack_traces.append(f"{time.time()}: {stack_trace}") - - # Keep only last N stack traces - if len(thread_info.stack_traces) > self.stack_trace_count: - thread_info.stack_traces = thread_info.stack_traces[-self.stack_trace_count:] - - return stack_trace - - except Exception as e: - logger.error(f"Error capturing stack trace for thread {thread_id}: {e}") - return None - - def detect_deadlocks(self) -> List[Dict[str, Any]]: - """ - Attempt to detect potential deadlocks by analyzing thread states. - - Returns: - List of potential deadlock scenarios - """ - deadlocks = [] - current_time = time.time() - - with self._lock: - # Look for threads that haven't had heartbeats for a long time - # and are supposedly alive - for thread_id, thread_info in self.monitored_threads.items(): - heartbeat_age = current_time - thread_info.last_heartbeat - - if heartbeat_age > self.heartbeat_timeout * 2: # Double the timeout - # Check if thread still exists - thread_exists = any( - t.ident == thread_id and t.is_alive() - for t in threading.enumerate() - ) - - if thread_exists: - # Thread exists but not responding - potential deadlock - stack_trace = self.capture_stack_trace(thread_id) - - deadlock_info = { - 'thread_id': thread_id, - 'thread_name': thread_info.thread_name, - 'heartbeat_age': heartbeat_age, - 'last_activity': thread_info.last_activity, - 'stack_trace': stack_trace, - 'detection_time': current_time - } - - deadlocks.append(deadlock_info) - logger.warning(f"Potential deadlock detected in thread {thread_info.thread_name}") - - return deadlocks - - def _responsiveness_test_loop(self): - """Background loop to test thread responsiveness.""" - logger.info("Thread responsiveness testing started") - - while True: - try: - time.sleep(self.responsiveness_test_interval) - - with self._lock: - thread_ids = list(self.monitored_threads.keys()) - - for thread_id in thread_ids: - try: - self.test_thread_responsiveness(thread_id) - except Exception as e: - logger.error(f"Error testing thread {thread_id}: {e}") - - except Exception as e: - logger.error(f"Error in responsiveness test loop: {e}") - time.sleep(10.0) # Fallback sleep - - def _perform_health_checks(self) -> List[HealthCheck]: - """Perform health checks for all monitored threads.""" - checks = [] - current_time = time.time() - - with self._lock: - for thread_id, thread_info in self.monitored_threads.items(): - checks.extend(self._check_thread_health(thread_id, thread_info, current_time)) - - # Check for deadlocks - deadlocks = self.detect_deadlocks() - for deadlock in deadlocks: - checks.append(HealthCheck( - name=f"deadlock_detection_{deadlock['thread_id']}", - status=HealthStatus.CRITICAL, - message=f"Potential deadlock in thread {deadlock['thread_name']} " - f"(unresponsive for {deadlock['heartbeat_age']:.1f}s)", - details=deadlock, - recovery_action="restart_thread" - )) - - return checks - - def _check_thread_health(self, thread_id: int, thread_info: ThreadInfo, current_time: float) -> List[HealthCheck]: - """Perform health checks for a single thread.""" - checks = [] - - # Check if thread still exists - thread_exists = any( - t.ident == thread_id and t.is_alive() - for t in threading.enumerate() - ) - - if not thread_exists: - checks.append(HealthCheck( - name=f"thread_{thread_info.thread_name}_alive", - status=HealthStatus.CRITICAL, - message=f"Thread {thread_info.thread_name} is no longer alive", - details={ - 'thread_id': thread_id, - 'uptime': current_time - thread_info.start_time, - 'last_heartbeat': thread_info.last_heartbeat - }, - recovery_action="restart_thread" - )) - return checks - - # Check heartbeat freshness - heartbeat_age = current_time - thread_info.last_heartbeat - - if heartbeat_age > self.heartbeat_timeout: - checks.append(HealthCheck( - name=f"thread_{thread_info.thread_name}_responsive", - status=HealthStatus.CRITICAL, - message=f"Thread {thread_info.thread_name} unresponsive for {heartbeat_age:.1f}s", - details={ - 'thread_id': thread_id, - 'heartbeat_age': heartbeat_age, - 'heartbeat_count': thread_info.heartbeat_count, - 'last_activity': thread_info.last_activity, - 'is_responsive': thread_info.is_responsive - }, - recovery_action="restart_thread" - )) - elif heartbeat_age > self.heartbeat_timeout * 0.5: # Warning at 50% of timeout - checks.append(HealthCheck( - name=f"thread_{thread_info.thread_name}_responsive", - status=HealthStatus.WARNING, - message=f"Thread {thread_info.thread_name} slow heartbeat: {heartbeat_age:.1f}s", - details={ - 'thread_id': thread_id, - 'heartbeat_age': heartbeat_age, - 'heartbeat_count': thread_info.heartbeat_count, - 'last_activity': thread_info.last_activity, - 'is_responsive': thread_info.is_responsive - } - )) - - # Check responsiveness test results - if not thread_info.is_responsive: - checks.append(HealthCheck( - name=f"thread_{thread_info.thread_name}_callback", - status=HealthStatus.WARNING, - message=f"Thread {thread_info.thread_name} failed responsiveness test", - details={ - 'thread_id': thread_id, - 'last_activity': thread_info.last_activity - } - )) - - return checks - - -# Global thread health monitor instance -thread_health_monitor = ThreadHealthMonitor() \ No newline at end of file diff --git a/core/storage/__init__.py b/core/storage/__init__.py deleted file mode 100644 index 973837a..0000000 --- a/core/storage/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -""" -Storage module for the Python Detector Worker. - -This module provides Redis and PostgreSQL operations for data persistence -and caching in the detection pipeline. -""" -from .redis import RedisManager -from .database import DatabaseManager - -__all__ = ['RedisManager', 'DatabaseManager'] \ No newline at end of file diff --git a/core/storage/database.py b/core/storage/database.py deleted file mode 100644 index a90df97..0000000 --- a/core/storage/database.py +++ /dev/null @@ -1,357 +0,0 @@ -""" -Database Operations Module. -Handles PostgreSQL operations for the detection pipeline. -""" -import psycopg2 -import psycopg2.extras -from typing import Optional, Dict, Any -import logging -import uuid - -logger = logging.getLogger(__name__) - - -class DatabaseManager: - """ - Manages PostgreSQL connections and operations for the detection pipeline. - Handles database operations and schema management. - """ - - def __init__(self, config: Dict[str, Any]): - """ - Initialize database manager with configuration. - - Args: - config: Database configuration dictionary - """ - self.config = config - self.connection: Optional[psycopg2.extensions.connection] = None - - def connect(self) -> bool: - """ - Connect to PostgreSQL database. - - Returns: - True if successful, False otherwise - """ - try: - self.connection = psycopg2.connect( - host=self.config['host'], - port=self.config['port'], - database=self.config['database'], - user=self.config['username'], - password=self.config['password'] - ) - logger.info("PostgreSQL connection established successfully") - return True - except Exception as e: - logger.error(f"Failed to connect to PostgreSQL: {e}") - return False - - def disconnect(self): - """Disconnect from PostgreSQL database.""" - if self.connection: - self.connection.close() - self.connection = None - logger.info("PostgreSQL connection closed") - - def is_connected(self) -> bool: - """ - Check if database connection is active. - - Returns: - True if connected, False otherwise - """ - try: - if self.connection and not self.connection.closed: - cur = self.connection.cursor() - cur.execute("SELECT 1") - cur.fetchone() - cur.close() - return True - except: - pass - return False - - def update_car_info(self, session_id: str, brand: str, model: str, body_type: str) -> bool: - """ - Update car information in the database. - - Args: - session_id: Session identifier - brand: Car brand - model: Car model - body_type: Car body type - - Returns: - True if successful, False otherwise - """ - if not self.is_connected(): - if not self.connect(): - return False - - try: - cur = self.connection.cursor() - query = """ - INSERT INTO car_frontal_info (session_id, car_brand, car_model, car_body_type, updated_at) - VALUES (%s, %s, %s, %s, NOW()) - ON CONFLICT (session_id) - DO UPDATE SET - car_brand = EXCLUDED.car_brand, - car_model = EXCLUDED.car_model, - car_body_type = EXCLUDED.car_body_type, - updated_at = NOW() - """ - cur.execute(query, (session_id, brand, model, body_type)) - self.connection.commit() - cur.close() - logger.info(f"Updated car info for session {session_id}: {brand} {model} ({body_type})") - return True - except Exception as e: - logger.error(f"Failed to update car info: {e}") - if self.connection: - self.connection.rollback() - return False - - def execute_update(self, table: str, key_field: str, key_value: str, fields: Dict[str, str]) -> bool: - """ - Execute a dynamic update query on the database. - - Args: - table: Table name - key_field: Primary key field name - key_value: Primary key value - fields: Dictionary of fields to update - - Returns: - True if successful, False otherwise - """ - if not self.is_connected(): - if not self.connect(): - return False - - try: - cur = self.connection.cursor() - - # Build the UPDATE query dynamically - set_clauses = [] - values = [] - - for field, value in fields.items(): - if value == "NOW()": - set_clauses.append(f"{field} = NOW()") - else: - set_clauses.append(f"{field} = %s") - values.append(value) - - # Add schema prefix if table doesn't already have it - full_table_name = table if '.' in table else f"gas_station_1.{table}" - - query = f""" - INSERT INTO {full_table_name} ({key_field}, {', '.join(fields.keys())}) - VALUES (%s, {', '.join(['%s'] * len(fields))}) - ON CONFLICT ({key_field}) - DO UPDATE SET {', '.join(set_clauses)} - """ - - # Add key_value to the beginning of values list - all_values = [key_value] + list(fields.values()) + values - - cur.execute(query, all_values) - self.connection.commit() - cur.close() - logger.info(f"Updated {table} for {key_field}={key_value}") - return True - except Exception as e: - logger.error(f"Failed to execute update on {table}: {e}") - if self.connection: - self.connection.rollback() - return False - - def create_car_frontal_info_table(self) -> bool: - """ - Create the car_frontal_info table in gas_station_1 schema if it doesn't exist. - - Returns: - True if successful, False otherwise - """ - if not self.is_connected(): - if not self.connect(): - return False - - try: - # Since the database already exists, just verify connection - cur = self.connection.cursor() - - # Simple verification that the table exists - cur.execute(""" - SELECT EXISTS ( - SELECT FROM information_schema.tables - WHERE table_schema = 'gas_station_1' - AND table_name = 'car_frontal_info' - ) - """) - - table_exists = cur.fetchone()[0] - cur.close() - - if table_exists: - logger.info("Verified car_frontal_info table exists") - return True - else: - logger.error("car_frontal_info table does not exist in the database") - return False - - except Exception as e: - logger.error(f"Failed to create car_frontal_info table: {e}") - if self.connection: - self.connection.rollback() - return False - - def insert_initial_detection(self, display_id: str, captured_timestamp: str, session_id: str = None) -> str: - """ - Insert initial detection record and return the session_id. - - Args: - display_id: Display identifier - captured_timestamp: Timestamp of the detection - session_id: Optional session ID, generates one if not provided - - Returns: - Session ID string or None on error - """ - if not self.is_connected(): - if not self.connect(): - return None - - # Generate session_id if not provided - if not session_id: - session_id = str(uuid.uuid4()) - - try: - # Ensure table exists - if not self.create_car_frontal_info_table(): - logger.error("Failed to create/verify table before insertion") - return None - - cur = self.connection.cursor() - insert_query = """ - INSERT INTO gas_station_1.car_frontal_info - (display_id, captured_timestamp, session_id, license_character, license_type, car_brand, car_model, car_body_type) - VALUES (%s, %s, %s, NULL, 'No model available', NULL, NULL, NULL) - ON CONFLICT (session_id) DO NOTHING - """ - - cur.execute(insert_query, (display_id, captured_timestamp, session_id)) - self.connection.commit() - cur.close() - logger.info(f"Inserted initial detection record with session_id: {session_id}") - return session_id - - except Exception as e: - logger.error(f"Failed to insert initial detection record: {e}") - if self.connection: - self.connection.rollback() - return None - - def get_session_info(self, session_id: str) -> Optional[Dict[str, Any]]: - """ - Get session information from the database. - - Args: - session_id: Session identifier - - Returns: - Dictionary with session data or None if not found - """ - if not self.is_connected(): - if not self.connect(): - return None - - try: - cur = self.connection.cursor(cursor_factory=psycopg2.extras.RealDictCursor) - query = "SELECT * FROM gas_station_1.car_frontal_info WHERE session_id = %s" - cur.execute(query, (session_id,)) - result = cur.fetchone() - cur.close() - - if result: - return dict(result) - else: - logger.debug(f"No session info found for session_id: {session_id}") - return None - - except Exception as e: - logger.error(f"Failed to get session info: {e}") - return None - - def delete_session(self, session_id: str) -> bool: - """ - Delete session record from the database. - - Args: - session_id: Session identifier - - Returns: - True if successful, False otherwise - """ - if not self.is_connected(): - if not self.connect(): - return False - - try: - cur = self.connection.cursor() - query = "DELETE FROM gas_station_1.car_frontal_info WHERE session_id = %s" - cur.execute(query, (session_id,)) - rows_affected = cur.rowcount - self.connection.commit() - cur.close() - - if rows_affected > 0: - logger.info(f"Deleted session record: {session_id}") - return True - else: - logger.warning(f"No session record found to delete: {session_id}") - return False - - except Exception as e: - logger.error(f"Failed to delete session: {e}") - if self.connection: - self.connection.rollback() - return False - - def get_statistics(self) -> Dict[str, Any]: - """ - Get database statistics. - - Returns: - Dictionary with database statistics - """ - stats = { - 'connected': self.is_connected(), - 'host': self.config.get('host', 'unknown'), - 'port': self.config.get('port', 'unknown'), - 'database': self.config.get('database', 'unknown') - } - - if self.is_connected(): - try: - cur = self.connection.cursor() - - # Get table record count - cur.execute("SELECT COUNT(*) FROM gas_station_1.car_frontal_info") - stats['total_records'] = cur.fetchone()[0] - - # Get recent records count (last hour) - cur.execute(""" - SELECT COUNT(*) FROM gas_station_1.car_frontal_info - WHERE created_at > NOW() - INTERVAL '1 hour' - """) - stats['recent_records'] = cur.fetchone()[0] - - cur.close() - except Exception as e: - logger.warning(f"Failed to get database statistics: {e}") - stats['error'] = str(e) - - return stats \ No newline at end of file diff --git a/core/storage/license_plate.py b/core/storage/license_plate.py deleted file mode 100644 index 19cbf73..0000000 --- a/core/storage/license_plate.py +++ /dev/null @@ -1,300 +0,0 @@ -""" -License Plate Manager Module. -Handles Redis subscription to license plate results from LPR service. -""" -import logging -import json -import asyncio -from typing import Dict, Optional, Any, Callable -import redis.asyncio as redis - -logger = logging.getLogger(__name__) - - -class LicensePlateManager: - """ - Manages license plate result subscription from Redis channel. - Subscribes to 'license_results' channel for license plate data from LPR service. - """ - - def __init__(self, redis_config: Dict[str, Any]): - """ - Initialize license plate manager with Redis configuration. - - Args: - redis_config: Redis configuration dictionary - """ - self.config = redis_config - self.redis_client: Optional[redis.Redis] = None - self.pubsub = None - self.subscription_task = None - self.callback = None - - # Connection parameters - self.host = redis_config.get('host', 'localhost') - self.port = redis_config.get('port', 6379) - self.password = redis_config.get('password') - self.db = redis_config.get('db', 0) - - # License plate data cache - store recent results by session_id - self.license_plate_cache: Dict[str, Dict[str, Any]] = {} - self.cache_ttl = 300 # 5 minutes TTL for cached results - - logger.info(f"LicensePlateManager initialized for {self.host}:{self.port}") - - async def initialize(self, callback: Optional[Callable] = None) -> bool: - """ - Initialize Redis connection and start subscription to license_results channel. - - Args: - callback: Optional callback function for processing license plate results - - Returns: - True if successful, False otherwise - """ - try: - # Create Redis connection - self.redis_client = redis.Redis( - host=self.host, - port=self.port, - password=self.password, - db=self.db, - decode_responses=True - ) - - # Test connection - await self.redis_client.ping() - logger.info(f"Connected to Redis for license plate subscription") - - # Set callback - self.callback = callback - - # Start subscription - await self._start_subscription() - - return True - - except Exception as e: - logger.error(f"Failed to initialize license plate manager: {e}", exc_info=True) - return False - - async def _start_subscription(self): - """Start Redis subscription to license_results channel.""" - try: - if not self.redis_client: - logger.error("Redis client not initialized") - return - - # Create pubsub and subscribe - self.pubsub = self.redis_client.pubsub() - await self.pubsub.subscribe('license_results') - - logger.info("Subscribed to Redis channel: license_results") - - # Start listening task - self.subscription_task = asyncio.create_task(self._listen_for_messages()) - - except Exception as e: - logger.error(f"Error starting license plate subscription: {e}", exc_info=True) - - async def _listen_for_messages(self): - """Listen for messages on the license_results channel.""" - listen_generator = None - try: - if not self.pubsub: - return - - listen_generator = self.pubsub.listen() - async for message in listen_generator: - if message['type'] == 'message': - try: - # Log the raw message from Redis channel - logger.info(f"[LICENSE PLATE RAW] Received from 'license_results' channel: {message['data']}") - - # Parse the license plate result message - data = json.loads(message['data']) - logger.info(f"[LICENSE PLATE PARSED] Parsed JSON data: {data}") - await self._process_license_plate_result(data) - except json.JSONDecodeError as e: - logger.error(f"[LICENSE PLATE ERROR] Invalid JSON in license plate message: {e}") - logger.error(f"[LICENSE PLATE ERROR] Raw message was: {message['data']}") - except Exception as e: - logger.error(f"Error processing license plate message: {e}", exc_info=True) - - except asyncio.CancelledError: - logger.info("License plate subscription task cancelled") - # Don't try to close generator here - let it be handled by the context - # The async generator will be properly closed by the cancellation mechanism - raise # Re-raise to maintain proper cancellation semantics - except Exception as e: - logger.error(f"Error in license plate message listener: {e}", exc_info=True) - # Only attempt cleanup if it's not a cancellation - finally: - # Safe cleanup of async generator - if listen_generator is not None: - try: - # Check if we can safely close without conflicting with ongoing operations - if hasattr(listen_generator, 'aclose') and not asyncio.current_task().cancelled(): - await listen_generator.aclose() - except (RuntimeError, AttributeError) as e: - # Generator is already closing or in invalid state - safe to ignore - logger.debug(f"Generator cleanup skipped (safe): {e}") - except Exception as e: - logger.debug(f"Generator cleanup error (non-critical): {e}") - - async def _process_license_plate_result(self, data: Dict[str, Any]): - """ - Process incoming license plate result from LPR service. - - Expected message format (from actual LPR service): - { - "session_id": "511", - "license_character": "ข3184" - } - or - { - "session_id": "508", - "display_id": "test3", - "license_plate_text": "ABC-123", - "confidence": 0.95, - "timestamp": "2025-09-24T21:10:00Z" - } - - Args: - data: License plate result data - """ - try: - session_id = data.get('session_id') - if not session_id: - logger.warning("License plate result missing session_id") - return - - # Handle different message formats - # Format 1: {"session_id": "511", "license_character": "ข3184"} - # Format 2: {"session_id": "508", "license_plate_text": "ABC-123", "confidence": 0.95, ...} - license_plate_text = data.get('license_plate_text') or data.get('license_character') - confidence = data.get('confidence', 1.0) # Default confidence for LPR service results - display_id = data.get('display_id', '') - timestamp = data.get('timestamp', '') - - logger.info(f"[LICENSE PLATE] Received result for session {session_id}: " - f"text='{license_plate_text}', confidence={confidence:.3f}") - - # Store in cache - self.license_plate_cache[session_id] = { - 'license_plate_text': license_plate_text, - 'confidence': confidence, - 'display_id': display_id, - 'timestamp': timestamp, - 'received_at': asyncio.get_event_loop().time() - } - - # Call callback if provided - if self.callback: - await self.callback(session_id, { - 'license_plate_text': license_plate_text, - 'confidence': confidence, - 'display_id': display_id, - 'timestamp': timestamp - }) - - except Exception as e: - logger.error(f"Error processing license plate result: {e}", exc_info=True) - - def get_license_plate_result(self, session_id: str) -> Optional[Dict[str, Any]]: - """ - Get cached license plate result for a session. - - Args: - session_id: Session identifier - - Returns: - License plate result dictionary or None if not found - """ - if session_id not in self.license_plate_cache: - return None - - result = self.license_plate_cache[session_id] - - # Check TTL - current_time = asyncio.get_event_loop().time() - if current_time - result.get('received_at', 0) > self.cache_ttl: - # Expired, remove from cache - del self.license_plate_cache[session_id] - return None - - return { - 'license_plate_text': result.get('license_plate_text'), - 'confidence': result.get('confidence'), - 'display_id': result.get('display_id'), - 'timestamp': result.get('timestamp') - } - - def cleanup_expired_results(self): - """Remove expired license plate results from cache.""" - try: - current_time = asyncio.get_event_loop().time() - expired_sessions = [] - - for session_id, result in self.license_plate_cache.items(): - if current_time - result.get('received_at', 0) > self.cache_ttl: - expired_sessions.append(session_id) - - for session_id in expired_sessions: - del self.license_plate_cache[session_id] - logger.debug(f"Removed expired license plate result for session {session_id}") - - except Exception as e: - logger.error(f"Error cleaning up expired license plate results: {e}", exc_info=True) - - async def close(self): - """Close Redis connection and cleanup resources.""" - try: - # Cancel subscription task first - if self.subscription_task and not self.subscription_task.done(): - self.subscription_task.cancel() - try: - await self.subscription_task - except asyncio.CancelledError: - logger.debug("License plate subscription task cancelled successfully") - except Exception as e: - logger.warning(f"Error waiting for subscription task cancellation: {e}") - - # Close pubsub connection properly - if self.pubsub: - try: - # First unsubscribe from channels - await self.pubsub.unsubscribe('license_results') - # Then close the pubsub connection - await self.pubsub.aclose() - except Exception as e: - logger.warning(f"Error closing pubsub connection: {e}") - finally: - self.pubsub = None - - # Close Redis connection - if self.redis_client: - try: - await self.redis_client.aclose() - except Exception as e: - logger.warning(f"Error closing Redis connection: {e}") - finally: - self.redis_client = None - - # Clear cache - self.license_plate_cache.clear() - - logger.info("License plate manager closed successfully") - - except Exception as e: - logger.error(f"Error closing license plate manager: {e}", exc_info=True) - - def get_statistics(self) -> Dict[str, Any]: - """Get license plate manager statistics.""" - return { - 'cached_results': len(self.license_plate_cache), - 'connected': self.redis_client is not None, - 'subscribed': self.pubsub is not None, - 'host': self.host, - 'port': self.port - } \ No newline at end of file diff --git a/core/storage/redis.py b/core/storage/redis.py deleted file mode 100644 index 6672a1b..0000000 --- a/core/storage/redis.py +++ /dev/null @@ -1,478 +0,0 @@ -""" -Redis Operations Module. -Handles Redis connections, image storage, and pub/sub messaging. -""" -import logging -import json -import time -from typing import Optional, Dict, Any, Union -import asyncio -import cv2 -import numpy as np -import redis.asyncio as redis -from redis.exceptions import ConnectionError, TimeoutError - -logger = logging.getLogger(__name__) - - -class RedisManager: - """ - Manages Redis connections and operations for the detection pipeline. - Handles image storage with region cropping and pub/sub messaging. - """ - - def __init__(self, redis_config: Dict[str, Any]): - """ - Initialize Redis manager with configuration. - - Args: - redis_config: Redis configuration dictionary - """ - self.config = redis_config - self.redis_client: Optional[redis.Redis] = None - - # Connection parameters - self.host = redis_config.get('host', 'localhost') - self.port = redis_config.get('port', 6379) - self.password = redis_config.get('password') - self.db = redis_config.get('db', 0) - self.decode_responses = redis_config.get('decode_responses', True) - - # Connection pool settings - self.max_connections = redis_config.get('max_connections', 10) - self.socket_timeout = redis_config.get('socket_timeout', 5) - self.socket_connect_timeout = redis_config.get('socket_connect_timeout', 5) - self.health_check_interval = redis_config.get('health_check_interval', 30) - - # Statistics - self.stats = { - 'images_stored': 0, - 'messages_published': 0, - 'connection_errors': 0, - 'operations_successful': 0, - 'operations_failed': 0 - } - - logger.info(f"RedisManager initialized for {self.host}:{self.port}") - - async def initialize(self) -> bool: - """ - Initialize Redis connection and test connectivity. - - Returns: - True if successful, False otherwise - """ - try: - # Validate configuration - if not self._validate_config(): - return False - - # Create Redis connection - self.redis_client = redis.Redis( - host=self.host, - port=self.port, - password=self.password, - db=self.db, - decode_responses=self.decode_responses, - max_connections=self.max_connections, - socket_timeout=self.socket_timeout, - socket_connect_timeout=self.socket_connect_timeout, - health_check_interval=self.health_check_interval - ) - - # Test connection - await self.redis_client.ping() - logger.info(f"Successfully connected to Redis at {self.host}:{self.port}") - return True - - except ConnectionError as e: - logger.error(f"Failed to connect to Redis: {e}") - self.stats['connection_errors'] += 1 - return False - except Exception as e: - logger.error(f"Error initializing Redis connection: {e}", exc_info=True) - self.stats['connection_errors'] += 1 - return False - - def _validate_config(self) -> bool: - """ - Validate Redis configuration parameters. - - Returns: - True if valid, False otherwise - """ - required_fields = ['host', 'port'] - for field in required_fields: - if field not in self.config: - logger.error(f"Missing required Redis config field: {field}") - return False - - if not isinstance(self.port, int) or self.port <= 0: - logger.error(f"Invalid Redis port: {self.port}") - return False - - return True - - async def is_connected(self) -> bool: - """ - Check if Redis connection is active. - - Returns: - True if connected, False otherwise - """ - try: - if self.redis_client: - await self.redis_client.ping() - return True - except Exception: - pass - return False - - async def save_image(self, - key: str, - image: np.ndarray, - expire_seconds: Optional[int] = None, - image_format: str = 'jpeg', - quality: int = 90) -> bool: - """ - Save image to Redis with optional expiration. - - Args: - key: Redis key for the image - image: Image array to save - expire_seconds: Optional expiration time in seconds - image_format: Image format ('jpeg' or 'png') - quality: JPEG quality (1-100) - - Returns: - True if successful, False otherwise - """ - try: - if not self.redis_client: - logger.error("Redis client not initialized") - self.stats['operations_failed'] += 1 - return False - - # Encode image - encoded_image = self._encode_image(image, image_format, quality) - if encoded_image is None: - logger.error("Failed to encode image") - self.stats['operations_failed'] += 1 - return False - - # Save to Redis - if expire_seconds: - await self.redis_client.setex(key, expire_seconds, encoded_image) - logger.debug(f"Saved image to Redis with key: {key} (expires in {expire_seconds}s)") - else: - await self.redis_client.set(key, encoded_image) - logger.debug(f"Saved image to Redis with key: {key}") - - self.stats['images_stored'] += 1 - self.stats['operations_successful'] += 1 - return True - - except Exception as e: - logger.error(f"Error saving image to Redis: {e}", exc_info=True) - self.stats['operations_failed'] += 1 - return False - - async def get_image(self, key: str) -> Optional[np.ndarray]: - """ - Retrieve image from Redis. - - Args: - key: Redis key for the image - - Returns: - Image array or None if not found - """ - try: - if not self.redis_client: - logger.error("Redis client not initialized") - self.stats['operations_failed'] += 1 - return None - - # Get image data from Redis - image_data = await self.redis_client.get(key) - if image_data is None: - logger.debug(f"Image not found for key: {key}") - return None - - # Decode image - image_array = np.frombuffer(image_data, np.uint8) - image = cv2.imdecode(image_array, cv2.IMREAD_COLOR) - - if image is not None: - logger.debug(f"Retrieved image from Redis with key: {key}") - self.stats['operations_successful'] += 1 - return image - else: - logger.error(f"Failed to decode image for key: {key}") - self.stats['operations_failed'] += 1 - return None - - except Exception as e: - logger.error(f"Error retrieving image from Redis: {e}", exc_info=True) - self.stats['operations_failed'] += 1 - return None - - async def delete_image(self, key: str) -> bool: - """ - Delete image from Redis. - - Args: - key: Redis key for the image - - Returns: - True if successful, False otherwise - """ - try: - if not self.redis_client: - logger.error("Redis client not initialized") - self.stats['operations_failed'] += 1 - return False - - result = await self.redis_client.delete(key) - if result > 0: - logger.debug(f"Deleted image from Redis with key: {key}") - self.stats['operations_successful'] += 1 - return True - else: - logger.debug(f"Image not found for deletion: {key}") - return False - - except Exception as e: - logger.error(f"Error deleting image from Redis: {e}", exc_info=True) - self.stats['operations_failed'] += 1 - return False - - async def publish_message(self, channel: str, message: Union[str, Dict]) -> int: - """ - Publish message to Redis channel. - - Args: - channel: Redis channel name - message: Message to publish (string or dict) - - Returns: - Number of subscribers that received the message, -1 on error - """ - try: - if not self.redis_client: - logger.error("Redis client not initialized") - self.stats['operations_failed'] += 1 - return -1 - - # Convert dict to JSON string if needed - if isinstance(message, dict): - message_str = json.dumps(message) - else: - message_str = str(message) - - # Test connection before publishing - await self.redis_client.ping() - - # Publish message - result = await self.redis_client.publish(channel, message_str) - - logger.info(f"Published message to Redis channel '{channel}': {message_str}") - logger.info(f"Redis publish result (subscribers count): {result}") - - if result == 0: - logger.warning(f"No subscribers listening to channel '{channel}'") - else: - logger.info(f"Message delivered to {result} subscriber(s)") - - self.stats['messages_published'] += 1 - self.stats['operations_successful'] += 1 - return result - - except Exception as e: - logger.error(f"Error publishing message to Redis: {e}", exc_info=True) - self.stats['operations_failed'] += 1 - return -1 - - async def subscribe_to_channel(self, channel: str, callback=None): - """ - Subscribe to Redis channel (for future use). - - Args: - channel: Redis channel name - callback: Optional callback function for messages - """ - try: - if not self.redis_client: - logger.error("Redis client not initialized") - return - - pubsub = self.redis_client.pubsub() - await pubsub.subscribe(channel) - - logger.info(f"Subscribed to Redis channel: {channel}") - - if callback: - async for message in pubsub.listen(): - if message['type'] == 'message': - try: - await callback(message['data']) - except Exception as e: - logger.error(f"Error in message callback: {e}") - - except Exception as e: - logger.error(f"Error subscribing to Redis channel: {e}", exc_info=True) - - async def set_key(self, key: str, value: Union[str, bytes], expire_seconds: Optional[int] = None) -> bool: - """ - Set a key-value pair in Redis. - - Args: - key: Redis key - value: Value to store - expire_seconds: Optional expiration time in seconds - - Returns: - True if successful, False otherwise - """ - try: - if not self.redis_client: - logger.error("Redis client not initialized") - self.stats['operations_failed'] += 1 - return False - - if expire_seconds: - await self.redis_client.setex(key, expire_seconds, value) - else: - await self.redis_client.set(key, value) - - logger.debug(f"Set Redis key: {key}") - self.stats['operations_successful'] += 1 - return True - - except Exception as e: - logger.error(f"Error setting Redis key: {e}", exc_info=True) - self.stats['operations_failed'] += 1 - return False - - async def get_key(self, key: str) -> Optional[Union[str, bytes]]: - """ - Get value for a Redis key. - - Args: - key: Redis key - - Returns: - Value or None if not found - """ - try: - if not self.redis_client: - logger.error("Redis client not initialized") - self.stats['operations_failed'] += 1 - return None - - value = await self.redis_client.get(key) - if value is not None: - logger.debug(f"Retrieved Redis key: {key}") - self.stats['operations_successful'] += 1 - - return value - - except Exception as e: - logger.error(f"Error getting Redis key: {e}", exc_info=True) - self.stats['operations_failed'] += 1 - return None - - async def delete_key(self, key: str) -> bool: - """ - Delete a Redis key. - - Args: - key: Redis key - - Returns: - True if successful, False otherwise - """ - try: - if not self.redis_client: - logger.error("Redis client not initialized") - self.stats['operations_failed'] += 1 - return False - - result = await self.redis_client.delete(key) - if result > 0: - logger.debug(f"Deleted Redis key: {key}") - self.stats['operations_successful'] += 1 - return True - else: - logger.debug(f"Redis key not found: {key}") - return False - - except Exception as e: - logger.error(f"Error deleting Redis key: {e}", exc_info=True) - self.stats['operations_failed'] += 1 - return False - - def _encode_image(self, image: np.ndarray, image_format: str, quality: int) -> Optional[bytes]: - """ - Encode image to bytes for Redis storage. - - Args: - image: Image array - image_format: Image format ('jpeg' or 'png') - quality: JPEG quality (1-100) - - Returns: - Encoded image bytes or None on error - """ - try: - format_lower = image_format.lower() - - if format_lower == 'jpeg' or format_lower == 'jpg': - encode_params = [cv2.IMWRITE_JPEG_QUALITY, quality] - success, buffer = cv2.imencode('.jpg', image, encode_params) - elif format_lower == 'png': - success, buffer = cv2.imencode('.png', image) - else: - logger.warning(f"Unknown image format '{image_format}', using JPEG") - encode_params = [cv2.IMWRITE_JPEG_QUALITY, quality] - success, buffer = cv2.imencode('.jpg', image, encode_params) - - if success: - return buffer.tobytes() - else: - logger.error(f"Failed to encode image as {image_format}") - return None - - except Exception as e: - logger.error(f"Error encoding image: {e}", exc_info=True) - return None - - def get_statistics(self) -> Dict[str, Any]: - """ - Get Redis manager statistics. - - Returns: - Dictionary with statistics - """ - return { - **self.stats, - 'connected': self.redis_client is not None, - 'host': self.host, - 'port': self.port, - 'db': self.db - } - - def cleanup(self): - """Cleanup Redis connection.""" - if self.redis_client: - # Note: redis.asyncio doesn't have a synchronous close method - # The connection will be closed when the event loop shuts down - self.redis_client = None - logger.info("Redis connection cleaned up") - - async def aclose(self): - """Async cleanup for Redis connection.""" - if self.redis_client: - await self.redis_client.aclose() - self.redis_client = None - logger.info("Redis connection closed") \ No newline at end of file diff --git a/core/streaming/__init__.py b/core/streaming/__init__.py deleted file mode 100644 index 93005ab..0000000 --- a/core/streaming/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -Streaming system for RTSP and HTTP camera feeds. -Provides modular frame readers, buffers, and stream management. -""" -from .readers import HTTPSnapshotReader, FFmpegRTSPReader -from .buffers import FrameBuffer, CacheBuffer, shared_frame_buffer, shared_cache_buffer -from .manager import StreamManager, StreamConfig, SubscriptionInfo, shared_stream_manager, initialize_stream_manager - -__all__ = [ - # Readers - 'HTTPSnapshotReader', - 'FFmpegRTSPReader', - - # Buffers - 'FrameBuffer', - 'CacheBuffer', - 'shared_frame_buffer', - 'shared_cache_buffer', - - # Manager - 'StreamManager', - 'StreamConfig', - 'SubscriptionInfo', - 'shared_stream_manager', - 'initialize_stream_manager' -] \ No newline at end of file diff --git a/core/streaming/buffers.py b/core/streaming/buffers.py deleted file mode 100644 index f2c5787..0000000 --- a/core/streaming/buffers.py +++ /dev/null @@ -1,295 +0,0 @@ -""" -Frame buffering and caching system optimized for different stream formats. -Supports 1280x720 RTSP streams and 2560x1440 HTTP snapshots. -""" -import threading -import time -import cv2 -import logging -import numpy as np -from typing import Optional, Dict, Any, Tuple -from collections import defaultdict - - -logger = logging.getLogger(__name__) - - -class FrameBuffer: - """Thread-safe frame buffer for all camera streams.""" - - def __init__(self, max_age_seconds: int = 5): - self.max_age_seconds = max_age_seconds - self._frames: Dict[str, Dict[str, Any]] = {} - self._lock = threading.RLock() - - def put_frame(self, camera_id: str, frame: np.ndarray): - """Store a frame for the given camera ID.""" - with self._lock: - # Validate frame - if not self._validate_frame(frame): - logger.warning(f"Frame validation failed for camera {camera_id}") - return - - self._frames[camera_id] = { - 'frame': frame.copy(), - 'timestamp': time.time(), - 'shape': frame.shape, - 'dtype': str(frame.dtype), - 'size_mb': frame.nbytes / (1024 * 1024) - } - - def get_frame(self, camera_id: str) -> Optional[np.ndarray]: - """Get the latest frame for the given camera ID.""" - with self._lock: - if camera_id not in self._frames: - return None - - frame_data = self._frames[camera_id] - - # Return frame regardless of age - frames persist until replaced - return frame_data['frame'].copy() - - def get_frame_info(self, camera_id: str) -> Optional[Dict[str, Any]]: - """Get frame metadata without copying the frame data.""" - with self._lock: - if camera_id not in self._frames: - return None - - frame_data = self._frames[camera_id] - age = time.time() - frame_data['timestamp'] - - # Return frame info regardless of age - frames persist until replaced - return { - 'timestamp': frame_data['timestamp'], - 'age': age, - 'shape': frame_data['shape'], - 'dtype': frame_data['dtype'], - 'size_mb': frame_data.get('size_mb', 0) - } - - def has_frame(self, camera_id: str) -> bool: - """Check if a valid frame exists for the camera.""" - return self.get_frame_info(camera_id) is not None - - def clear_camera(self, camera_id: str): - """Remove all frames for a specific camera.""" - with self._lock: - if camera_id in self._frames: - del self._frames[camera_id] - logger.debug(f"Cleared frames for camera {camera_id}") - - def clear_all(self): - """Clear all stored frames.""" - with self._lock: - count = len(self._frames) - self._frames.clear() - logger.debug(f"Cleared all frames ({count} cameras)") - - def get_camera_list(self) -> list: - """Get list of cameras with frames - all frames persist until replaced.""" - with self._lock: - # Return all cameras that have frames - no age-based filtering - return list(self._frames.keys()) - - def get_stats(self) -> Dict[str, Any]: - """Get buffer statistics.""" - with self._lock: - current_time = time.time() - stats = { - 'total_cameras': len(self._frames), - 'recent_cameras': 0, - 'stale_cameras': 0, - 'total_memory_mb': 0, - 'cameras': {} - } - - for camera_id, frame_data in self._frames.items(): - age = current_time - frame_data['timestamp'] - size_mb = frame_data.get('size_mb', 0) - - # All frames are valid/available, but categorize by freshness for monitoring - if age <= self.max_age_seconds: - stats['recent_cameras'] += 1 - else: - stats['stale_cameras'] += 1 - - stats['total_memory_mb'] += size_mb - - stats['cameras'][camera_id] = { - 'age': age, - 'recent': age <= self.max_age_seconds, # Recent but all frames available - 'shape': frame_data['shape'], - 'dtype': frame_data['dtype'], - 'size_mb': size_mb - } - - return stats - - def _validate_frame(self, frame: np.ndarray) -> bool: - """Validate frame - basic validation for any stream type.""" - if frame is None or frame.size == 0: - return False - - h, w = frame.shape[:2] - size_mb = frame.nbytes / (1024 * 1024) - - # Basic size validation - reject extremely large frames regardless of type - max_size_mb = 50 # Generous limit for any frame type - if size_mb > max_size_mb: - logger.warning(f"Frame too large: {size_mb:.2f}MB (max {max_size_mb}MB) for {w}x{h}") - return False - - # Basic dimension validation - if w < 100 or h < 100: - logger.warning(f"Frame too small: {w}x{h}") - return False - - return True - - -class CacheBuffer: - """Enhanced frame cache with support for cropping.""" - - def __init__(self, max_age_seconds: int = 10): - self.frame_buffer = FrameBuffer(max_age_seconds) - self._crop_cache: Dict[str, Dict[str, Any]] = {} - self._cache_lock = threading.RLock() - self.jpeg_quality = 95 # High quality for all frames - - def put_frame(self, camera_id: str, frame: np.ndarray): - """Store a frame and clear any associated crop cache.""" - self.frame_buffer.put_frame(camera_id, frame) - - # Clear crop cache for this camera since we have a new frame - with self._cache_lock: - keys_to_remove = [key for key in self._crop_cache.keys() if key.startswith(f"{camera_id}_")] - for key in keys_to_remove: - del self._crop_cache[key] - - def get_frame(self, camera_id: str, crop_coords: Optional[Tuple[int, int, int, int]] = None) -> Optional[np.ndarray]: - """Get frame with optional cropping.""" - if crop_coords is None: - return self.frame_buffer.get_frame(camera_id) - - # Check crop cache first - crop_key = f"{camera_id}_{crop_coords}" - with self._cache_lock: - if crop_key in self._crop_cache: - cache_entry = self._crop_cache[crop_key] - age = time.time() - cache_entry['timestamp'] - if age <= self.frame_buffer.max_age_seconds: - return cache_entry['cropped_frame'].copy() - else: - del self._crop_cache[crop_key] - - # Get original frame and crop it - original_frame = self.frame_buffer.get_frame(camera_id) - if original_frame is None: - return None - - try: - x1, y1, x2, y2 = crop_coords - - # Ensure coordinates are within frame bounds - h, w = original_frame.shape[:2] - x1 = max(0, min(x1, w)) - y1 = max(0, min(y1, h)) - x2 = max(x1, min(x2, w)) - y2 = max(y1, min(y2, h)) - - cropped_frame = original_frame[y1:y2, x1:x2] - - # Cache the cropped frame - with self._cache_lock: - # Limit cache size to prevent memory issues - if len(self._crop_cache) > 100: - # Remove oldest entries - oldest_keys = sorted(self._crop_cache.keys(), - key=lambda k: self._crop_cache[k]['timestamp'])[:50] - for key in oldest_keys: - del self._crop_cache[key] - - self._crop_cache[crop_key] = { - 'cropped_frame': cropped_frame.copy(), - 'timestamp': time.time(), - 'crop_coords': (x1, y1, x2, y2) - } - - return cropped_frame - - except Exception as e: - logger.error(f"Error cropping frame for camera {camera_id}: {e}") - return original_frame - - def get_frame_as_jpeg(self, camera_id: str, crop_coords: Optional[Tuple[int, int, int, int]] = None, - quality: Optional[int] = None) -> Optional[bytes]: - """Get frame as JPEG bytes.""" - frame = self.get_frame(camera_id, crop_coords) - if frame is None: - return None - - try: - # Use specified quality or default - if quality is None: - quality = self.jpeg_quality - - # Encode as JPEG with specified quality - encode_params = [cv2.IMWRITE_JPEG_QUALITY, quality] - success, encoded_img = cv2.imencode('.jpg', frame, encode_params) - - if success: - jpeg_bytes = encoded_img.tobytes() - logger.debug(f"Encoded JPEG for camera {camera_id}: quality={quality}, size={len(jpeg_bytes)} bytes") - return jpeg_bytes - - return None - - except Exception as e: - logger.error(f"Error encoding frame as JPEG for camera {camera_id}: {e}") - return None - - def has_frame(self, camera_id: str) -> bool: - """Check if a valid frame exists for the camera.""" - return self.frame_buffer.has_frame(camera_id) - - def clear_camera(self, camera_id: str): - """Remove all frames and cache for a specific camera.""" - self.frame_buffer.clear_camera(camera_id) - with self._cache_lock: - # Clear crop cache entries for this camera - keys_to_remove = [key for key in self._crop_cache.keys() if key.startswith(f"{camera_id}_")] - for key in keys_to_remove: - del self._crop_cache[key] - - def clear_all(self): - """Clear all stored frames and cache.""" - self.frame_buffer.clear_all() - with self._cache_lock: - self._crop_cache.clear() - - def get_stats(self) -> Dict[str, Any]: - """Get comprehensive buffer and cache statistics.""" - buffer_stats = self.frame_buffer.get_stats() - - with self._cache_lock: - cache_stats = { - 'crop_cache_entries': len(self._crop_cache), - 'crop_cache_cameras': len(set(key.split('_')[0] for key in self._crop_cache.keys() if '_' in key)), - 'crop_cache_memory_mb': sum( - entry['cropped_frame'].nbytes / (1024 * 1024) - for entry in self._crop_cache.values() - ) - } - - return { - 'buffer': buffer_stats, - 'cache': cache_stats, - 'total_memory_mb': buffer_stats.get('total_memory_mb', 0) + cache_stats.get('crop_cache_memory_mb', 0) - } - - -# Global shared instances for application use -shared_frame_buffer = FrameBuffer(max_age_seconds=5) -shared_cache_buffer = CacheBuffer(max_age_seconds=10) - - diff --git a/core/streaming/manager.py b/core/streaming/manager.py deleted file mode 100644 index c4ebd77..0000000 --- a/core/streaming/manager.py +++ /dev/null @@ -1,722 +0,0 @@ -""" -Stream coordination and lifecycle management. -Optimized for 1280x720@6fps RTSP and 2560x1440 HTTP snapshots. -""" -import logging -import threading -import time -import queue -import asyncio -from typing import Dict, Set, Optional, List, Any -from dataclasses import dataclass -from collections import defaultdict - -from .readers import HTTPSnapshotReader, FFmpegRTSPReader -from .buffers import shared_cache_buffer -from ..tracking.integration import TrackingPipelineIntegration - - -logger = logging.getLogger(__name__) - - -@dataclass -class StreamConfig: - """Configuration for a stream.""" - camera_id: str - rtsp_url: Optional[str] = None - snapshot_url: Optional[str] = None - snapshot_interval: int = 5000 # milliseconds - max_retries: int = 3 - - -@dataclass -class SubscriptionInfo: - """Information about a subscription.""" - subscription_id: str - camera_id: str - stream_config: StreamConfig - created_at: float - crop_coords: Optional[tuple] = None - model_id: Optional[str] = None - model_url: Optional[str] = None - tracking_integration: Optional[TrackingPipelineIntegration] = None - - -class StreamManager: - """Manages multiple camera streams with shared optimization.""" - - def __init__(self, max_streams: int = 10): - self.max_streams = max_streams - self._streams: Dict[str, Any] = {} # camera_id -> reader instance - self._subscriptions: Dict[str, SubscriptionInfo] = {} # subscription_id -> info - self._camera_subscribers: Dict[str, Set[str]] = defaultdict(set) # camera_id -> set of subscription_ids - self._lock = threading.RLock() - - # Fair tracking queue system - per camera queues - self._tracking_queues: Dict[str, queue.Queue] = {} # camera_id -> queue - self._tracking_workers = [] - self._stop_workers = threading.Event() - self._dropped_frame_counts: Dict[str, int] = {} # per-camera drop counts - - # Round-robin scheduling state - self._camera_list = [] # Ordered list of active cameras - self._camera_round_robin_index = 0 - self._round_robin_lock = threading.Lock() - - # Start worker threads for tracking processing - num_workers = min(4, max_streams // 2 + 1) # Scale with streams - for i in range(num_workers): - worker = threading.Thread( - target=self._tracking_worker_loop, - name=f"TrackingWorker-{i}", - daemon=True - ) - worker.start() - self._tracking_workers.append(worker) - - logger.info(f"Started {num_workers} tracking worker threads") - - def _ensure_camera_queue(self, camera_id: str): - """Ensure a tracking queue exists for the camera.""" - if camera_id not in self._tracking_queues: - self._tracking_queues[camera_id] = queue.Queue(maxsize=10) # 10 frames per camera - self._dropped_frame_counts[camera_id] = 0 - - with self._round_robin_lock: - if camera_id not in self._camera_list: - self._camera_list.append(camera_id) - logger.info(f"Created tracking queue for camera {camera_id}") - else: - logger.debug(f"Camera {camera_id} already has tracking queue") - - def _remove_camera_queue(self, camera_id: str): - """Remove tracking queue for a camera that's no longer active.""" - if camera_id in self._tracking_queues: - # Clear any remaining items - while not self._tracking_queues[camera_id].empty(): - try: - self._tracking_queues[camera_id].get_nowait() - except queue.Empty: - break - - del self._tracking_queues[camera_id] - del self._dropped_frame_counts[camera_id] - - with self._round_robin_lock: - if camera_id in self._camera_list: - self._camera_list.remove(camera_id) - # Reset index if needed - if self._camera_round_robin_index >= len(self._camera_list): - self._camera_round_robin_index = 0 - - logger.info(f"Removed tracking queue for camera {camera_id}") - - def add_subscription(self, subscription_id: str, stream_config: StreamConfig, - crop_coords: Optional[tuple] = None, - model_id: Optional[str] = None, - model_url: Optional[str] = None, - tracking_integration: Optional[TrackingPipelineIntegration] = None) -> bool: - """Add a new subscription. Returns True if successful.""" - with self._lock: - if subscription_id in self._subscriptions: - logger.warning(f"Subscription {subscription_id} already exists") - return False - - camera_id = stream_config.camera_id - - # Create subscription info - subscription_info = SubscriptionInfo( - subscription_id=subscription_id, - camera_id=camera_id, - stream_config=stream_config, - created_at=time.time(), - crop_coords=crop_coords, - model_id=model_id, - model_url=model_url, - tracking_integration=tracking_integration - ) - - # Pass subscription info to tracking integration for snapshot access - if tracking_integration: - tracking_integration.set_subscription_info(subscription_info) - - self._subscriptions[subscription_id] = subscription_info - self._camera_subscribers[camera_id].add(subscription_id) - - # Start stream if not already running - if camera_id not in self._streams: - if len(self._streams) >= self.max_streams: - logger.error(f"Maximum streams ({self.max_streams}) reached, cannot add {camera_id}") - self._remove_subscription_internal(subscription_id) - return False - - success = self._start_stream(camera_id, stream_config) - if not success: - self._remove_subscription_internal(subscription_id) - return False - else: - # Stream already exists, but ensure queue exists too - logger.info(f"Stream already exists for {camera_id}, ensuring queue exists") - self._ensure_camera_queue(camera_id) - - logger.info(f"Added subscription {subscription_id} for camera {camera_id} " - f"({len(self._camera_subscribers[camera_id])} total subscribers)") - return True - - def remove_subscription(self, subscription_id: str) -> bool: - """Remove a subscription. Returns True if found and removed.""" - with self._lock: - return self._remove_subscription_internal(subscription_id) - - def _remove_subscription_internal(self, subscription_id: str) -> bool: - """Internal method to remove subscription (assumes lock is held).""" - if subscription_id not in self._subscriptions: - logger.warning(f"Subscription {subscription_id} not found") - return False - - subscription_info = self._subscriptions[subscription_id] - camera_id = subscription_info.camera_id - - # Remove from tracking - del self._subscriptions[subscription_id] - self._camera_subscribers[camera_id].discard(subscription_id) - - # Stop stream if no more subscribers - if not self._camera_subscribers[camera_id]: - self._stop_stream(camera_id) - del self._camera_subscribers[camera_id] - - logger.info(f"Removed subscription {subscription_id} for camera {camera_id} " - f"({len(self._camera_subscribers[camera_id])} remaining subscribers)") - return True - - def _start_stream(self, camera_id: str, stream_config: StreamConfig) -> bool: - """Start a stream for the given camera.""" - try: - if stream_config.rtsp_url: - # RTSP stream using FFmpeg subprocess with CUDA acceleration - logger.info(f"\033[94m[RTSP] Starting {camera_id}\033[0m") - reader = FFmpegRTSPReader( - camera_id=camera_id, - rtsp_url=stream_config.rtsp_url, - max_retries=stream_config.max_retries - ) - reader.set_frame_callback(self._frame_callback) - reader.start() - self._streams[camera_id] = reader - self._ensure_camera_queue(camera_id) # Create tracking queue - logger.info(f"\033[92m[RTSP] {camera_id} connected\033[0m") - - elif stream_config.snapshot_url: - # HTTP snapshot stream - logger.info(f"\033[95m[HTTP] Starting {camera_id}\033[0m") - reader = HTTPSnapshotReader( - camera_id=camera_id, - snapshot_url=stream_config.snapshot_url, - interval_ms=stream_config.snapshot_interval, - max_retries=stream_config.max_retries - ) - reader.set_frame_callback(self._frame_callback) - reader.start() - self._streams[camera_id] = reader - self._ensure_camera_queue(camera_id) # Create tracking queue - logger.info(f"\033[92m[HTTP] {camera_id} connected\033[0m") - - else: - logger.error(f"No valid URL provided for camera {camera_id}") - return False - - return True - - except Exception as e: - logger.error(f"Error starting stream for camera {camera_id}: {e}") - return False - - def _stop_stream(self, camera_id: str): - """Stop a stream for the given camera.""" - if camera_id in self._streams: - try: - self._streams[camera_id].stop() - del self._streams[camera_id] - self._remove_camera_queue(camera_id) # Remove tracking queue - # DON'T clear frames - they should persist until replaced - # shared_cache_buffer.clear_camera(camera_id) # REMOVED - frames should persist - logger.info(f"Stopped stream for camera {camera_id} (frames preserved in buffer)") - except Exception as e: - logger.error(f"Error stopping stream for camera {camera_id}: {e}") - - def _frame_callback(self, camera_id: str, frame): - """Callback for when a new frame is available.""" - try: - # Store frame in shared buffer - shared_cache_buffer.put_frame(camera_id, frame) - # Quieter frame callback logging - only log occasionally - if hasattr(self, '_frame_log_count'): - self._frame_log_count += 1 - else: - self._frame_log_count = 1 - - # Log every 100 frames to avoid spam - if self._frame_log_count % 100 == 0: - available_cameras = shared_cache_buffer.frame_buffer.get_camera_list() - logger.info(f"\033[96m[BUFFER] {len(available_cameras)} active cameras: {', '.join(available_cameras)}\033[0m") - - # Queue for tracking processing (non-blocking) - route to camera-specific queue - if camera_id in self._tracking_queues: - try: - self._tracking_queues[camera_id].put_nowait({ - 'frame': frame, - 'timestamp': time.time() - }) - except queue.Full: - # Drop frame if camera queue is full (maintain real-time) - self._dropped_frame_counts[camera_id] += 1 - - if self._dropped_frame_counts[camera_id] % 50 == 0: - logger.warning(f"Dropped {self._dropped_frame_counts[camera_id]} frames for camera {camera_id} due to full queue") - - except Exception as e: - logger.error(f"Error in frame callback for camera {camera_id}: {e}") - - def _process_tracking_for_camera(self, camera_id: str, frame): - """Process tracking for all subscriptions of a camera.""" - try: - with self._lock: - for subscription_id in self._camera_subscribers[camera_id]: - subscription_info = self._subscriptions[subscription_id] - - # Skip if no tracking integration - if not subscription_info.tracking_integration: - continue - - # Extract display_id from subscription_id - display_id = subscription_id.split(';')[0] if ';' in subscription_id else subscription_id - - # Process frame through tracking asynchronously - # Note: This is synchronous for now, can be made async in future - try: - # Create a simple asyncio event loop for this frame - import asyncio - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - try: - result = loop.run_until_complete( - subscription_info.tracking_integration.process_frame( - frame, display_id, subscription_id - ) - ) - # Log tracking results - if result: - tracked_count = len(result.get('tracked_vehicles', [])) - validated_vehicle = result.get('validated_vehicle') - pipeline_result = result.get('pipeline_result') - - if tracked_count > 0: - logger.info(f"[Tracking] {camera_id}: {tracked_count} vehicles tracked") - - if validated_vehicle: - logger.info(f"[Tracking] {camera_id}: Vehicle {validated_vehicle['track_id']} " - f"validated as {validated_vehicle['state']} " - f"(confidence: {validated_vehicle['confidence']:.2f})") - - if pipeline_result: - logger.info(f"[Pipeline] {camera_id}: {pipeline_result.get('status', 'unknown')} - " - f"{pipeline_result.get('message', 'no message')}") - finally: - loop.close() - except Exception as track_e: - logger.error(f"Error in tracking for {subscription_id}: {track_e}") - - except Exception as e: - logger.error(f"Error processing tracking for camera {camera_id}: {e}") - - def _tracking_worker_loop(self): - """Worker thread loop for round-robin processing of camera queues.""" - logger.info(f"Tracking worker {threading.current_thread().name} started") - - consecutive_empty = 0 - max_consecutive_empty = 10 # Sleep if all cameras empty this many times - - while not self._stop_workers.is_set(): - try: - # Get next camera in round-robin fashion - camera_id, item = self._get_next_camera_item() - - if camera_id is None: - # No cameras have items, sleep briefly - consecutive_empty += 1 - if consecutive_empty >= max_consecutive_empty: - time.sleep(0.1) # Sleep 100ms if nothing to process - consecutive_empty = 0 - continue - - consecutive_empty = 0 # Reset counter when we find work - - frame = item['frame'] - timestamp = item['timestamp'] - - # Check if frame is too old (drop if > 1 second old) - age = time.time() - timestamp - if age > 1.0: - logger.debug(f"Dropping old frame for {camera_id} (age: {age:.2f}s)") - continue - - # Process tracking for this camera's frame - self._process_tracking_for_camera_sync(camera_id, frame) - - except Exception as e: - logger.error(f"Error in tracking worker: {e}", exc_info=True) - - logger.info(f"Tracking worker {threading.current_thread().name} stopped") - - def _get_next_camera_item(self): - """Get next item from camera queues using round-robin scheduling.""" - with self._round_robin_lock: - # Get current list of cameras from actual tracking queues (central state) - camera_list = list(self._tracking_queues.keys()) - - if not camera_list: - return None, None - - attempts = 0 - max_attempts = len(camera_list) - - while attempts < max_attempts: - # Get current camera using round-robin index - if self._camera_round_robin_index >= len(camera_list): - self._camera_round_robin_index = 0 - - camera_id = camera_list[self._camera_round_robin_index] - - # Move to next camera for next call - self._camera_round_robin_index = (self._camera_round_robin_index + 1) % len(camera_list) - - # Try to get item from this camera's queue - try: - item = self._tracking_queues[camera_id].get_nowait() - return camera_id, item - except queue.Empty: - pass # Try next camera - - attempts += 1 - - return None, None # All cameras empty - - def _process_tracking_for_camera_sync(self, camera_id: str, frame): - """Synchronous version of tracking processing for worker threads.""" - try: - with self._lock: - subscription_ids = list(self._camera_subscribers.get(camera_id, [])) - - for subscription_id in subscription_ids: - subscription_info = self._subscriptions.get(subscription_id) - - if not subscription_info: - logger.warning(f"No subscription info found for {subscription_id}") - continue - - if not subscription_info.tracking_integration: - logger.debug(f"No tracking integration for {subscription_id} (camera {camera_id}), skipping inference") - continue - - display_id = subscription_id.split(';')[0] if ';' in subscription_id else subscription_id - - try: - # Run async tracking in thread's event loop - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - try: - result = loop.run_until_complete( - subscription_info.tracking_integration.process_frame( - frame, display_id, subscription_id - ) - ) - - # Log tracking results - if result: - tracked_count = len(result.get('tracked_vehicles', [])) - validated_vehicle = result.get('validated_vehicle') - pipeline_result = result.get('pipeline_result') - - if tracked_count > 0: - logger.info(f"[Tracking] {camera_id}: {tracked_count} vehicles tracked") - - if validated_vehicle: - logger.info(f"[Tracking] {camera_id}: Vehicle {validated_vehicle['track_id']} " - f"validated as {validated_vehicle['state']} " - f"(confidence: {validated_vehicle['confidence']:.2f})") - - if pipeline_result: - logger.info(f"[Pipeline] {camera_id}: {pipeline_result.get('status', 'unknown')} - " - f"{pipeline_result.get('message', 'no message')}") - finally: - loop.close() - - except Exception as track_e: - logger.error(f"Error in tracking for {subscription_id}: {track_e}") - - except Exception as e: - logger.error(f"Error processing tracking for camera {camera_id}: {e}") - - def get_frame(self, camera_id: str, crop_coords: Optional[tuple] = None): - """Get the latest frame for a camera with optional cropping.""" - return shared_cache_buffer.get_frame(camera_id, crop_coords) - - def get_frame_as_jpeg(self, camera_id: str, crop_coords: Optional[tuple] = None, - quality: int = 100) -> Optional[bytes]: - """Get frame as JPEG bytes for HTTP responses with highest quality by default.""" - return shared_cache_buffer.get_frame_as_jpeg(camera_id, crop_coords, quality) - - def has_frame(self, camera_id: str) -> bool: - """Check if a frame is available for the camera.""" - return shared_cache_buffer.has_frame(camera_id) - - def get_subscription_info(self, subscription_id: str) -> Optional[SubscriptionInfo]: - """Get information about a subscription.""" - with self._lock: - return self._subscriptions.get(subscription_id) - - def get_camera_subscribers(self, camera_id: str) -> Set[str]: - """Get all subscription IDs for a camera.""" - with self._lock: - return self._camera_subscribers[camera_id].copy() - - def get_active_cameras(self) -> List[str]: - """Get list of cameras with active streams.""" - with self._lock: - return list(self._streams.keys()) - - def get_all_subscriptions(self) -> List[SubscriptionInfo]: - """Get all active subscriptions.""" - with self._lock: - return list(self._subscriptions.values()) - - def reconcile_subscriptions(self, target_subscriptions: List[Dict[str, Any]]) -> Dict[str, Any]: - """ - Reconcile current subscriptions with target list. - Returns summary of changes made. - """ - with self._lock: - current_subscription_ids = set(self._subscriptions.keys()) - target_subscription_ids = {sub['subscriptionIdentifier'] for sub in target_subscriptions} - - # Find subscriptions to remove and add - to_remove = current_subscription_ids - target_subscription_ids - to_add = target_subscription_ids - current_subscription_ids - - # Remove old subscriptions - removed_count = 0 - for subscription_id in to_remove: - if self._remove_subscription_internal(subscription_id): - removed_count += 1 - - # Add new subscriptions - added_count = 0 - failed_count = 0 - for target_sub in target_subscriptions: - subscription_id = target_sub['subscriptionIdentifier'] - if subscription_id in to_add: - success = self._add_subscription_from_payload(subscription_id, target_sub) - if success: - added_count += 1 - else: - failed_count += 1 - - result = { - 'removed': removed_count, - 'added': added_count, - 'failed': failed_count, - 'total_active': len(self._subscriptions), - 'active_streams': len(self._streams) - } - - logger.info(f"Subscription reconciliation: {result}") - return result - - def _add_subscription_from_payload(self, subscription_id: str, payload: Dict[str, Any]) -> bool: - """Add subscription from WebSocket payload format.""" - try: - # Extract camera ID from subscription identifier - # Format: "display-001;cam-001" -> camera_id = "cam-001" - camera_id = subscription_id.split(';')[-1] - - # Extract crop coordinates if present - crop_coords = None - if all(key in payload for key in ['cropX1', 'cropY1', 'cropX2', 'cropY2']): - crop_coords = ( - payload['cropX1'], - payload['cropY1'], - payload['cropX2'], - payload['cropY2'] - ) - - # Create stream configuration - stream_config = StreamConfig( - camera_id=camera_id, - rtsp_url=payload.get('rtspUrl'), - snapshot_url=payload.get('snapshotUrl'), - snapshot_interval=payload.get('snapshotInterval', 5000), - max_retries=3, - ) - - return self.add_subscription( - subscription_id, - stream_config, - crop_coords, - model_id=payload.get('modelId'), - model_url=payload.get('modelUrl') - ) - - except Exception as e: - logger.error(f"Error adding subscription from payload {subscription_id}: {e}") - return False - - def stop_all(self): - """Stop all streams and clear all subscriptions.""" - # Signal workers to stop - self._stop_workers.set() - - # Clear all camera queues - for camera_id, camera_queue in list(self._tracking_queues.items()): - while not camera_queue.empty(): - try: - camera_queue.get_nowait() - except queue.Empty: - break - - # Wait for workers to finish - for worker in self._tracking_workers: - worker.join(timeout=2.0) - - # Clear queue management structures - self._tracking_queues.clear() - self._dropped_frame_counts.clear() - with self._round_robin_lock: - self._camera_list.clear() - self._camera_round_robin_index = 0 - - logger.info("Stopped all tracking worker threads") - - with self._lock: - # Stop all streams - for camera_id in list(self._streams.keys()): - self._stop_stream(camera_id) - - # Clear all tracking - self._subscriptions.clear() - self._camera_subscribers.clear() - shared_cache_buffer.clear_all() - - logger.info("Stopped all streams and cleared all subscriptions") - - def set_session_id(self, display_id: str, session_id: str): - """Set session ID for tracking integration.""" - # Ensure session_id is always a string for consistent type handling - session_id = str(session_id) if session_id is not None else None - with self._lock: - for subscription_info in self._subscriptions.values(): - # Check if this subscription matches the display_id - subscription_display_id = subscription_info.subscription_id.split(';')[0] - if subscription_display_id == display_id and subscription_info.tracking_integration: - # Pass the full subscription_id (displayId;cameraId) to the tracking integration - subscription_info.tracking_integration.set_session_id( - display_id, - session_id, - subscription_id=subscription_info.subscription_id - ) - logger.debug(f"Set session {session_id} for display {display_id} with subscription {subscription_info.subscription_id}") - - def clear_session_id(self, session_id: str): - """Clear session ID from the specific tracking integration handling this session.""" - with self._lock: - # Find the subscription that's handling this session - session_subscription = None - for subscription_info in self._subscriptions.values(): - if subscription_info.tracking_integration: - # Check if this integration is handling the given session_id - integration = subscription_info.tracking_integration - if session_id in integration.session_vehicles: - session_subscription = subscription_info - break - - if session_subscription and session_subscription.tracking_integration: - session_subscription.tracking_integration.clear_session_id(session_id) - logger.debug(f"Cleared session {session_id} from subscription {session_subscription.subscription_id}") - else: - logger.warning(f"No tracking integration found for session {session_id}, broadcasting to all subscriptions") - # Fallback: broadcast to all (original behavior) - for subscription_info in self._subscriptions.values(): - if subscription_info.tracking_integration: - subscription_info.tracking_integration.clear_session_id(session_id) - - def set_progression_stage(self, session_id: str, stage: str): - """Set progression stage for the specific tracking integration handling this session.""" - with self._lock: - # Find the subscription that's handling this session - session_subscription = None - for subscription_info in self._subscriptions.values(): - if subscription_info.tracking_integration: - # Check if this integration is handling the given session_id - # We need to check the integration's active sessions - integration = subscription_info.tracking_integration - if session_id in integration.session_vehicles: - session_subscription = subscription_info - break - - if session_subscription and session_subscription.tracking_integration: - session_subscription.tracking_integration.set_progression_stage(session_id, stage) - logger.debug(f"Set progression stage for session {session_id}: {stage} on subscription {session_subscription.subscription_id}") - else: - logger.warning(f"No tracking integration found for session {session_id}, broadcasting to all subscriptions") - # Fallback: broadcast to all (original behavior) - for subscription_info in self._subscriptions.values(): - if subscription_info.tracking_integration: - subscription_info.tracking_integration.set_progression_stage(session_id, stage) - - def get_tracking_stats(self) -> Dict[str, Any]: - """Get tracking statistics from all subscriptions.""" - stats = {} - with self._lock: - for subscription_id, subscription_info in self._subscriptions.items(): - if subscription_info.tracking_integration: - stats[subscription_id] = subscription_info.tracking_integration.get_statistics() - return stats - - - def get_stats(self) -> Dict[str, Any]: - """Get comprehensive streaming statistics.""" - with self._lock: - buffer_stats = shared_cache_buffer.get_stats() - tracking_stats = self.get_tracking_stats() - - return { - 'active_subscriptions': len(self._subscriptions), - 'active_streams': len(self._streams), - 'cameras_with_subscribers': len(self._camera_subscribers), - 'max_streams': self.max_streams, - 'subscriptions_by_camera': { - camera_id: len(subscribers) - for camera_id, subscribers in self._camera_subscribers.items() - }, - 'buffer_stats': buffer_stats, - 'tracking_stats': tracking_stats, - 'memory_usage_mb': buffer_stats.get('total_memory_mb', 0) - } - - -# Global shared instance for application use -# Default initialization, will be updated with config value in app.py -shared_stream_manager = StreamManager(max_streams=20) - -def initialize_stream_manager(max_streams: int = 10): - """Re-initialize the global stream manager with config value.""" - global shared_stream_manager - # Release old manager if exists - if shared_stream_manager: - try: - # Stop all existing streams gracefully - shared_stream_manager.stop_all() - except Exception as e: - logger.warning(f"Error stopping previous stream manager: {e}") - shared_stream_manager = StreamManager(max_streams=max_streams) - return shared_stream_manager \ No newline at end of file diff --git a/core/streaming/readers/__init__.py b/core/streaming/readers/__init__.py deleted file mode 100644 index 0903d6d..0000000 --- a/core/streaming/readers/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -""" -Stream readers for RTSP and HTTP camera feeds. -""" -from .base import VideoReader -from .ffmpeg_rtsp import FFmpegRTSPReader -from .http_snapshot import HTTPSnapshotReader -from .utils import log_success, log_warning, log_error, log_info, Colors - -__all__ = [ - 'VideoReader', - 'FFmpegRTSPReader', - 'HTTPSnapshotReader', - 'log_success', - 'log_warning', - 'log_error', - 'log_info', - 'Colors' -] \ No newline at end of file diff --git a/core/streaming/readers/base.py b/core/streaming/readers/base.py deleted file mode 100644 index 56c41cb..0000000 --- a/core/streaming/readers/base.py +++ /dev/null @@ -1,65 +0,0 @@ -""" -Abstract base class for video stream readers. -""" -from abc import ABC, abstractmethod -from typing import Optional, Callable -import numpy as np - - -class VideoReader(ABC): - """Abstract base class for video stream readers.""" - - def __init__(self, camera_id: str, source_url: str, max_retries: int = 3): - """ - Initialize the video reader. - - Args: - camera_id: Unique identifier for the camera - source_url: URL or path to the video source - max_retries: Maximum number of retry attempts - """ - self.camera_id = camera_id - self.source_url = source_url - self.max_retries = max_retries - self.frame_callback: Optional[Callable[[str, np.ndarray], None]] = None - - @abstractmethod - def start(self) -> None: - """Start the video reader.""" - pass - - @abstractmethod - def stop(self) -> None: - """Stop the video reader.""" - pass - - @abstractmethod - def set_frame_callback(self, callback: Callable[[str, np.ndarray], None]) -> None: - """ - Set callback function to handle captured frames. - - Args: - callback: Function that takes (camera_id, frame) as arguments - """ - pass - - @property - @abstractmethod - def is_running(self) -> bool: - """Check if the reader is currently running.""" - pass - - @property - @abstractmethod - def reader_type(self) -> str: - """Get the type of reader (e.g., 'rtsp', 'http_snapshot').""" - pass - - def __enter__(self): - """Context manager entry.""" - self.start() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - """Context manager exit.""" - self.stop() \ No newline at end of file diff --git a/core/streaming/readers/ffmpeg_rtsp.py b/core/streaming/readers/ffmpeg_rtsp.py deleted file mode 100644 index 88f45ae..0000000 --- a/core/streaming/readers/ffmpeg_rtsp.py +++ /dev/null @@ -1,436 +0,0 @@ -""" -FFmpeg RTSP stream reader using subprocess piping frames directly to buffer. -Enhanced with comprehensive health monitoring and automatic recovery. -""" -import cv2 -import time -import threading -import numpy as np -import subprocess -import struct -from typing import Optional, Callable, Dict, Any - -from .base import VideoReader -from .utils import log_success, log_warning, log_error, log_info -from ...monitoring.stream_health import stream_health_tracker -from ...monitoring.thread_health import thread_health_monitor -from ...monitoring.recovery import recovery_manager, RecoveryAction - - -class FFmpegRTSPReader(VideoReader): - """RTSP stream reader using subprocess FFmpeg piping frames directly to buffer.""" - - def __init__(self, camera_id: str, rtsp_url: str, max_retries: int = 3): - super().__init__(camera_id, rtsp_url, max_retries) - self.rtsp_url = rtsp_url - self.process = None - self.stop_event = threading.Event() - self.thread = None - self.stderr_thread = None - - # Expected stream specs (for reference, actual dimensions read from PPM header) - self.width = 1280 - self.height = 720 - - # Watchdog timers for stream reliability - self.process_start_time = None - self.last_frame_time = None - self.is_restart = False # Track if this is a restart (shorter timeout) - self.first_start_timeout = 30.0 # 30s timeout on first start - self.restart_timeout = 15.0 # 15s timeout after restart - - # Health monitoring setup - self.last_heartbeat = time.time() - self.consecutive_errors = 0 - self.ffmpeg_restart_count = 0 - - # Register recovery handlers - recovery_manager.register_recovery_handler( - RecoveryAction.RESTART_STREAM, - self._handle_restart_recovery - ) - recovery_manager.register_recovery_handler( - RecoveryAction.RECONNECT, - self._handle_reconnect_recovery - ) - - @property - def is_running(self) -> bool: - """Check if the reader is currently running.""" - return self.thread is not None and self.thread.is_alive() - - @property - def reader_type(self) -> str: - """Get the type of reader.""" - return "rtsp_ffmpeg" - - def set_frame_callback(self, callback: Callable[[str, np.ndarray], None]): - """Set callback function to handle captured frames.""" - self.frame_callback = callback - - def start(self): - """Start the FFmpeg subprocess reader.""" - if self.thread and self.thread.is_alive(): - log_warning(self.camera_id, "FFmpeg reader already running") - return - - self.stop_event.clear() - self.thread = threading.Thread(target=self._read_frames, daemon=True) - self.thread.start() - - # Register with health monitoring - stream_health_tracker.register_stream(self.camera_id, "rtsp_ffmpeg", self.rtsp_url) - thread_health_monitor.register_thread(self.thread, self._heartbeat_callback) - - log_success(self.camera_id, "Stream started with health monitoring") - - def stop(self): - """Stop the FFmpeg subprocess reader.""" - self.stop_event.set() - - # Unregister from health monitoring - if self.thread: - thread_health_monitor.unregister_thread(self.thread.ident) - - if self.process: - self.process.terminate() - try: - self.process.wait(timeout=5) - except subprocess.TimeoutExpired: - self.process.kill() - - if self.thread: - self.thread.join(timeout=5.0) - if self.stderr_thread: - self.stderr_thread.join(timeout=2.0) - - stream_health_tracker.unregister_stream(self.camera_id) - - log_info(self.camera_id, "Stream stopped") - - def _start_ffmpeg_process(self): - """Start FFmpeg subprocess outputting BMP frames to stdout pipe.""" - cmd = [ - 'ffmpeg', - # DO NOT REMOVE - '-hwaccel', 'cuda', - '-hwaccel_device', '0', - # Real-time input flags - '-fflags', 'nobuffer+genpts', - '-flags', 'low_delay', - '-max_delay', '0', # No reordering delay - # RTSP configuration - '-rtsp_transport', 'tcp', - '-i', self.rtsp_url, - # Output configuration (keeping BMP) - '-f', 'image2pipe', # Output images to pipe - '-vcodec', 'bmp', # BMP format with header containing dimensions - '-vsync', 'passthrough', # Pass frames as-is - # Use native stream resolution and framerate - '-an', # No audio - '-' # Output to stdout - ] - - try: - # Start FFmpeg with stdout pipe to read frames directly - self.process = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, # Capture stdout for frame data - stderr=subprocess.PIPE, # Capture stderr for error logging - bufsize=0 # Unbuffered for real-time processing - ) - - # Start stderr reading thread - if self.stderr_thread and self.stderr_thread.is_alive(): - # Stop previous stderr thread - try: - self.stderr_thread.join(timeout=1.0) - except: - pass - - self.stderr_thread = threading.Thread(target=self._read_stderr, daemon=True) - self.stderr_thread.start() - - # Set process start time for watchdog - self.process_start_time = time.time() - self.last_frame_time = None # Reset frame time - - # After successful restart, next timeout will be back to 30s - if self.is_restart: - log_info(self.camera_id, f"FFmpeg restarted successfully, next timeout: {self.first_start_timeout}s") - self.is_restart = False - - return True - except Exception as e: - log_error(self.camera_id, f"FFmpeg startup failed: {e}") - return False - - def _read_bmp_frame(self, pipe): - """Read BMP frame from pipe - BMP header contains dimensions.""" - try: - # Read BMP header (14 bytes file header + 40 bytes info header = 54 bytes minimum) - header_data = b'' - bytes_to_read = 54 - - while len(header_data) < bytes_to_read: - chunk = pipe.read(bytes_to_read - len(header_data)) - if not chunk: - return None # Silent end of stream - header_data += chunk - - # Parse BMP header - if header_data[:2] != b'BM': - return None # Invalid format, skip frame silently - - # Extract file size from header (bytes 2-5) - file_size = struct.unpack(' bool: - """Check if watchdog timeout has been exceeded.""" - if not self.process_start_time: - return False - - current_time = time.time() - time_since_start = current_time - self.process_start_time - - # Determine timeout based on whether this is a restart - timeout = self.restart_timeout if self.is_restart else self.first_start_timeout - - # If no frames received yet, check against process start time - if not self.last_frame_time: - if time_since_start > timeout: - log_warning(self.camera_id, f"Watchdog timeout: No frames for {time_since_start:.1f}s (limit: {timeout}s)") - return True - else: - # Check time since last frame - time_since_frame = current_time - self.last_frame_time - if time_since_frame > timeout: - log_warning(self.camera_id, f"Watchdog timeout: No frames for {time_since_frame:.1f}s (limit: {timeout}s)") - return True - - return False - - def _restart_ffmpeg_process(self): - """Restart FFmpeg process due to watchdog timeout.""" - log_warning(self.camera_id, "Watchdog triggered FFmpeg restart") - - # Terminate current process - if self.process: - try: - self.process.terminate() - self.process.wait(timeout=3) - except subprocess.TimeoutExpired: - self.process.kill() - except Exception: - pass - self.process = None - - # Mark as restart for shorter timeout - self.is_restart = True - - # Small delay before restart - time.sleep(1.0) - - def _read_frames(self): - """Read frames directly from FFmpeg stdout pipe.""" - frame_count = 0 - last_log_time = time.time() - - while not self.stop_event.is_set(): - try: - # Send heartbeat for thread health monitoring - self._send_heartbeat("reading_frames") - - # Check watchdog timeout if process is running - if self.process and self.process.poll() is None: - if self._check_watchdog_timeout(): - self._restart_ffmpeg_process() - continue - - # Start FFmpeg if not running - if not self.process or self.process.poll() is not None: - if self.process and self.process.poll() is not None: - log_warning(self.camera_id, "Stream disconnected, reconnecting...") - stream_health_tracker.report_error( - self.camera_id, - "FFmpeg process disconnected" - ) - - if not self._start_ffmpeg_process(): - self.consecutive_errors += 1 - stream_health_tracker.report_error( - self.camera_id, - "Failed to start FFmpeg process" - ) - time.sleep(5.0) - continue - - # Read frames directly from FFmpeg stdout - try: - if self.process and self.process.stdout: - # Read BMP frame data - frame = self._read_bmp_frame(self.process.stdout) - if frame is None: - continue - - # Update watchdog - we got a frame - self.last_frame_time = time.time() - - # Reset error counter on successful frame - self.consecutive_errors = 0 - - # Report successful frame to health monitoring - frame_size = frame.nbytes - stream_health_tracker.report_frame_received(self.camera_id, frame_size) - - # Call frame callback - if self.frame_callback: - try: - self.frame_callback(self.camera_id, frame) - except Exception as e: - stream_health_tracker.report_error( - self.camera_id, - f"Frame callback error: {e}" - ) - - frame_count += 1 - - # Log progress every 60 seconds (quieter) - current_time = time.time() - if current_time - last_log_time >= 60: - log_success(self.camera_id, f"{frame_count} frames captured ({frame.shape[1]}x{frame.shape[0]})") - last_log_time = current_time - - except Exception as e: - # Process might have died, let it restart on next iteration - stream_health_tracker.report_error( - self.camera_id, - f"Frame reading error: {e}" - ) - if self.process: - self.process.terminate() - self.process = None - time.sleep(1.0) - - except Exception as e: - stream_health_tracker.report_error( - self.camera_id, - f"Main loop error: {e}" - ) - time.sleep(1.0) - - # Cleanup - if self.process: - self.process.terminate() - - # Health monitoring methods - def _send_heartbeat(self, activity: str = "running"): - """Send heartbeat to thread health monitor.""" - self.last_heartbeat = time.time() - thread_health_monitor.heartbeat(activity=activity) - - def _heartbeat_callback(self) -> bool: - """Heartbeat callback for thread responsiveness testing.""" - try: - # Check if thread is responsive by checking recent heartbeat - current_time = time.time() - age = current_time - self.last_heartbeat - - # Thread is responsive if heartbeat is recent - return age < 30.0 # 30 second responsiveness threshold - - except Exception: - return False - - def _handle_restart_recovery(self, component: str, details: Dict[str, Any]) -> bool: - """Handle restart recovery action.""" - try: - log_info(self.camera_id, "Restarting FFmpeg RTSP reader for health recovery") - - # Stop current instance - self.stop() - - # Small delay - time.sleep(2.0) - - # Restart - self.start() - - # Report successful restart - stream_health_tracker.report_reconnect(self.camera_id, "health_recovery_restart") - self.ffmpeg_restart_count += 1 - - return True - - except Exception as e: - log_error(self.camera_id, f"Failed to restart FFmpeg RTSP reader: {e}") - return False - - def _handle_reconnect_recovery(self, component: str, details: Dict[str, Any]) -> bool: - """Handle reconnect recovery action.""" - try: - log_info(self.camera_id, "Reconnecting FFmpeg RTSP reader for health recovery") - - # Force restart FFmpeg process - self._restart_ffmpeg_process() - - # Reset error counters - self.consecutive_errors = 0 - stream_health_tracker.report_reconnect(self.camera_id, "health_recovery_reconnect") - - return True - - except Exception as e: - log_error(self.camera_id, f"Failed to reconnect FFmpeg RTSP reader: {e}") - return False \ No newline at end of file diff --git a/core/streaming/readers/http_snapshot.py b/core/streaming/readers/http_snapshot.py deleted file mode 100644 index bbbf943..0000000 --- a/core/streaming/readers/http_snapshot.py +++ /dev/null @@ -1,378 +0,0 @@ -""" -HTTP snapshot reader optimized for 2560x1440 (2K) high quality images. -Enhanced with comprehensive health monitoring and automatic recovery. -""" -import cv2 -import logging -import time -import threading -import requests -import numpy as np -from typing import Optional, Callable, Dict, Any - -from .base import VideoReader -from .utils import log_success, log_warning, log_error, log_info -from ...monitoring.stream_health import stream_health_tracker -from ...monitoring.thread_health import thread_health_monitor -from ...monitoring.recovery import recovery_manager, RecoveryAction - -logger = logging.getLogger(__name__) - - -class HTTPSnapshotReader(VideoReader): - """HTTP snapshot reader optimized for 2560x1440 (2K) high quality images.""" - - def __init__(self, camera_id: str, snapshot_url: str, interval_ms: int = 5000, max_retries: int = 3): - super().__init__(camera_id, snapshot_url, max_retries) - self.snapshot_url = snapshot_url - self.interval_ms = interval_ms - self.stop_event = threading.Event() - self.thread = None - - # Expected snapshot specifications - self.expected_width = 2560 - self.expected_height = 1440 - self.max_file_size = 10 * 1024 * 1024 # 10MB max for 2K image - - # Health monitoring setup - self.last_heartbeat = time.time() - self.consecutive_errors = 0 - self.connection_test_interval = 300 # Test connection every 5 minutes - self.last_connection_test = None - - # Register recovery handlers - recovery_manager.register_recovery_handler( - RecoveryAction.RESTART_STREAM, - self._handle_restart_recovery - ) - recovery_manager.register_recovery_handler( - RecoveryAction.RECONNECT, - self._handle_reconnect_recovery - ) - - @property - def is_running(self) -> bool: - """Check if the reader is currently running.""" - return self.thread is not None and self.thread.is_alive() - - @property - def reader_type(self) -> str: - """Get the type of reader.""" - return "http_snapshot" - - def set_frame_callback(self, callback: Callable[[str, np.ndarray], None]): - """Set callback function to handle captured frames.""" - self.frame_callback = callback - - def start(self): - """Start the snapshot reader thread.""" - if self.thread and self.thread.is_alive(): - logger.warning(f"Snapshot reader for {self.camera_id} already running") - return - - self.stop_event.clear() - self.thread = threading.Thread(target=self._read_snapshots, daemon=True) - self.thread.start() - - # Register with health monitoring - stream_health_tracker.register_stream(self.camera_id, "http_snapshot", self.snapshot_url) - thread_health_monitor.register_thread(self.thread, self._heartbeat_callback) - - logger.info(f"Started snapshot reader for camera {self.camera_id} with health monitoring") - - def stop(self): - """Stop the snapshot reader thread.""" - self.stop_event.set() - - # Unregister from health monitoring - if self.thread: - thread_health_monitor.unregister_thread(self.thread.ident) - self.thread.join(timeout=5.0) - - stream_health_tracker.unregister_stream(self.camera_id) - - logger.info(f"Stopped snapshot reader for camera {self.camera_id}") - - def _read_snapshots(self): - """Main snapshot reading loop for high quality 2K images.""" - retries = 0 - frame_count = 0 - last_log_time = time.time() - last_connection_test = time.time() - interval_seconds = self.interval_ms / 1000.0 - - logger.info(f"Snapshot interval for camera {self.camera_id}: {interval_seconds}s") - - while not self.stop_event.is_set(): - try: - # Send heartbeat for thread health monitoring - self._send_heartbeat("fetching_snapshot") - - start_time = time.time() - frame = self._fetch_snapshot() - - if frame is None: - retries += 1 - self.consecutive_errors += 1 - - # Report error to health monitoring - stream_health_tracker.report_error( - self.camera_id, - f"Failed to fetch snapshot (retry {retries}/{self.max_retries})" - ) - - logger.warning(f"Failed to fetch snapshot for camera {self.camera_id}, retry {retries}/{self.max_retries}") - - if self.max_retries != -1 and retries > self.max_retries: - logger.error(f"Max retries reached for snapshot camera {self.camera_id}") - break - - time.sleep(min(2.0, interval_seconds)) - continue - - # Accept any valid image dimensions - don't force specific resolution - if frame.shape[1] <= 0 or frame.shape[0] <= 0: - logger.warning(f"Camera {self.camera_id}: Invalid frame dimensions {frame.shape[1]}x{frame.shape[0]}") - stream_health_tracker.report_error( - self.camera_id, - f"Invalid frame dimensions: {frame.shape[1]}x{frame.shape[0]}" - ) - continue - - # Reset retry counter on successful fetch - retries = 0 - self.consecutive_errors = 0 - frame_count += 1 - - # Report successful frame to health monitoring - frame_size = frame.nbytes - stream_health_tracker.report_frame_received(self.camera_id, frame_size) - - # Call frame callback - if self.frame_callback: - try: - self.frame_callback(self.camera_id, frame) - except Exception as e: - logger.error(f"Camera {self.camera_id}: Frame callback error: {e}") - stream_health_tracker.report_error(self.camera_id, f"Frame callback error: {e}") - - # Periodic connection health test - current_time = time.time() - if current_time - last_connection_test >= self.connection_test_interval: - self._test_connection_health() - last_connection_test = current_time - - # Log progress every 30 seconds - if current_time - last_log_time >= 30: - logger.info(f"Camera {self.camera_id}: {frame_count} snapshots processed") - last_log_time = current_time - - # Wait for next interval - elapsed = time.time() - start_time - sleep_time = max(0, interval_seconds - elapsed) - if sleep_time > 0: - self.stop_event.wait(sleep_time) - - except Exception as e: - logger.error(f"Error in snapshot loop for camera {self.camera_id}: {e}") - stream_health_tracker.report_error(self.camera_id, f"Snapshot loop error: {e}") - retries += 1 - if self.max_retries != -1 and retries > self.max_retries: - break - time.sleep(min(2.0, interval_seconds)) - - logger.info(f"Snapshot reader thread ended for camera {self.camera_id}") - - def _fetch_snapshot(self) -> Optional[np.ndarray]: - """Fetch a single high quality snapshot from HTTP URL.""" - try: - # Parse URL for authentication - from urllib.parse import urlparse - parsed_url = urlparse(self.snapshot_url) - - headers = { - 'User-Agent': 'Python-Detector-Worker/1.0', - 'Accept': 'image/jpeg, image/png, image/*' - } - auth = None - - if parsed_url.username and parsed_url.password: - from requests.auth import HTTPBasicAuth, HTTPDigestAuth - auth = HTTPBasicAuth(parsed_url.username, parsed_url.password) - - # Reconstruct URL without credentials - clean_url = f"{parsed_url.scheme}://{parsed_url.hostname}" - if parsed_url.port: - clean_url += f":{parsed_url.port}" - clean_url += parsed_url.path - if parsed_url.query: - clean_url += f"?{parsed_url.query}" - - # Try Basic Auth first - response = requests.get(clean_url, auth=auth, timeout=15, headers=headers, - stream=True, verify=False) - - # If Basic Auth fails, try Digest Auth - if response.status_code == 401: - auth = HTTPDigestAuth(parsed_url.username, parsed_url.password) - response = requests.get(clean_url, auth=auth, timeout=15, headers=headers, - stream=True, verify=False) - else: - response = requests.get(self.snapshot_url, timeout=15, headers=headers, - stream=True, verify=False) - - if response.status_code == 200: - # Check content size - content_length = int(response.headers.get('content-length', 0)) - if content_length > self.max_file_size: - logger.warning(f"Snapshot too large for camera {self.camera_id}: {content_length} bytes") - return None - - # Read content - content = response.content - - # Convert to numpy array - image_array = np.frombuffer(content, np.uint8) - - # Decode as high quality image - frame = cv2.imdecode(image_array, cv2.IMREAD_COLOR) - - if frame is None: - logger.error(f"Failed to decode snapshot for camera {self.camera_id}") - return None - - logger.debug(f"Fetched snapshot for camera {self.camera_id}: {frame.shape[1]}x{frame.shape[0]}") - return frame - else: - logger.warning(f"HTTP {response.status_code} from {self.camera_id}") - return None - - except requests.RequestException as e: - logger.error(f"Request error fetching snapshot for {self.camera_id}: {e}") - return None - except Exception as e: - logger.error(f"Error decoding snapshot for {self.camera_id}: {e}") - return None - - def fetch_single_snapshot(self) -> Optional[np.ndarray]: - """ - Fetch a single high-quality snapshot on demand for pipeline processing. - This method is for one-time fetch from HTTP URL, not continuous streaming. - - Returns: - High quality 2K snapshot frame or None if failed - """ - logger.info(f"[SNAPSHOT] Fetching snapshot for {self.camera_id} from {self.snapshot_url}") - - # Try to fetch snapshot with retries - for attempt in range(self.max_retries): - frame = self._fetch_snapshot() - - if frame is not None: - logger.info(f"[SNAPSHOT] Successfully fetched {frame.shape[1]}x{frame.shape[0]} snapshot for {self.camera_id}") - return frame - - if attempt < self.max_retries - 1: - logger.warning(f"[SNAPSHOT] Attempt {attempt + 1}/{self.max_retries} failed for {self.camera_id}, retrying...") - time.sleep(0.5) - - logger.error(f"[SNAPSHOT] Failed to fetch snapshot for {self.camera_id} after {self.max_retries} attempts") - return None - - def _resize_maintain_aspect(self, frame: np.ndarray, target_width: int, target_height: int) -> np.ndarray: - """Resize image while maintaining aspect ratio for high quality.""" - h, w = frame.shape[:2] - aspect = w / h - target_aspect = target_width / target_height - - if aspect > target_aspect: - # Image is wider - new_width = target_width - new_height = int(target_width / aspect) - else: - # Image is taller - new_height = target_height - new_width = int(target_height * aspect) - - # Use INTER_LANCZOS4 for high quality downsampling - resized = cv2.resize(frame, (new_width, new_height), interpolation=cv2.INTER_LANCZOS4) - - # Pad to target size if needed - if new_width < target_width or new_height < target_height: - top = (target_height - new_height) // 2 - bottom = target_height - new_height - top - left = (target_width - new_width) // 2 - right = target_width - new_width - left - resized = cv2.copyMakeBorder(resized, top, bottom, left, right, cv2.BORDER_CONSTANT, value=[0, 0, 0]) - - return resized - - # Health monitoring methods - def _send_heartbeat(self, activity: str = "running"): - """Send heartbeat to thread health monitor.""" - self.last_heartbeat = time.time() - thread_health_monitor.heartbeat(activity=activity) - - def _heartbeat_callback(self) -> bool: - """Heartbeat callback for thread responsiveness testing.""" - try: - # Check if thread is responsive by checking recent heartbeat - current_time = time.time() - age = current_time - self.last_heartbeat - - # Thread is responsive if heartbeat is recent - return age < 30.0 # 30 second responsiveness threshold - - except Exception: - return False - - def _test_connection_health(self): - """Test HTTP connection health.""" - try: - stream_health_tracker.test_http_connection(self.camera_id, self.snapshot_url) - except Exception as e: - logger.error(f"Error testing connection health for {self.camera_id}: {e}") - - def _handle_restart_recovery(self, component: str, details: Dict[str, Any]) -> bool: - """Handle restart recovery action.""" - try: - logger.info(f"Restarting HTTP snapshot reader for {self.camera_id}") - - # Stop current instance - self.stop() - - # Small delay - time.sleep(2.0) - - # Restart - self.start() - - # Report successful restart - stream_health_tracker.report_reconnect(self.camera_id, "health_recovery_restart") - - return True - - except Exception as e: - logger.error(f"Failed to restart HTTP snapshot reader for {self.camera_id}: {e}") - return False - - def _handle_reconnect_recovery(self, component: str, details: Dict[str, Any]) -> bool: - """Handle reconnect recovery action.""" - try: - logger.info(f"Reconnecting HTTP snapshot reader for {self.camera_id}") - - # Test connection first - success = stream_health_tracker.test_http_connection(self.camera_id, self.snapshot_url) - - if success: - # Reset error counters - self.consecutive_errors = 0 - stream_health_tracker.report_reconnect(self.camera_id, "health_recovery_reconnect") - return True - else: - logger.warning(f"Connection test failed during recovery for {self.camera_id}") - return False - - except Exception as e: - logger.error(f"Failed to reconnect HTTP snapshot reader for {self.camera_id}: {e}") - return False \ No newline at end of file diff --git a/core/streaming/readers/utils.py b/core/streaming/readers/utils.py deleted file mode 100644 index 813f49f..0000000 --- a/core/streaming/readers/utils.py +++ /dev/null @@ -1,38 +0,0 @@ -""" -Utility functions for stream readers. -""" -import logging -import os - -# Keep OpenCV errors visible but allow FFmpeg stderr logging -os.environ["OPENCV_LOG_LEVEL"] = "ERROR" - -logger = logging.getLogger(__name__) - -# Color codes for pretty logging -class Colors: - GREEN = '\033[92m' - YELLOW = '\033[93m' - RED = '\033[91m' - BLUE = '\033[94m' - PURPLE = '\033[95m' - CYAN = '\033[96m' - WHITE = '\033[97m' - BOLD = '\033[1m' - END = '\033[0m' - -def log_success(camera_id: str, message: str): - """Log success messages in green""" - logger.info(f"{Colors.GREEN}[{camera_id}] {message}{Colors.END}") - -def log_warning(camera_id: str, message: str): - """Log warnings in yellow""" - logger.warning(f"{Colors.YELLOW}[{camera_id}] {message}{Colors.END}") - -def log_error(camera_id: str, message: str): - """Log errors in red""" - logger.error(f"{Colors.RED}[{camera_id}] {message}{Colors.END}") - -def log_info(camera_id: str, message: str): - """Log info in cyan""" - logger.info(f"{Colors.CYAN}[{camera_id}] {message}{Colors.END}") \ No newline at end of file diff --git a/core/tracking/__init__.py b/core/tracking/__init__.py deleted file mode 100644 index a493062..0000000 --- a/core/tracking/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Tracking module for vehicle tracking and validation - -from .tracker import VehicleTracker, TrackedVehicle -from .validator import StableCarValidator, ValidationResult, VehicleState -from .integration import TrackingPipelineIntegration - -__all__ = [ - 'VehicleTracker', - 'TrackedVehicle', - 'StableCarValidator', - 'ValidationResult', - 'VehicleState', - 'TrackingPipelineIntegration' -] \ No newline at end of file diff --git a/core/tracking/bot_sort_tracker.py b/core/tracking/bot_sort_tracker.py deleted file mode 100644 index f487a6a..0000000 --- a/core/tracking/bot_sort_tracker.py +++ /dev/null @@ -1,408 +0,0 @@ -""" -BoT-SORT Multi-Object Tracker with Camera Isolation -Based on BoT-SORT: Robust Associations Multi-Pedestrian Tracking -""" - -import logging -import time -import numpy as np -from typing import Dict, List, Optional, Tuple, Any -from dataclasses import dataclass -from scipy.optimize import linear_sum_assignment -from filterpy.kalman import KalmanFilter -import cv2 - -logger = logging.getLogger(__name__) - - -@dataclass -class TrackState: - """Track state enumeration""" - TENTATIVE = "tentative" # New track, not confirmed yet - CONFIRMED = "confirmed" # Confirmed track - DELETED = "deleted" # Track to be deleted - - -class Track: - """ - Individual track representation with Kalman filter for motion prediction - """ - - def __init__(self, detection, track_id: int, camera_id: str): - """ - Initialize a new track - - Args: - detection: Initial detection (bbox, confidence, class) - track_id: Unique track identifier within camera - camera_id: Camera identifier - """ - self.track_id = track_id - self.camera_id = camera_id - self.state = TrackState.TENTATIVE - - # Time tracking - self.start_time = time.time() - self.last_update_time = time.time() - - # Appearance and motion - self.bbox = detection.bbox # [x1, y1, x2, y2] - self.confidence = detection.confidence - self.class_name = detection.class_name - - # Track management - self.hit_streak = 1 - self.time_since_update = 0 - self.age = 1 - - # Kalman filter for motion prediction - self.kf = self._create_kalman_filter() - self._update_kalman_filter(detection.bbox) - - # Track history - self.history = [detection.bbox] - self.max_history = 10 - - def _create_kalman_filter(self) -> KalmanFilter: - """Create Kalman filter for bbox tracking (x, y, w, h, vx, vy, vw, vh)""" - kf = KalmanFilter(dim_x=8, dim_z=4) - - # State transition matrix (constant velocity model) - kf.F = np.array([ - [1, 0, 0, 0, 1, 0, 0, 0], - [0, 1, 0, 0, 0, 1, 0, 0], - [0, 0, 1, 0, 0, 0, 1, 0], - [0, 0, 0, 1, 0, 0, 0, 1], - [0, 0, 0, 0, 1, 0, 0, 0], - [0, 0, 0, 0, 0, 1, 0, 0], - [0, 0, 0, 0, 0, 0, 1, 0], - [0, 0, 0, 0, 0, 0, 0, 1] - ]) - - # Measurement matrix (observe x, y, w, h) - kf.H = np.array([ - [1, 0, 0, 0, 0, 0, 0, 0], - [0, 1, 0, 0, 0, 0, 0, 0], - [0, 0, 1, 0, 0, 0, 0, 0], - [0, 0, 0, 1, 0, 0, 0, 0] - ]) - - # Process noise - kf.Q *= 0.01 - - # Measurement noise - kf.R *= 10 - - # Initial covariance - kf.P *= 100 - - return kf - - def _update_kalman_filter(self, bbox: List[float]): - """Update Kalman filter with new bbox""" - # Convert [x1, y1, x2, y2] to [cx, cy, w, h] - x1, y1, x2, y2 = bbox - cx = (x1 + x2) / 2 - cy = (y1 + y2) / 2 - w = x2 - x1 - h = y2 - y1 - - # Properly assign to column vector - self.kf.x[:4, 0] = [cx, cy, w, h] - - def predict(self) -> np.ndarray: - """Predict next position using Kalman filter""" - self.kf.predict() - - # Convert back to [x1, y1, x2, y2] format - cx, cy, w, h = self.kf.x[:4, 0] # Extract from column vector - x1 = cx - w/2 - y1 = cy - h/2 - x2 = cx + w/2 - y2 = cy + h/2 - - return np.array([x1, y1, x2, y2]) - - def update(self, detection): - """Update track with new detection""" - self.last_update_time = time.time() - self.time_since_update = 0 - self.hit_streak += 1 - self.age += 1 - - # Update track properties - self.bbox = detection.bbox - self.confidence = detection.confidence - - # Update Kalman filter - x1, y1, x2, y2 = detection.bbox - cx = (x1 + x2) / 2 - cy = (y1 + y2) / 2 - w = x2 - x1 - h = y2 - y1 - - self.kf.update([cx, cy, w, h]) - - # Update history - self.history.append(detection.bbox) - if len(self.history) > self.max_history: - self.history.pop(0) - - # Update state - if self.state == TrackState.TENTATIVE and self.hit_streak >= 3: - self.state = TrackState.CONFIRMED - - def mark_missed(self): - """Mark track as missed in this frame""" - self.time_since_update += 1 - self.age += 1 - - if self.time_since_update > 5: # Delete after 5 missed frames - self.state = TrackState.DELETED - - def is_confirmed(self) -> bool: - """Check if track is confirmed""" - return self.state == TrackState.CONFIRMED - - def is_deleted(self) -> bool: - """Check if track should be deleted""" - return self.state == TrackState.DELETED - - -class CameraTracker: - """ - BoT-SORT tracker for a single camera - """ - - def __init__(self, camera_id: str, max_disappeared: int = 10): - """ - Initialize camera tracker - - Args: - camera_id: Unique camera identifier - max_disappeared: Maximum frames a track can be missed before deletion - """ - self.camera_id = camera_id - self.max_disappeared = max_disappeared - - # Track management - self.tracks: Dict[int, Track] = {} - self.next_id = 1 - self.frame_count = 0 - - logger.info(f"Initialized BoT-SORT tracker for camera {camera_id}") - - def update(self, detections: List) -> List[Track]: - """ - Update tracker with new detections - - Args: - detections: List of Detection objects - - Returns: - List of active confirmed tracks - """ - self.frame_count += 1 - - # Predict all existing tracks - for track in self.tracks.values(): - track.predict() - - # Associate detections to tracks - matched_tracks, unmatched_detections, unmatched_tracks = self._associate(detections) - - # Update matched tracks - for track_id, detection in matched_tracks: - self.tracks[track_id].update(detection) - - # Mark unmatched tracks as missed - for track_id in unmatched_tracks: - self.tracks[track_id].mark_missed() - - # Create new tracks for unmatched detections - for detection in unmatched_detections: - track = Track(detection, self.next_id, self.camera_id) - self.tracks[self.next_id] = track - self.next_id += 1 - - # Remove deleted tracks - tracks_to_remove = [tid for tid, track in self.tracks.items() if track.is_deleted()] - for tid in tracks_to_remove: - del self.tracks[tid] - - # Return confirmed tracks - confirmed_tracks = [track for track in self.tracks.values() if track.is_confirmed()] - - return confirmed_tracks - - def _associate(self, detections: List) -> Tuple[List[Tuple[int, Any]], List[Any], List[int]]: - """ - Associate detections to existing tracks using IoU distance - - Returns: - (matched_tracks, unmatched_detections, unmatched_tracks) - """ - if not detections or not self.tracks: - return [], detections, list(self.tracks.keys()) - - # Calculate IoU distance matrix - track_ids = list(self.tracks.keys()) - cost_matrix = np.zeros((len(track_ids), len(detections))) - - for i, track_id in enumerate(track_ids): - track = self.tracks[track_id] - predicted_bbox = track.predict() - - for j, detection in enumerate(detections): - iou = self._calculate_iou(predicted_bbox, detection.bbox) - cost_matrix[i, j] = 1 - iou # Convert IoU to distance - - # Solve assignment problem - row_indices, col_indices = linear_sum_assignment(cost_matrix) - - # Filter matches by IoU threshold - iou_threshold = 0.3 - matched_tracks = [] - matched_detection_indices = set() - matched_track_indices = set() - - for row, col in zip(row_indices, col_indices): - if cost_matrix[row, col] <= (1 - iou_threshold): - track_id = track_ids[row] - detection = detections[col] - matched_tracks.append((track_id, detection)) - matched_detection_indices.add(col) - matched_track_indices.add(row) - - # Find unmatched detections and tracks - unmatched_detections = [detections[i] for i in range(len(detections)) - if i not in matched_detection_indices] - unmatched_tracks = [track_ids[i] for i in range(len(track_ids)) - if i not in matched_track_indices] - - return matched_tracks, unmatched_detections, unmatched_tracks - - def _calculate_iou(self, bbox1: np.ndarray, bbox2: List[float]) -> float: - """Calculate IoU between two bounding boxes""" - x1_1, y1_1, x2_1, y2_1 = bbox1 - x1_2, y1_2, x2_2, y2_2 = bbox2 - - # Calculate intersection area - x1_i = max(x1_1, x1_2) - y1_i = max(y1_1, y1_2) - x2_i = min(x2_1, x2_2) - y2_i = min(y2_1, y2_2) - - if x2_i <= x1_i or y2_i <= y1_i: - return 0.0 - - intersection = (x2_i - x1_i) * (y2_i - y1_i) - - # Calculate union area - area1 = (x2_1 - x1_1) * (y2_1 - y1_1) - area2 = (x2_2 - x1_2) * (y2_2 - y1_2) - union = area1 + area2 - intersection - - return intersection / union if union > 0 else 0.0 - - -class MultiCameraBoTSORT: - """ - Multi-camera BoT-SORT tracker with complete camera isolation - """ - - def __init__(self, trigger_classes: List[str], min_confidence: float = 0.6): - """ - Initialize multi-camera tracker - - Args: - trigger_classes: List of class names to track - min_confidence: Minimum detection confidence threshold - """ - self.trigger_classes = trigger_classes - self.min_confidence = min_confidence - - # Camera-specific trackers - self.camera_trackers: Dict[str, CameraTracker] = {} - - logger.info(f"Initialized MultiCameraBoTSORT with classes={trigger_classes}, " - f"min_confidence={min_confidence}") - - def get_or_create_tracker(self, camera_id: str) -> CameraTracker: - """Get or create tracker for specific camera""" - if camera_id not in self.camera_trackers: - self.camera_trackers[camera_id] = CameraTracker(camera_id) - logger.info(f"Created new tracker for camera {camera_id}") - - return self.camera_trackers[camera_id] - - def update(self, camera_id: str, inference_result) -> List[Dict]: - """ - Update tracker for specific camera with detections - - Args: - camera_id: Camera identifier - inference_result: InferenceResult with detections - - Returns: - List of track information dictionaries - """ - # Filter detections by confidence and trigger classes - filtered_detections = [] - - if hasattr(inference_result, 'detections') and inference_result.detections: - for detection in inference_result.detections: - if (detection.confidence >= self.min_confidence and - detection.class_name in self.trigger_classes): - filtered_detections.append(detection) - - # Get camera tracker and update - tracker = self.get_or_create_tracker(camera_id) - confirmed_tracks = tracker.update(filtered_detections) - - # Convert tracks to output format - track_results = [] - for track in confirmed_tracks: - track_results.append({ - 'track_id': track.track_id, - 'camera_id': track.camera_id, - 'bbox': track.bbox, - 'confidence': track.confidence, - 'class_name': track.class_name, - 'hit_streak': track.hit_streak, - 'age': track.age - }) - - return track_results - - def get_statistics(self) -> Dict[str, Any]: - """Get tracking statistics across all cameras""" - stats = {} - total_tracks = 0 - - for camera_id, tracker in self.camera_trackers.items(): - camera_stats = { - 'active_tracks': len([t for t in tracker.tracks.values() if t.is_confirmed()]), - 'total_tracks': len(tracker.tracks), - 'frame_count': tracker.frame_count - } - stats[camera_id] = camera_stats - total_tracks += camera_stats['active_tracks'] - - stats['summary'] = { - 'total_cameras': len(self.camera_trackers), - 'total_active_tracks': total_tracks - } - - return stats - - def reset_camera(self, camera_id: str): - """Reset tracking for specific camera""" - if camera_id in self.camera_trackers: - del self.camera_trackers[camera_id] - logger.info(f"Reset tracking for camera {camera_id}") - - def reset_all(self): - """Reset all camera trackers""" - self.camera_trackers.clear() - logger.info("Reset all camera trackers") \ No newline at end of file diff --git a/core/tracking/integration.py b/core/tracking/integration.py deleted file mode 100644 index 2fba002..0000000 --- a/core/tracking/integration.py +++ /dev/null @@ -1,858 +0,0 @@ -""" -Tracking-Pipeline Integration Module. -Connects the tracking system with the main detection pipeline and manages the flow. -""" -import logging -import time -import uuid -from typing import Dict, Optional, Any, List, Tuple -from concurrent.futures import ThreadPoolExecutor -import asyncio -import numpy as np - -from .tracker import VehicleTracker, TrackedVehicle -from .validator import StableCarValidator -from ..models.inference import YOLOWrapper -from ..models.pipeline import PipelineParser -from ..detection.pipeline import DetectionPipeline - -logger = logging.getLogger(__name__) - - -class TrackingPipelineIntegration: - """ - Integrates vehicle tracking with the detection pipeline. - Manages tracking state transitions and pipeline execution triggers. - """ - - def __init__(self, pipeline_parser: PipelineParser, model_manager: Any, model_id: int, message_sender=None): - """ - Initialize tracking-pipeline integration. - - Args: - pipeline_parser: Pipeline parser with loaded configuration - model_manager: Model manager for loading models - model_id: The model ID to use for loading models - message_sender: Optional callback function for sending WebSocket messages - """ - self.pipeline_parser = pipeline_parser - self.model_manager = model_manager - self.model_id = model_id - self.message_sender = message_sender - - # Store subscription info for snapshot access - self.subscription_info = None - - # Initialize tracking components - tracking_config = pipeline_parser.tracking_config.__dict__ if pipeline_parser.tracking_config else {} - self.tracker = VehicleTracker(tracking_config) - self.validator = StableCarValidator() - - # Tracking model - self.tracking_model: Optional[YOLOWrapper] = None - self.tracking_model_id = None - - # Detection pipeline (Phase 5) - self.detection_pipeline: Optional[DetectionPipeline] = None - - # Session management - self.active_sessions: Dict[str, str] = {} # display_id -> session_id - self.session_vehicles: Dict[str, int] = {} # session_id -> track_id - self.cleared_sessions: Dict[str, float] = {} # session_id -> clear_time - self.pending_vehicles: Dict[str, int] = {} # display_id -> track_id (waiting for session ID) - self.pending_processing_data: Dict[str, Dict] = {} # display_id -> processing data (waiting for session ID) - self.display_to_subscription: Dict[str, str] = {} # display_id -> subscription_id (for fallback) - - # Additional validators for enhanced flow control - self.permanently_processed: Dict[str, float] = {} # "camera_id:track_id" -> process_time (never process again) - self.progression_stages: Dict[str, str] = {} # session_id -> current_stage - self.last_detection_time: Dict[str, float] = {} # display_id -> last_detection_timestamp - self.abandonment_timeout = 3.0 # seconds to wait before declaring car abandoned - - # Thread pool for pipeline execution - self.executor = ThreadPoolExecutor(max_workers=2) - - # Min bbox filtering configuration - # TODO: Make this configurable via pipeline.json in the future - self.min_bbox_area_percentage = 3.5 # 3.5% of frame area minimum - - # Statistics - self.stats = { - 'frames_processed': 0, - 'vehicles_detected': 0, - 'vehicles_validated': 0, - 'pipelines_executed': 0, - 'frontals_filtered_small': 0 # Track filtered detections - } - - - logger.info("TrackingPipelineIntegration initialized") - - async def initialize_tracking_model(self) -> bool: - """ - Load and initialize the tracking model. - - Returns: - True if successful, False otherwise - """ - try: - if not self.pipeline_parser.tracking_config: - logger.warning("No tracking configuration found in pipeline") - return False - - model_file = self.pipeline_parser.tracking_config.model_file - model_id = self.pipeline_parser.tracking_config.model_id - - if not model_file: - logger.warning("No tracking model file specified") - return False - - # Load tracking model - logger.info(f"Loading tracking model: {model_id} ({model_file})") - self.tracking_model = self.model_manager.get_yolo_model(self.model_id, model_file) - if not self.tracking_model: - logger.error(f"Failed to load tracking model {model_file} from model {self.model_id}") - return False - self.tracking_model_id = model_id - - if self.tracking_model: - logger.info(f"Tracking model {model_id} loaded successfully") - - # Initialize detection pipeline (Phase 5) - await self._initialize_detection_pipeline() - - return True - else: - logger.error(f"Failed to load tracking model {model_id}") - return False - - except Exception as e: - logger.error(f"Error initializing tracking model: {e}", exc_info=True) - return False - - async def _initialize_detection_pipeline(self) -> bool: - """ - Initialize the detection pipeline for main detection processing. - - Returns: - True if successful, False otherwise - """ - try: - if not self.pipeline_parser: - logger.warning("No pipeline parser available for detection pipeline") - return False - - # Create detection pipeline with message sender capability - self.detection_pipeline = DetectionPipeline(self.pipeline_parser, self.model_manager, self.model_id, self.message_sender) - - # Initialize detection pipeline - if await self.detection_pipeline.initialize(): - logger.info("Detection pipeline initialized successfully") - return True - else: - logger.error("Failed to initialize detection pipeline") - return False - - except Exception as e: - logger.error(f"Error initializing detection pipeline: {e}", exc_info=True) - return False - - async def process_frame(self, - frame: np.ndarray, - display_id: str, - subscription_id: str, - session_id: Optional[str] = None) -> Dict[str, Any]: - """ - Process a frame through tracking and potentially the detection pipeline. - - Args: - frame: Input frame to process - display_id: Display identifier - subscription_id: Full subscription identifier - session_id: Optional session ID from backend - - Returns: - Dictionary with processing results - """ - start_time = time.time() - result = { - 'tracked_vehicles': [], - 'validated_vehicle': None, - 'pipeline_result': None, - 'session_id': session_id, - 'processing_time': 0.0 - } - - try: - # Update stats - self.stats['frames_processed'] += 1 - - # Run tracking model - if self.tracking_model: - # Run detection-only (tracking handled by our own tracker) - tracking_results = self.tracking_model.track( - frame, - confidence_threshold=self.tracker.min_confidence, - trigger_classes=self.tracker.trigger_classes, - persist=True - ) - - # Debug: Log raw detection results - if tracking_results and hasattr(tracking_results, 'detections'): - raw_detections = len(tracking_results.detections) - if raw_detections > 0: - class_names = [detection.class_name for detection in tracking_results.detections] - logger.debug(f"Raw detections: {raw_detections}, classes: {class_names}") - else: - logger.debug(f"No raw detections found") - else: - logger.debug(f"No tracking results or detections attribute") - - # Filter out small frontal detections (neighboring pumps/distant cars) - if tracking_results and hasattr(tracking_results, 'detections'): - tracking_results = self._filter_small_frontals(tracking_results, frame) - - # Process tracking results - tracked_vehicles = self.tracker.process_detections( - tracking_results, - display_id, - frame - ) - - # Update last detection time for abandonment detection - # Update when vehicles ARE detected, so when they leave, timestamp ages - if tracked_vehicles: - self.last_detection_time[display_id] = time.time() - logger.debug(f"Updated last_detection_time for {display_id}: {len(tracked_vehicles)} vehicles") - - # Check for car abandonment (vehicle left after getting car_wait_staff stage) - await self._check_car_abandonment(display_id, subscription_id) - - result['tracked_vehicles'] = [ - { - 'track_id': v.track_id, - 'bbox': v.bbox, - 'confidence': v.confidence, - 'is_stable': v.is_stable, - 'session_id': v.session_id - } - for v in tracked_vehicles - ] - - # Log tracking info periodically - if self.stats['frames_processed'] % 30 == 0: # Every 30 frames - logger.debug(f"Tracking: {len(tracked_vehicles)} vehicles, " - f"display={display_id}") - - # Get stable vehicles for validation - stable_vehicles = self.tracker.get_stable_vehicles(display_id) - - # Validate and potentially process stable vehicles - for vehicle in stable_vehicles: - # Check if vehicle is already processed or has session - if vehicle.processed_pipeline: - continue - - # Check for session cleared (post-fueling) - if session_id and vehicle.session_id == session_id: - # Same vehicle with same session, skip - continue - - # Check if this was a recently cleared session - session_cleared = False - if vehicle.session_id in self.cleared_sessions: - clear_time = self.cleared_sessions[vehicle.session_id] - if (time.time() - clear_time) < 30: # 30 second cooldown - session_cleared = True - - # Skip same car after session clear or if permanently processed - if self.validator.should_skip_same_car(vehicle, session_cleared, self.permanently_processed): - continue - - # Validate vehicle - validation_result = self.validator.validate_vehicle(vehicle, frame.shape) - - if validation_result.is_valid and validation_result.should_process: - logger.info(f"Vehicle {vehicle.track_id} validated for processing: " - f"{validation_result.reason}") - - result['validated_vehicle'] = { - 'track_id': vehicle.track_id, - 'state': validation_result.state.value, - 'confidence': validation_result.confidence - } - - # Execute detection pipeline - this will send real imageDetection when detection is found - - # Mark vehicle as pending session ID assignment - self.pending_vehicles[display_id] = vehicle.track_id - logger.info(f"Vehicle {vehicle.track_id} waiting for session ID from backend") - - # Execute detection pipeline (placeholder for Phase 5) - pipeline_result = await self._execute_pipeline( - frame, - vehicle, - display_id, - None, # No session ID yet - subscription_id - ) - - result['pipeline_result'] = pipeline_result - # No session_id in result yet - backend will provide it - self.stats['pipelines_executed'] += 1 - - # Only process one vehicle per frame - break - - self.stats['vehicles_detected'] = len(tracked_vehicles) - self.stats['vehicles_validated'] = len(stable_vehicles) - - else: - logger.warning("No tracking model available") - - except Exception as e: - logger.error(f"Error in tracking pipeline: {e}", exc_info=True) - - - result['processing_time'] = time.time() - start_time - return result - - async def _execute_pipeline(self, - frame: np.ndarray, - vehicle: TrackedVehicle, - display_id: str, - session_id: str, - subscription_id: str) -> Dict[str, Any]: - """ - Execute the main detection pipeline for a validated vehicle. - - Args: - frame: Input frame - vehicle: Validated tracked vehicle - display_id: Display identifier - session_id: Session identifier - subscription_id: Full subscription identifier - - Returns: - Pipeline execution results - """ - logger.info(f"Executing detection pipeline for vehicle {vehicle.track_id}, " - f"session={session_id}, display={display_id}") - - try: - # Check if detection pipeline is available - if not self.detection_pipeline: - logger.warning("Detection pipeline not initialized, using fallback") - return { - 'status': 'error', - 'message': 'Detection pipeline not available', - 'vehicle_id': vehicle.track_id, - 'session_id': session_id - } - - # Execute only the detection phase (first phase) - # This will run detection and send imageDetection message to backend - detection_result = await self.detection_pipeline.execute_detection_phase( - frame=frame, - display_id=display_id, - subscription_id=subscription_id - ) - - # Add vehicle information to result - detection_result['vehicle_id'] = vehicle.track_id - detection_result['vehicle_bbox'] = vehicle.bbox - detection_result['vehicle_confidence'] = vehicle.confidence - detection_result['phase'] = 'detection' - - logger.info(f"Detection phase executed for vehicle {vehicle.track_id}: " - f"status={detection_result.get('status', 'unknown')}, " - f"message_sent={detection_result.get('message_sent', False)}, " - f"processing_time={detection_result.get('processing_time', 0):.3f}s") - - # Store frame and detection results for processing phase - if detection_result['message_sent']: - # Store for later processing when sessionId is received - self.pending_processing_data[display_id] = { - 'frame': frame.copy(), # Store copy of frame for processing phase - 'vehicle': vehicle, - 'subscription_id': subscription_id, - 'detection_result': detection_result, - 'timestamp': time.time() - } - logger.info(f"Stored processing data for {display_id}, waiting for sessionId from backend") - - return detection_result - - except Exception as e: - logger.error(f"Error executing detection pipeline: {e}", exc_info=True) - return { - 'status': 'error', - 'message': str(e), - 'vehicle_id': vehicle.track_id, - 'session_id': session_id, - 'processing_time': 0.0 - } - - async def _execute_processing_phase(self, - processing_data: Dict[str, Any], - session_id: str, - display_id: str) -> None: - """ - Execute the processing phase after receiving sessionId from backend. - This includes branch processing and database operations. - - Args: - processing_data: Stored processing data from detection phase - session_id: Session ID from backend - display_id: Display identifier - """ - try: - vehicle = processing_data['vehicle'] - subscription_id = processing_data['subscription_id'] - detection_result = processing_data['detection_result'] - - logger.info(f"Executing processing phase for session {session_id}, vehicle {vehicle.track_id}") - - # Capture high-quality snapshot for pipeline processing - logger.info(f"[PROCESSING PHASE] Fetching 2K snapshot for session {session_id}") - frame = self._fetch_snapshot() - - if frame is None: - logger.warning(f"[PROCESSING PHASE] Failed to capture snapshot, falling back to RTSP frame") - # Fall back to RTSP frame if snapshot fails - frame = processing_data['frame'] - - # Extract detected regions from detection phase result if available - detected_regions = detection_result.get('detected_regions', {}) - logger.info(f"[INTEGRATION] Passing detected_regions to processing phase: {list(detected_regions.keys())}") - - # Execute processing phase with detection pipeline - if self.detection_pipeline: - processing_result = await self.detection_pipeline.execute_processing_phase( - frame=frame, - display_id=display_id, - session_id=session_id, - subscription_id=subscription_id, - detected_regions=detected_regions - ) - - logger.info(f"Processing phase completed for session {session_id}: " - f"status={processing_result.get('status', 'unknown')}, " - f"branches={len(processing_result.get('branch_results', {}))}, " - f"actions={len(processing_result.get('actions_executed', []))}, " - f"processing_time={processing_result.get('processing_time', 0):.3f}s") - - # Update stats - self.stats['pipelines_executed'] += 1 - - else: - logger.error("Detection pipeline not available for processing phase") - - except Exception as e: - logger.error(f"Error in processing phase for session {session_id}: {e}", exc_info=True) - - - def set_subscription_info(self, subscription_info): - """ - Set subscription info to access snapshot URL and other stream details. - - Args: - subscription_info: SubscriptionInfo object containing stream config - """ - self.subscription_info = subscription_info - logger.debug(f"Set subscription info with snapshot_url: {subscription_info.stream_config.snapshot_url if subscription_info else None}") - - def set_session_id(self, display_id: str, session_id: str, subscription_id: str = None): - """ - Set session ID for a display (from backend). - This is called when backend sends setSessionId after receiving imageDetection. - - Args: - display_id: Display identifier - session_id: Session identifier - subscription_id: Subscription identifier (displayId;cameraId) - needed for fallback - """ - # Ensure session_id is always a string for consistent type handling - session_id = str(session_id) if session_id is not None else None - self.active_sessions[display_id] = session_id - - # Store subscription_id for fallback usage - if subscription_id: - self.display_to_subscription[display_id] = subscription_id - logger.info(f"Set session {session_id} for display {display_id} with subscription {subscription_id}") - else: - logger.info(f"Set session {session_id} for display {display_id}") - - # Check if we have a pending vehicle for this display - if display_id in self.pending_vehicles: - track_id = self.pending_vehicles[display_id] - - # Mark vehicle as processed with the session ID - self.tracker.mark_processed(track_id, session_id) - self.session_vehicles[session_id] = track_id - - # Mark vehicle as permanently processed (won't process again even after session clear) - # Use composite key to distinguish same track IDs across different cameras - camera_id = display_id # Using display_id as camera_id for isolation - permanent_key = f"{camera_id}:{track_id}" - self.permanently_processed[permanent_key] = time.time() - - # Remove from pending - del self.pending_vehicles[display_id] - - logger.info(f"Assigned session {session_id} to vehicle {track_id}, marked as permanently processed") - else: - logger.warning(f"No pending vehicle found for display {display_id} when setting session {session_id}") - - # Check if we have pending processing data for this display - if display_id in self.pending_processing_data: - processing_data = self.pending_processing_data[display_id] - - # Trigger the processing phase asynchronously - asyncio.create_task(self._execute_processing_phase( - processing_data=processing_data, - session_id=session_id, - display_id=display_id - )) - - # Remove from pending processing - del self.pending_processing_data[display_id] - - logger.info(f"Triggered processing phase for session {session_id} on display {display_id}") - else: - logger.warning(f"No pending processing data found for display {display_id} when setting session {session_id}") - - # FALLBACK: Execute pipeline for POS-initiated sessions - # Skip if session_id is None (no car present or car has left) - if session_id is not None: - # Use stored subscription_id instead of creating fake one - stored_subscription_id = self.display_to_subscription.get(display_id) - if stored_subscription_id: - logger.info(f"[FALLBACK] Triggering fallback pipeline for session {session_id} on display {display_id} with subscription {stored_subscription_id}") - - # Trigger the fallback pipeline asynchronously with real subscription_id - asyncio.create_task(self._execute_fallback_pipeline( - display_id=display_id, - session_id=session_id, - subscription_id=stored_subscription_id - )) - else: - logger.error(f"[FALLBACK] No subscription_id stored for display {display_id}, cannot execute fallback pipeline") - else: - logger.debug(f"[FALLBACK] Skipping pipeline execution for session_id=None on display {display_id}") - - def clear_session_id(self, session_id: str): - """ - Clear session ID (post-fueling). - - Args: - session_id: Session identifier to clear - """ - # Mark session as cleared - self.cleared_sessions[session_id] = time.time() - - # Clear from tracker - self.tracker.clear_session(session_id) - - # Remove from active sessions - display_to_remove = None - for display_id, sess_id in self.active_sessions.items(): - if sess_id == session_id: - display_to_remove = display_id - break - - if display_to_remove: - del self.active_sessions[display_to_remove] - - if session_id in self.session_vehicles: - del self.session_vehicles[session_id] - - logger.info(f"Cleared session {session_id}") - - # Clean old cleared sessions (older than 5 minutes) - current_time = time.time() - old_sessions = [ - sid for sid, clear_time in self.cleared_sessions.items() - if (current_time - clear_time) > 300 - ] - for sid in old_sessions: - del self.cleared_sessions[sid] - - def get_session_for_display(self, display_id: str) -> Optional[str]: - """Get active session for a display.""" - return self.active_sessions.get(display_id) - - def reset_tracking(self): - """Reset all tracking state.""" - self.tracker.reset_tracking() - self.active_sessions.clear() - self.session_vehicles.clear() - self.cleared_sessions.clear() - self.pending_vehicles.clear() - self.pending_processing_data.clear() - self.display_to_subscription.clear() - self.permanently_processed.clear() - self.progression_stages.clear() - self.last_detection_time.clear() - logger.info("Tracking pipeline integration reset") - - def get_statistics(self) -> Dict[str, Any]: - """Get comprehensive statistics.""" - tracker_stats = self.tracker.get_statistics() - validator_stats = self.validator.get_statistics() - - return { - 'integration': self.stats, - 'tracker': tracker_stats, - 'validator': validator_stats, - 'active_sessions': len(self.active_sessions), - 'cleared_sessions': len(self.cleared_sessions) - } - - async def _check_car_abandonment(self, display_id: str, subscription_id: str): - """ - Check if a car has abandoned the fueling process (left after getting car_wait_staff stage). - - Args: - display_id: Display identifier - subscription_id: Subscription identifier - """ - current_time = time.time() - - # Check all sessions in car_wait_staff stage - abandoned_sessions = [] - for session_id, stage in self.progression_stages.items(): - if stage == "car_wait_staff": - # Check if we have recent detections for this session's display - session_display = None - for disp_id, sess_id in self.active_sessions.items(): - if sess_id == session_id: - session_display = disp_id - break - - if session_display: - last_detection = self.last_detection_time.get(session_display, 0) - time_since_detection = current_time - last_detection - - logger.info(f"[ABANDON CHECK] Session {session_id} (display: {session_display}): " - f"time_since_detection={time_since_detection:.1f}s, " - f"timeout={self.abandonment_timeout}s") - - if time_since_detection > self.abandonment_timeout: - logger.warning(f"🚨 Car abandonment detected: session {session_id}, " - f"no detection for {time_since_detection:.1f}s") - abandoned_sessions.append(session_id) - else: - logger.debug(f"[ABANDON CHECK] Session {session_id} has no associated display") - - # Send abandonment detection for each abandoned session - for session_id in abandoned_sessions: - await self._send_abandonment_detection(subscription_id, session_id) - # Remove from progression stages to avoid repeated detection - if session_id in self.progression_stages: - del self.progression_stages[session_id] - logger.info(f"[ABANDON] Removed session {session_id} from progression_stages after notification") - - async def _send_abandonment_detection(self, subscription_id: str, session_id: str): - """ - Send imageDetection with null detection to indicate car abandonment. - - Args: - subscription_id: Subscription identifier - session_id: Session ID of the abandoned car - """ - try: - # Import here to avoid circular imports - from ..communication.messages import create_image_detection - - # Create abandonment detection message with null detection - detection_message = create_image_detection( - subscription_identifier=subscription_id, - detection_data=None, # Null detection indicates abandonment - model_id=self.model_id, - model_name=self.pipeline_parser.tracking_config.model_id if self.pipeline_parser.tracking_config else "tracking_model" - ) - - # Send to backend via WebSocket if sender is available - if self.message_sender: - await self.message_sender(detection_message) - logger.info(f"[CAR ABANDONMENT] Sent null detection for session {session_id}") - else: - logger.info(f"[CAR ABANDONMENT] No message sender available, would send: {detection_message}") - - except Exception as e: - logger.error(f"Error sending abandonment detection: {e}", exc_info=True) - - def set_progression_stage(self, session_id: str, stage: str): - """ - Set progression stage for a session (from backend setProgessionStage message). - - Args: - session_id: Session identifier - stage: Progression stage (e.g., "car_wait_staff") - """ - self.progression_stages[session_id] = stage - logger.info(f"Set progression stage for session {session_id}: {stage}") - - # If car reaches car_wait_staff, start monitoring for abandonment - if stage == "car_wait_staff": - logger.info(f"Started monitoring session {session_id} for car abandonment") - - def _fetch_snapshot(self) -> Optional[np.ndarray]: - """ - Fetch high-quality snapshot from camera's snapshot URL. - Reusable method for both processing phase and fallback pipeline. - - Returns: - Snapshot frame or None if unavailable - """ - if not (self.subscription_info and self.subscription_info.stream_config.snapshot_url): - logger.warning("[SNAPSHOT] No subscription info or snapshot URL available") - return None - - try: - from ..streaming.readers import HTTPSnapshotReader - - logger.info(f"[SNAPSHOT] Fetching snapshot for {self.subscription_info.camera_id}") - snapshot_reader = HTTPSnapshotReader( - camera_id=self.subscription_info.camera_id, - snapshot_url=self.subscription_info.stream_config.snapshot_url, - max_retries=3 - ) - - frame = snapshot_reader.fetch_single_snapshot() - - if frame is not None: - logger.info(f"[SNAPSHOT] Successfully fetched {frame.shape[1]}x{frame.shape[0]} snapshot") - return frame - else: - logger.warning("[SNAPSHOT] Failed to fetch snapshot") - return None - - except Exception as e: - logger.error(f"[SNAPSHOT] Error fetching snapshot: {e}", exc_info=True) - return None - - async def _execute_fallback_pipeline(self, display_id: str, session_id: str, subscription_id: str): - """ - Execute fallback pipeline when sessionId is received without prior detection. - This handles POS-initiated sessions where backend starts transaction before car detection. - - Args: - display_id: Display identifier - session_id: Session ID from backend - subscription_id: Subscription identifier for pipeline execution - """ - try: - logger.info(f"[FALLBACK PIPELINE] Executing for session {session_id}, display {display_id}") - - # Fetch fresh snapshot from camera - frame = self._fetch_snapshot() - - if frame is None: - logger.error(f"[FALLBACK] Failed to fetch snapshot for session {session_id}, cannot execute pipeline") - return - - logger.info(f"[FALLBACK] Using snapshot frame {frame.shape[1]}x{frame.shape[0]} for session {session_id}") - - # Check if detection pipeline is available - if not self.detection_pipeline: - logger.error(f"[FALLBACK] Detection pipeline not available for session {session_id}") - return - - # Execute detection phase to get detected regions - detection_result = await self.detection_pipeline.execute_detection_phase( - frame=frame, - display_id=display_id, - subscription_id=subscription_id - ) - - logger.info(f"[FALLBACK] Detection phase completed for session {session_id}: " - f"status={detection_result.get('status', 'unknown')}, " - f"regions={list(detection_result.get('detected_regions', {}).keys())}") - - # If detection found regions, execute processing phase - detected_regions = detection_result.get('detected_regions', {}) - if detected_regions: - processing_result = await self.detection_pipeline.execute_processing_phase( - frame=frame, - display_id=display_id, - session_id=session_id, - subscription_id=subscription_id, - detected_regions=detected_regions - ) - - logger.info(f"[FALLBACK] Processing phase completed for session {session_id}: " - f"status={processing_result.get('status', 'unknown')}, " - f"branches={len(processing_result.get('branch_results', {}))}, " - f"actions={len(processing_result.get('actions_executed', []))}") - - # Update statistics - self.stats['pipelines_executed'] += 1 - - else: - logger.warning(f"[FALLBACK] No detections found in snapshot for session {session_id}") - - except Exception as e: - logger.error(f"[FALLBACK] Error executing fallback pipeline for session {session_id}: {e}", exc_info=True) - - def _filter_small_frontals(self, tracking_results, frame): - """ - Filter out frontal detections that are smaller than minimum bbox area percentage. - This prevents processing of cars from neighboring pumps that appear in camera view. - - Args: - tracking_results: YOLO tracking results with detections - frame: Input frame for calculating frame area - - Returns: - Modified tracking_results with small frontals removed - """ - if not hasattr(tracking_results, 'detections') or not tracking_results.detections: - return tracking_results - - # Calculate frame area and minimum bbox area threshold - frame_area = frame.shape[0] * frame.shape[1] # height * width - min_bbox_area = frame_area * (self.min_bbox_area_percentage / 100.0) - - # Filter detections - filtered_detections = [] - filtered_count = 0 - - for detection in tracking_results.detections: - # Calculate detection bbox area - bbox = detection.bbox # Assuming bbox is [x1, y1, x2, y2] - bbox_area = (bbox[2] - bbox[0]) * (bbox[3] - bbox[1]) - - if bbox_area >= min_bbox_area: - # Keep detection - bbox is large enough - filtered_detections.append(detection) - else: - # Filter out small detection - filtered_count += 1 - area_percentage = (bbox_area / frame_area) * 100 - logger.debug(f"Filtered small frontal: area={bbox_area:.0f}px² ({area_percentage:.1f}% of frame, " - f"min required: {self.min_bbox_area_percentage}%)") - - # Update tracking results with filtered detections - tracking_results.detections = filtered_detections - - # Update statistics - if filtered_count > 0: - self.stats['frontals_filtered_small'] += filtered_count - logger.info(f"Filtered {filtered_count} small frontal detections, " - f"{len(filtered_detections)} remaining (total filtered: {self.stats['frontals_filtered_small']})") - - return tracking_results - - def cleanup(self): - """Cleanup resources.""" - self.executor.shutdown(wait=False) - self.reset_tracking() - - - # Cleanup detection pipeline - if self.detection_pipeline: - self.detection_pipeline.cleanup() - - logger.info("Tracking pipeline integration cleaned up") \ No newline at end of file diff --git a/core/tracking/tracker.py b/core/tracking/tracker.py deleted file mode 100644 index 63d0299..0000000 --- a/core/tracking/tracker.py +++ /dev/null @@ -1,293 +0,0 @@ -""" -Vehicle Tracking Module - BoT-SORT based tracking with camera isolation -Implements vehicle identification, persistence, and motion analysis using external tracker. -""" -import logging -import time -import uuid -from typing import Dict, List, Optional, Tuple, Any -from dataclasses import dataclass, field -import numpy as np -from threading import Lock - -from .bot_sort_tracker import MultiCameraBoTSORT - -logger = logging.getLogger(__name__) - - -@dataclass -class TrackedVehicle: - """Represents a tracked vehicle with all its state information.""" - track_id: int - camera_id: str - first_seen: float - last_seen: float - session_id: Optional[str] = None - display_id: Optional[str] = None - confidence: float = 0.0 - bbox: Tuple[int, int, int, int] = (0, 0, 0, 0) # x1, y1, x2, y2 - center: Tuple[float, float] = (0.0, 0.0) - stable_frames: int = 0 - total_frames: int = 0 - is_stable: bool = False - processed_pipeline: bool = False - last_position_history: List[Tuple[float, float]] = field(default_factory=list) - avg_confidence: float = 0.0 - hit_streak: int = 0 - age: int = 0 - - def update_position(self, bbox: Tuple[int, int, int, int], confidence: float): - """Update vehicle position and confidence.""" - self.bbox = bbox - self.center = ((bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2) - self.last_seen = time.time() - self.confidence = confidence - self.total_frames += 1 - - # Update confidence average - self.avg_confidence = ((self.avg_confidence * (self.total_frames - 1)) + confidence) / self.total_frames - - # Maintain position history (last 10 positions) - self.last_position_history.append(self.center) - if len(self.last_position_history) > 10: - self.last_position_history.pop(0) - - def calculate_stability(self) -> float: - """Calculate stability score based on position history.""" - if len(self.last_position_history) < 2: - return 0.0 - - # Calculate movement variance - positions = np.array(self.last_position_history) - if len(positions) < 2: - return 0.0 - - # Calculate standard deviation of positions - std_x = np.std(positions[:, 0]) - std_y = np.std(positions[:, 1]) - - # Lower variance means more stable (inverse relationship) - # Normalize to 0-1 range (assuming max reasonable std is 50 pixels) - stability = max(0, 1 - (std_x + std_y) / 100) - return stability - - def is_expired(self, timeout_seconds: float = 2.0) -> bool: - """Check if vehicle tracking has expired.""" - return (time.time() - self.last_seen) > timeout_seconds - - -class VehicleTracker: - """ - Main vehicle tracking implementation using BoT-SORT with camera isolation. - Manages continuous tracking, vehicle identification, and state persistence. - """ - - def __init__(self, tracking_config: Optional[Dict] = None): - """ - Initialize the vehicle tracker. - - Args: - tracking_config: Configuration from pipeline.json tracking section - """ - self.config = tracking_config or {} - self.trigger_classes = self.config.get('trigger_classes', self.config.get('triggerClasses', ['frontal'])) - self.min_confidence = self.config.get('minConfidence', 0.6) - - # BoT-SORT multi-camera tracker - self.bot_sort = MultiCameraBoTSORT(self.trigger_classes, self.min_confidence) - - # Tracking state - maintain compatibility with existing code - self.tracked_vehicles: Dict[str, Dict[int, TrackedVehicle]] = {} # camera_id -> {track_id: vehicle} - self.lock = Lock() - - # Tracking parameters - self.stability_threshold = 0.7 - self.min_stable_frames = 5 - self.timeout_seconds = 2.0 - - logger.info(f"VehicleTracker initialized with BoT-SORT: trigger_classes={self.trigger_classes}, " - f"min_confidence={self.min_confidence}") - - def process_detections(self, - results: Any, - display_id: str, - frame: np.ndarray) -> List[TrackedVehicle]: - """ - Process detection results using BoT-SORT tracking. - - Args: - results: Detection results (InferenceResult) - display_id: Display identifier for this stream - frame: Current frame being processed - - Returns: - List of currently tracked vehicles - """ - current_time = time.time() - - # Extract camera_id from display_id for tracking isolation - camera_id = display_id # Using display_id as camera_id for isolation - - with self.lock: - # Update BoT-SORT tracker - track_results = self.bot_sort.update(camera_id, results) - - # Ensure camera tracking dict exists - if camera_id not in self.tracked_vehicles: - self.tracked_vehicles[camera_id] = {} - - # Update tracked vehicles based on BoT-SORT results - current_tracks = {} - active_tracks = [] - - for track_result in track_results: - track_id = track_result['track_id'] - - # Create or update TrackedVehicle - if track_id in self.tracked_vehicles[camera_id]: - # Update existing vehicle - vehicle = self.tracked_vehicles[camera_id][track_id] - vehicle.update_position(track_result['bbox'], track_result['confidence']) - vehicle.hit_streak = track_result['hit_streak'] - vehicle.age = track_result['age'] - - # Update stability based on hit_streak - if vehicle.hit_streak >= self.min_stable_frames: - vehicle.is_stable = True - vehicle.stable_frames = vehicle.hit_streak - - logger.debug(f"Updated track {track_id}: conf={vehicle.confidence:.2f}, " - f"stable={vehicle.is_stable}, hit_streak={vehicle.hit_streak}") - else: - # Create new vehicle - x1, y1, x2, y2 = track_result['bbox'] - vehicle = TrackedVehicle( - track_id=track_id, - camera_id=camera_id, - first_seen=current_time, - last_seen=current_time, - display_id=display_id, - confidence=track_result['confidence'], - bbox=tuple(track_result['bbox']), - center=((x1 + x2) / 2, (y1 + y2) / 2), - total_frames=1, - hit_streak=track_result['hit_streak'], - age=track_result['age'] - ) - vehicle.last_position_history.append(vehicle.center) - logger.info(f"New vehicle tracked: ID={track_id}, camera={camera_id}, display={display_id}") - - current_tracks[track_id] = vehicle - active_tracks.append(vehicle) - - # Update the camera's tracked vehicles - self.tracked_vehicles[camera_id] = current_tracks - - return active_tracks - - def get_stable_vehicles(self, display_id: Optional[str] = None) -> List[TrackedVehicle]: - """ - Get all stable vehicles, optionally filtered by display. - - Args: - display_id: Optional display ID to filter by - - Returns: - List of stable tracked vehicles - """ - with self.lock: - stable = [] - camera_id = display_id # Using display_id as camera_id - - if camera_id in self.tracked_vehicles: - for vehicle in self.tracked_vehicles[camera_id].values(): - if (vehicle.is_stable and not vehicle.is_expired(self.timeout_seconds) and - (display_id is None or vehicle.display_id == display_id)): - stable.append(vehicle) - - return stable - - def get_vehicle_by_session(self, session_id: str) -> Optional[TrackedVehicle]: - """ - Get a tracked vehicle by its session ID. - - Args: - session_id: Session ID to look up - - Returns: - Tracked vehicle if found, None otherwise - """ - with self.lock: - # Search across all cameras - for camera_vehicles in self.tracked_vehicles.values(): - for vehicle in camera_vehicles.values(): - if vehicle.session_id == session_id: - return vehicle - return None - - def mark_processed(self, track_id: int, session_id: str): - """ - Mark a vehicle as processed through the pipeline. - - Args: - track_id: Track ID of the vehicle - session_id: Session ID assigned to this vehicle - """ - with self.lock: - # Search across all cameras for the track_id - for camera_vehicles in self.tracked_vehicles.values(): - if track_id in camera_vehicles: - vehicle = camera_vehicles[track_id] - vehicle.processed_pipeline = True - vehicle.session_id = session_id - logger.info(f"Marked vehicle {track_id} as processed with session {session_id}") - return - - def clear_session(self, session_id: str): - """ - Clear session ID from a tracked vehicle (post-fueling). - - Args: - session_id: Session ID to clear - """ - with self.lock: - # Search across all cameras - for camera_vehicles in self.tracked_vehicles.values(): - for vehicle in camera_vehicles.values(): - if vehicle.session_id == session_id: - logger.info(f"Clearing session {session_id} from vehicle {vehicle.track_id}") - vehicle.session_id = None - # Keep processed_pipeline=True to prevent re-processing - - def reset_tracking(self): - """Reset all tracking state.""" - with self.lock: - self.tracked_vehicles.clear() - self.bot_sort.reset_all() - logger.info("Vehicle tracking state reset") - - def get_statistics(self) -> Dict: - """Get tracking statistics.""" - with self.lock: - total = 0 - stable = 0 - processed = 0 - all_confidences = [] - - # Aggregate stats across all cameras - for camera_vehicles in self.tracked_vehicles.values(): - total += len(camera_vehicles) - for vehicle in camera_vehicles.values(): - if vehicle.is_stable: - stable += 1 - if vehicle.processed_pipeline: - processed += 1 - all_confidences.append(vehicle.avg_confidence) - - return { - 'total_tracked': total, - 'stable_vehicles': stable, - 'processed_vehicles': processed, - 'avg_confidence': np.mean(all_confidences) if all_confidences else 0.0, - 'bot_sort_stats': self.bot_sort.get_statistics() - } \ No newline at end of file diff --git a/core/tracking/validator.py b/core/tracking/validator.py deleted file mode 100644 index d86a3f6..0000000 --- a/core/tracking/validator.py +++ /dev/null @@ -1,402 +0,0 @@ -""" -Vehicle Validation Module - Stable car detection and validation logic. -Differentiates between stable (fueling) cars and passing-by vehicles. -""" -import logging -import time -import numpy as np -from typing import List, Optional, Tuple, Dict, Any -from dataclasses import dataclass -from enum import Enum - -from .tracker import TrackedVehicle - -logger = logging.getLogger(__name__) - - -class VehicleState(Enum): - """Vehicle state classification.""" - UNKNOWN = "unknown" - ENTERING = "entering" - STABLE = "stable" - LEAVING = "leaving" - PASSING_BY = "passing_by" - - -@dataclass -class ValidationResult: - """Result of vehicle validation.""" - is_valid: bool - state: VehicleState - confidence: float - reason: str - should_process: bool = False - track_id: Optional[int] = None - - -class StableCarValidator: - """ - Validates whether a tracked vehicle should be processed through the pipeline. - - Updated for BoT-SORT integration: Trusts the sophisticated BoT-SORT tracking algorithm - for stability determination and focuses on business logic validation: - - Duration requirements for processing - - Confidence thresholds - - Session management and cooldowns - - Camera isolation with composite keys - """ - - def __init__(self, config: Optional[Dict] = None): - """ - Initialize the validator with configuration. - - Args: - config: Optional configuration dictionary - """ - self.config = config or {} - - # Validation thresholds - self.min_stable_duration = self.config.get('min_stable_duration', 3.0) # seconds - self.min_stable_frames = self.config.get('min_stable_frames', 10) - self.position_variance_threshold = self.config.get('position_variance_threshold', 25.0) # pixels - self.min_confidence = self.config.get('min_confidence', 0.7) - self.velocity_threshold = self.config.get('velocity_threshold', 5.0) # pixels/frame - self.entering_zone_ratio = self.config.get('entering_zone_ratio', 0.3) # 30% of frame - self.leaving_zone_ratio = self.config.get('leaving_zone_ratio', 0.3) - - # Frame dimensions (will be updated on first frame) - self.frame_width = 1920 - self.frame_height = 1080 - - # History for validation - self.validation_history: Dict[int, List[VehicleState]] = {} - self.last_processed_vehicles: Dict[int, float] = {} # track_id -> last_process_time - - logger.info(f"StableCarValidator initialized with min_duration={self.min_stable_duration}s, " - f"min_frames={self.min_stable_frames}, position_variance={self.position_variance_threshold}") - - def update_frame_dimensions(self, width: int, height: int): - """Update frame dimensions for zone calculations.""" - self.frame_width = width - self.frame_height = height - # Commented out verbose frame dimension logging - # logger.debug(f"Updated frame dimensions: {width}x{height}") - - def validate_vehicle(self, vehicle: TrackedVehicle, frame_shape: Optional[Tuple] = None) -> ValidationResult: - """ - Validate whether a tracked vehicle is stable and should be processed. - - Args: - vehicle: The tracked vehicle to validate - frame_shape: Optional frame shape (height, width, channels) - - Returns: - ValidationResult with validation status and reasoning - """ - # Update frame dimensions if provided - if frame_shape: - self.update_frame_dimensions(frame_shape[1], frame_shape[0]) - - # Initialize validation history for new vehicles - if vehicle.track_id not in self.validation_history: - self.validation_history[vehicle.track_id] = [] - - # Check if already processed - if vehicle.processed_pipeline: - return ValidationResult( - is_valid=False, - state=VehicleState.STABLE, - confidence=1.0, - reason="Already processed through pipeline", - should_process=False, - track_id=vehicle.track_id - ) - - # Check if recently processed (cooldown period) - if vehicle.track_id in self.last_processed_vehicles: - time_since_process = time.time() - self.last_processed_vehicles[vehicle.track_id] - if time_since_process < 10.0: # 10 second cooldown - return ValidationResult( - is_valid=False, - state=VehicleState.STABLE, - confidence=1.0, - reason=f"Recently processed ({time_since_process:.1f}s ago)", - should_process=False, - track_id=vehicle.track_id - ) - - # Determine vehicle state - state = self._determine_vehicle_state(vehicle) - - # Update history - self.validation_history[vehicle.track_id].append(state) - if len(self.validation_history[vehicle.track_id]) > 20: - self.validation_history[vehicle.track_id].pop(0) - - # Validate based on state - if state == VehicleState.STABLE: - return self._validate_stable_vehicle(vehicle) - elif state == VehicleState.PASSING_BY: - return ValidationResult( - is_valid=False, - state=state, - confidence=0.8, - reason="Vehicle is passing by", - should_process=False, - track_id=vehicle.track_id - ) - elif state == VehicleState.ENTERING: - return ValidationResult( - is_valid=False, - state=state, - confidence=0.5, - reason="Vehicle is entering, waiting for stability", - should_process=False, - track_id=vehicle.track_id - ) - elif state == VehicleState.LEAVING: - return ValidationResult( - is_valid=False, - state=state, - confidence=0.5, - reason="Vehicle is leaving", - should_process=False, - track_id=vehicle.track_id - ) - else: - return ValidationResult( - is_valid=False, - state=state, - confidence=0.0, - reason="Unknown vehicle state", - should_process=False, - track_id=vehicle.track_id - ) - - def _determine_vehicle_state(self, vehicle: TrackedVehicle) -> VehicleState: - """ - Determine the current state of the vehicle based on BoT-SORT tracking results. - - BoT-SORT provides sophisticated tracking, so we trust its stability determination - and focus on business logic validation. - - Args: - vehicle: The tracked vehicle - - Returns: - Current vehicle state - """ - # Trust BoT-SORT's stability determination - if vehicle.is_stable: - # Check if it's been stable long enough for processing - duration = time.time() - vehicle.first_seen - if duration >= self.min_stable_duration: - return VehicleState.STABLE - else: - return VehicleState.ENTERING - - # For non-stable vehicles, use simplified state determination - if len(vehicle.last_position_history) < 2: - return VehicleState.UNKNOWN - - # Calculate velocity for movement classification - velocity = self._calculate_velocity(vehicle) - - # Basic movement classification - if velocity > self.velocity_threshold: - # Vehicle is moving - classify as passing by or entering/leaving - x_position = vehicle.center[0] / self.frame_width - - # Simple heuristic: vehicles near edges are entering/leaving, center vehicles are passing - if x_position < 0.2 or x_position > 0.8: - return VehicleState.ENTERING - else: - return VehicleState.PASSING_BY - - # Low velocity but not marked stable by tracker - likely entering - return VehicleState.ENTERING - - def _validate_stable_vehicle(self, vehicle: TrackedVehicle) -> ValidationResult: - """ - Perform business logic validation of a stable vehicle. - - Since BoT-SORT already determined the vehicle is stable, we focus on: - - Duration requirements for processing - - Confidence thresholds - - Business logic constraints - - Args: - vehicle: The stable vehicle to validate - - Returns: - Detailed validation result - """ - # Check duration (business requirement) - duration = time.time() - vehicle.first_seen - if duration < self.min_stable_duration: - return ValidationResult( - is_valid=False, - state=VehicleState.STABLE, - confidence=0.6, - reason=f"Not stable long enough ({duration:.1f}s < {self.min_stable_duration}s)", - should_process=False, - track_id=vehicle.track_id - ) - - # Check confidence (business requirement) - if vehicle.avg_confidence < self.min_confidence: - return ValidationResult( - is_valid=False, - state=VehicleState.STABLE, - confidence=vehicle.avg_confidence, - reason=f"Confidence too low ({vehicle.avg_confidence:.2f} < {self.min_confidence})", - should_process=False, - track_id=vehicle.track_id - ) - - # Trust BoT-SORT's stability determination - skip position variance check - # BoT-SORT's sophisticated tracking already ensures consistent positioning - - # Simplified state history check - just ensure recent stability - if vehicle.track_id in self.validation_history: - history = self.validation_history[vehicle.track_id][-3:] # Last 3 states - stable_count = sum(1 for s in history if s == VehicleState.STABLE) - if len(history) >= 2 and stable_count == 0: # Only fail if clear instability - return ValidationResult( - is_valid=False, - state=VehicleState.STABLE, - confidence=0.7, - reason="Recent state history shows instability", - should_process=False, - track_id=vehicle.track_id - ) - - # All checks passed - vehicle is valid for processing - self.last_processed_vehicles[vehicle.track_id] = time.time() - - return ValidationResult( - is_valid=True, - state=VehicleState.STABLE, - confidence=vehicle.avg_confidence, - reason="Vehicle is stable and ready for processing (BoT-SORT validated)", - should_process=True, - track_id=vehicle.track_id - ) - - def _calculate_velocity(self, vehicle: TrackedVehicle) -> float: - """ - Calculate the velocity of the vehicle based on position history. - - Args: - vehicle: The tracked vehicle - - Returns: - Velocity in pixels per frame - """ - if len(vehicle.last_position_history) < 2: - return 0.0 - - positions = np.array(vehicle.last_position_history) - if len(positions) < 2: - return 0.0 - - # Calculate velocity over last 3 frames - recent_positions = positions[-min(3, len(positions)):] - velocities = [] - - for i in range(1, len(recent_positions)): - dx = recent_positions[i][0] - recent_positions[i-1][0] - dy = recent_positions[i][1] - recent_positions[i-1][1] - velocity = np.sqrt(dx**2 + dy**2) - velocities.append(velocity) - - return np.mean(velocities) if velocities else 0.0 - - def _calculate_position_variance(self, vehicle: TrackedVehicle) -> float: - """ - Calculate the position variance of the vehicle. - - Args: - vehicle: The tracked vehicle - - Returns: - Position variance in pixels - """ - if len(vehicle.last_position_history) < 2: - return 0.0 - - positions = np.array(vehicle.last_position_history) - variance_x = np.var(positions[:, 0]) - variance_y = np.var(positions[:, 1]) - - return np.sqrt(variance_x + variance_y) - - def should_skip_same_car(self, - vehicle: TrackedVehicle, - session_cleared: bool = False, - permanently_processed: Dict[str, float] = None) -> bool: - """ - Determine if we should skip processing for the same car after session clear. - - Args: - vehicle: The tracked vehicle - session_cleared: Whether the session was recently cleared - permanently_processed: Dict of permanently processed vehicles (camera_id:track_id -> time) - - Returns: - True if we should skip this vehicle - """ - # Check if this vehicle was permanently processed (never process again) - if permanently_processed: - # Create composite key using camera_id and track_id - permanent_key = f"{vehicle.camera_id}:{vehicle.track_id}" - if permanent_key in permanently_processed: - process_time = permanently_processed[permanent_key] - time_since = time.time() - process_time - logger.debug(f"Skipping permanently processed vehicle {vehicle.track_id} on camera {vehicle.camera_id} " - f"(processed {time_since:.1f}s ago)") - return True - - # If vehicle has a session_id but it was cleared, skip for a period - if vehicle.session_id is None and vehicle.processed_pipeline and session_cleared: - # Check if enough time has passed since processing - if vehicle.track_id in self.last_processed_vehicles: - time_since = time.time() - self.last_processed_vehicles[vehicle.track_id] - if time_since < 30.0: # 30 second cooldown after session clear - logger.debug(f"Skipping same car {vehicle.track_id} after session clear " - f"({time_since:.1f}s since processing)") - return True - - return False - - def reset_vehicle(self, track_id: int): - """ - Reset validation state for a specific vehicle. - - Args: - track_id: Track ID of the vehicle to reset - """ - if track_id in self.validation_history: - del self.validation_history[track_id] - if track_id in self.last_processed_vehicles: - del self.last_processed_vehicles[track_id] - logger.debug(f"Reset validation state for vehicle {track_id}") - - def get_statistics(self) -> Dict: - """Get validation statistics.""" - return { - 'vehicles_in_history': len(self.validation_history), - 'recently_processed': len(self.last_processed_vehicles), - 'state_distribution': self._get_state_distribution() - } - - def _get_state_distribution(self) -> Dict[str, int]: - """Get distribution of current vehicle states.""" - distribution = {state.value: 0 for state in VehicleState} - - for history in self.validation_history.values(): - if history: - current_state = history[-1] - distribution[current_state.value] += 1 - - return distribution \ No newline at end of file diff --git a/core/utils/ffmpeg_detector.py b/core/utils/ffmpeg_detector.py deleted file mode 100644 index 565713c..0000000 --- a/core/utils/ffmpeg_detector.py +++ /dev/null @@ -1,214 +0,0 @@ -""" -FFmpeg hardware acceleration detection and configuration -""" - -import subprocess -import logging -import re -from typing import Dict, List, Optional - -logger = logging.getLogger("detector_worker") - - -class FFmpegCapabilities: - """Detect and configure FFmpeg hardware acceleration capabilities.""" - - def __init__(self): - """Initialize FFmpeg capabilities detector.""" - self.hwaccels = [] - self.codecs = {} - self.nvidia_support = False - self.vaapi_support = False - self.qsv_support = False - - self._detect_capabilities() - - def _detect_capabilities(self): - """Detect available hardware acceleration methods.""" - try: - # Get hardware accelerators - result = subprocess.run( - ['ffmpeg', '-hide_banner', '-hwaccels'], - capture_output=True, text=True, timeout=10 - ) - if result.returncode == 0: - self.hwaccels = [line.strip() for line in result.stdout.strip().split('\n')[1:] if line.strip()] - logger.info(f"Available FFmpeg hardware accelerators: {', '.join(self.hwaccels)}") - - # Check for NVIDIA support - self.nvidia_support = any(hw in self.hwaccels for hw in ['cuda', 'cuvid', 'nvdec']) - self.vaapi_support = 'vaapi' in self.hwaccels - self.qsv_support = 'qsv' in self.hwaccels - - # Get decoder information - self._detect_decoders() - - # Log capabilities - if self.nvidia_support: - logger.info("NVIDIA hardware acceleration available (CUDA/CUVID/NVDEC)") - logger.info(f"Detected hardware codecs: {self.codecs}") - if self.vaapi_support: - logger.info("VAAPI hardware acceleration available") - if self.qsv_support: - logger.info("Intel QuickSync hardware acceleration available") - - except Exception as e: - logger.warning(f"Failed to detect FFmpeg capabilities: {e}") - - def _detect_decoders(self): - """Detect available hardware decoders.""" - try: - result = subprocess.run( - ['ffmpeg', '-hide_banner', '-decoders'], - capture_output=True, text=True, timeout=10 - ) - if result.returncode == 0: - # Parse decoder output to find hardware decoders - for line in result.stdout.split('\n'): - if 'cuvid' in line or 'nvdec' in line: - match = re.search(r'(\w+)\s+.*?(\w+(?:_cuvid|_nvdec))', line) - if match: - codec_type, decoder = match.groups() - if 'h264' in decoder: - self.codecs['h264_hw'] = decoder - elif 'hevc' in decoder or 'h265' in decoder: - self.codecs['h265_hw'] = decoder - elif 'vaapi' in line: - match = re.search(r'(\w+)\s+.*?(\w+_vaapi)', line) - if match: - codec_type, decoder = match.groups() - if 'h264' in decoder: - self.codecs['h264_vaapi'] = decoder - - except Exception as e: - logger.debug(f"Failed to detect decoders: {e}") - - def get_optimal_capture_options(self, codec: str = 'h264') -> Dict[str, str]: - """ - Get optimal FFmpeg capture options for the given codec. - - Args: - codec: Video codec (h264, h265, etc.) - - Returns: - Dictionary of FFmpeg options - """ - options = { - 'rtsp_transport': 'tcp', - 'buffer_size': '1024k', - 'max_delay': '500000', # 500ms - 'fflags': '+genpts', - 'flags': '+low_delay', - 'probesize': '32', - 'analyzeduration': '0' - } - - # Add hardware acceleration if available - if self.nvidia_support: - # Force enable CUDA hardware acceleration for H.264 if CUDA is available - if codec == 'h264': - options.update({ - 'hwaccel': 'cuda', - 'hwaccel_device': '0' - }) - logger.info("Using NVIDIA NVDEC hardware acceleration for H.264") - elif codec == 'h265': - options.update({ - 'hwaccel': 'cuda', - 'hwaccel_device': '0', - 'video_codec': 'hevc_cuvid', - 'hwaccel_output_format': 'cuda' - }) - logger.info("Using NVIDIA CUVID hardware acceleration for H.265") - - elif self.vaapi_support: - if codec == 'h264': - options.update({ - 'hwaccel': 'vaapi', - 'hwaccel_device': '/dev/dri/renderD128', - 'video_codec': 'h264_vaapi' - }) - logger.debug("Using VAAPI hardware acceleration") - - return options - - def format_opencv_options(self, options: Dict[str, str]) -> str: - """ - Format options for OpenCV FFmpeg backend. - - Args: - options: Dictionary of FFmpeg options - - Returns: - Formatted options string for OpenCV - """ - return '|'.join(f"{key};{value}" for key, value in options.items()) - - def get_hardware_encoder_options(self, codec: str = 'h264', quality: str = 'fast') -> Dict[str, str]: - """ - Get optimal hardware encoding options. - - Args: - codec: Video codec for encoding - quality: Quality preset (fast, medium, slow) - - Returns: - Dictionary of encoding options - """ - options = {} - - if self.nvidia_support: - if codec == 'h264': - options.update({ - 'video_codec': 'h264_nvenc', - 'preset': quality, - 'tune': 'zerolatency', - 'gpu': '0', - 'rc': 'cbr_hq', - 'surfaces': '64' - }) - elif codec == 'h265': - options.update({ - 'video_codec': 'hevc_nvenc', - 'preset': quality, - 'tune': 'zerolatency', - 'gpu': '0' - }) - - elif self.vaapi_support: - if codec == 'h264': - options.update({ - 'video_codec': 'h264_vaapi', - 'vaapi_device': '/dev/dri/renderD128' - }) - - return options - - -# Global instance -_ffmpeg_caps = None - -def get_ffmpeg_capabilities() -> FFmpegCapabilities: - """Get or create the global FFmpeg capabilities instance.""" - global _ffmpeg_caps - if _ffmpeg_caps is None: - _ffmpeg_caps = FFmpegCapabilities() - return _ffmpeg_caps - -def get_optimal_rtsp_options(rtsp_url: str) -> str: - """ - Get optimal OpenCV FFmpeg options for RTSP streaming. - - Args: - rtsp_url: RTSP stream URL - - Returns: - Formatted options string for cv2.VideoCapture - """ - caps = get_ffmpeg_capabilities() - - # Detect codec from URL or assume H.264 - codec = 'h265' if any(x in rtsp_url.lower() for x in ['h265', 'hevc']) else 'h264' - - options = caps.get_optimal_capture_options(codec) - return caps.format_opencv_options(options) \ No newline at end of file diff --git a/core/utils/hardware_encoder.py b/core/utils/hardware_encoder.py deleted file mode 100644 index 45bbb35..0000000 --- a/core/utils/hardware_encoder.py +++ /dev/null @@ -1,173 +0,0 @@ -""" -Hardware-accelerated image encoding using NVIDIA NVENC or Intel QuickSync -""" - -import cv2 -import numpy as np -import logging -from typing import Optional, Tuple -import os - -logger = logging.getLogger("detector_worker") - - -class HardwareEncoder: - """Hardware-accelerated JPEG encoder using GPU.""" - - def __init__(self): - """Initialize hardware encoder.""" - self.nvenc_available = False - self.vaapi_available = False - self.turbojpeg_available = False - - # Check for TurboJPEG (fastest CPU-based option) - try: - from turbojpeg import TurboJPEG - self.turbojpeg = TurboJPEG() - self.turbojpeg_available = True - logger.info("TurboJPEG accelerated encoding available") - except ImportError: - logger.debug("TurboJPEG not available") - - # Check for NVIDIA NVENC support - try: - # Test if we can create an NVENC encoder - test_frame = np.zeros((720, 1280, 3), dtype=np.uint8) - fourcc = cv2.VideoWriter_fourcc(*'H264') - test_writer = cv2.VideoWriter( - "test.mp4", - fourcc, - 30, - (1280, 720), - [cv2.CAP_PROP_HW_ACCELERATION, cv2.VIDEO_ACCELERATION_ANY] - ) - if test_writer.isOpened(): - self.nvenc_available = True - logger.info("NVENC hardware encoding available") - test_writer.release() - if os.path.exists("test.mp4"): - os.remove("test.mp4") - except Exception as e: - logger.debug(f"NVENC not available: {e}") - - def encode_jpeg(self, frame: np.ndarray, quality: int = 85) -> Optional[bytes]: - """ - Encode frame to JPEG using the fastest available method. - - Args: - frame: BGR image frame - quality: JPEG quality (1-100) - - Returns: - Encoded JPEG bytes or None on failure - """ - try: - # Method 1: TurboJPEG (3-5x faster than cv2.imencode) - if self.turbojpeg_available: - # Convert BGR to RGB for TurboJPEG - rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - encoded = self.turbojpeg.encode(rgb_frame, quality=quality) - return encoded - - # Method 2: Hardware-accelerated encoding via GStreamer (if available) - if self.nvenc_available: - return self._encode_with_nvenc(frame, quality) - - # Fallback: Standard OpenCV encoding - encode_params = [cv2.IMWRITE_JPEG_QUALITY, quality] - success, encoded = cv2.imencode('.jpg', frame, encode_params) - if success: - return encoded.tobytes() - - return None - - except Exception as e: - logger.error(f"Failed to encode frame: {e}") - return None - - def _encode_with_nvenc(self, frame: np.ndarray, quality: int) -> Optional[bytes]: - """ - Encode using NVIDIA NVENC hardware encoder. - - This is complex to implement directly, so we'll use a GStreamer pipeline - if available. - """ - try: - # Create a GStreamer pipeline for hardware encoding - height, width = frame.shape[:2] - gst_pipeline = ( - f"appsrc ! " - f"video/x-raw,format=BGR,width={width},height={height},framerate=30/1 ! " - f"videoconvert ! " - f"nvvideoconvert ! " # GPU color conversion - f"nvjpegenc quality={quality} ! " # Hardware JPEG encoder - f"appsink" - ) - - # This would require GStreamer Python bindings - # For now, fall back to TurboJPEG or standard encoding - logger.debug("NVENC JPEG encoding not fully implemented, using fallback") - encode_params = [cv2.IMWRITE_JPEG_QUALITY, quality] - success, encoded = cv2.imencode('.jpg', frame, encode_params) - if success: - return encoded.tobytes() - - return None - - except Exception as e: - logger.error(f"NVENC encoding failed: {e}") - return None - - def encode_batch(self, frames: list, quality: int = 85) -> list: - """ - Batch encode multiple frames for better GPU utilization. - - Args: - frames: List of BGR frames - quality: JPEG quality - - Returns: - List of encoded JPEG bytes - """ - encoded_frames = [] - - if self.turbojpeg_available: - # TurboJPEG can handle batch encoding efficiently - for frame in frames: - rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - encoded = self.turbojpeg.encode(rgb_frame, quality=quality) - encoded_frames.append(encoded) - else: - # Fallback to sequential encoding - for frame in frames: - encoded = self.encode_jpeg(frame, quality) - encoded_frames.append(encoded) - - return encoded_frames - - -# Global encoder instance -_hardware_encoder = None - - -def get_hardware_encoder() -> HardwareEncoder: - """Get or create the global hardware encoder instance.""" - global _hardware_encoder - if _hardware_encoder is None: - _hardware_encoder = HardwareEncoder() - return _hardware_encoder - - -def encode_frame_hardware(frame: np.ndarray, quality: int = 85) -> Optional[bytes]: - """ - Convenience function to encode a frame using hardware acceleration. - - Args: - frame: BGR image frame - quality: JPEG quality (1-100) - - Returns: - Encoded JPEG bytes or None on failure - """ - encoder = get_hardware_encoder() - return encoder.encode_jpeg(frame, quality) \ No newline at end of file diff --git a/debug/test_camera_indices.py b/debug/test_camera_indices.py new file mode 100644 index 0000000..f88bc87 --- /dev/null +++ b/debug/test_camera_indices.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +""" +Test script to check available camera indices +""" + +import cv2 +import logging +import sys +import subprocess + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s" +) +logger = logging.getLogger("camera_index_test") + +def test_camera_index(index): + """Test if a camera index is available""" + try: + cap = cv2.VideoCapture(index) + if cap.isOpened(): + ret, frame = cap.read() + if ret and frame is not None: + width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + fps = cap.get(cv2.CAP_PROP_FPS) + + cap.release() + return True, f"{width}x{height} @ {fps}fps" + else: + cap.release() + return False, "Can open but cannot read frames" + else: + cap.release() + return False, "Cannot open camera" + except Exception as e: + return False, f"Error: {str(e)}" + +def get_windows_cameras_ffmpeg(): + """Get available cameras on Windows using FFmpeg""" + try: + result = subprocess.run(['ffmpeg', '-f', 'dshow', '-list_devices', 'true', '-i', 'dummy'], + capture_output=True, text=True, timeout=10, encoding='utf-8', errors='ignore') + output = result.stderr + + lines = output.split('\n') + video_devices = [] + + # Parse the output - look for lines with (video) that contain device names in quotes + for line in lines: + if '[dshow @' in line and '(video)' in line and '"' in line: + # Extract device name between first pair of quotes + start = line.find('"') + 1 + end = line.find('"', start) + if start > 0 and end > start: + device_name = line[start:end] + video_devices.append(device_name) + + logger.info(f"FFmpeg detected video devices: {video_devices}") + return video_devices + except Exception as e: + logger.error(f"Failed to get Windows camera names: {e}") + return [] + +def main(): + logger.info("=== Camera Index Test ===") + + # Check FFmpeg availability for Windows device detection + ffmpeg_available = False + try: + result = subprocess.run(['ffmpeg', '-version'], capture_output=True, text=True, timeout=5) + if result.returncode == 0: + ffmpeg_available = True + logger.info("FFmpeg is available") + except: + logger.info("FFmpeg not available") + + # Get Windows camera names if possible + if sys.platform.startswith('win') and ffmpeg_available: + logger.info("\n=== Windows Camera Devices (FFmpeg) ===") + cameras = get_windows_cameras_ffmpeg() + if cameras: + for i, camera in enumerate(cameras): + logger.info(f"Device {i}: {camera}") + else: + logger.info("No cameras detected via FFmpeg") + + # Test camera indices 0-9 + logger.info("\n=== Testing Camera Indices ===") + available_cameras = [] + + for index in range(10): + logger.info(f"Testing camera index {index}...") + is_available, info = test_camera_index(index) + + if is_available: + logger.info(f"✓ Camera {index}: AVAILABLE - {info}") + available_cameras.append(index) + else: + logger.info(f"✗ Camera {index}: NOT AVAILABLE - {info}") + + # Summary + logger.info("\n=== Summary ===") + if available_cameras: + logger.info(f"Available camera indices: {available_cameras}") + logger.info(f"Default camera index to use: {available_cameras[0]}") + + # Test the first available camera more thoroughly + logger.info(f"\n=== Detailed Test for Camera {available_cameras[0]} ===") + cap = cv2.VideoCapture(available_cameras[0]) + if cap.isOpened(): + # Get properties + width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + fps = cap.get(cv2.CAP_PROP_FPS) + backend = cap.getBackendName() + + logger.info(f"Resolution: {width}x{height}") + logger.info(f"FPS: {fps}") + logger.info(f"Backend: {backend}") + + # Test frame capture + ret, frame = cap.read() + if ret and frame is not None: + logger.info(f"Frame capture: SUCCESS") + logger.info(f"Frame shape: {frame.shape}") + logger.info(f"Frame dtype: {frame.dtype}") + else: + logger.info(f"Frame capture: FAILED") + + cap.release() + else: + logger.error("No cameras available!") + logger.info("Possible solutions:") + logger.info("1. Check if camera is connected and not used by another application") + logger.info("2. Check camera permissions") + logger.info("3. Try different camera indices") + logger.info("4. Install camera drivers") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/docs/MasterElection.md b/docs/MasterElection.md new file mode 100644 index 0000000..c5980b8 --- /dev/null +++ b/docs/MasterElection.md @@ -0,0 +1,1449 @@ +# Master Election Service Specification - Distributed Process Coordination + +## Overview + +The MasterElection service implements a Redis-based distributed leadership election and process coordination system for the CMS backend cluster. This service provides robust master-slave coordination with automatic failover, process registration, and TTL-based cleanup for multi-process backend deployments. + +**Key Architectural Principle**: Redis-based coordination with atomic Lua scripts ensures consistency and prevents split-brain scenarios while providing automatic cleanup through per-entry TTL expiration. + +## Architecture Components + +### Two-Tier Process Coordination + +The system manages two distinct coordination layers: + +1. **Master Election Layer**: Single leader election across all backend processes +2. **Process Registry Layer**: Individual process registration and heartbeat management + +### Leadership Election Pattern + +- **Single Master**: Only one backend process holds master lock at any time +- **Automatic Failover**: Master election triggers immediately when current master fails +- **Heartbeat-Based**: Master must renew lock every 10 seconds or lose leadership +- **Lua Script Atomicity**: All Redis operations use atomic Lua scripts to prevent race conditions +- **Event-Driven Transitions**: Role changes emit events for dependent services integration + +## Core Components + +### MasterElection Class +`cms-backend/services/MasterElection.ts` + +Primary coordination service that handles distributed leadership election and process lifecycle management. + +**Key Responsibilities:** +- Manages master lock acquisition and renewal using atomic Redis operations +- Provides process registration with automatic TTL-based expiration (45 seconds) +- Emits role transition events for dependent service coordination +- Handles slave registration and heartbeat management +- Maintains process-to-channel mapping for message routing + +### Process Management System + +**Process Registration:** +- Each backend process registers with unique UUID-based identifier +- Process metadata includes role, channel name, and capabilities +- TTL-based expiration (45 seconds) with heartbeat renewal +- Automatic cleanup of stale process entries without manual intervention + +**Channel Assignment:** +- Each process gets assigned a unique Redis pub/sub channel +- Channel mapping stored persistently for message routing +- Master process maintains channel-to-process mapping + +## Data Structures + +### MasterElectionEvents +```typescript +interface MasterElectionEvents { + 'master-acquired': () => void; // This process became master + 'master-lost': () => void; // This process lost master status + 'election-started': () => void; // Election process initiated + 'election-completed': (isMaster: boolean) => void; // Election finished + 'slave-registered': (slave: SlaveNode) => void; // New slave joined + 'slave-removed': (nodeId: string) => void; // Slave left/expired + 'error': (error: Error) => void; // Election/coordination errors +} +``` + +### ProcessInfo +```typescript +interface ProcessInfo { + processId: string; // Unique process identifier (UUID) + nodeId: string; // Node identifier (same as processId) + role: 'master' | 'slave'; // Current process role + lastSeen: string; // Last heartbeat timestamp (ISO string) + capabilities: ProcessCapabilities; // Process feature capabilities +} + +// Channel name derived as: `worker:slave:${processInfo.processId}` +``` + +### ProcessCapabilities +```typescript +interface ProcessCapabilities { + canProcessDetections: boolean; // Can handle AI detection processing + maxSubscriptions: number; // Maximum camera subscriptions supported + preferredWorkload: number; // Preferred subscription load (0-100) +} +``` + +### SlaveNode +```typescript +interface SlaveNode { + nodeId: string; // Unique slave node identifier + identifier: string; // Human-readable process identifier + registeredAt: string; // Initial registration timestamp + lastSeen: string; // Last heartbeat timestamp + metadata?: Record; // Optional process metadata +} +``` + +## Redis Data Architecture + +### Master Election Keys +- `master-election:master` - Current master process identifier with TTL lock +- `master-election:heartbeat` - Master heartbeat timestamp for liveness detection +- `master-election:master_process` - Detailed master process information (JSON) + +### Process Registry Keys (TTL-Enabled) +- `master-election:processes` - Hash map of all active processes with per-entry TTL (45s) +- Channel names derived directly from process ID: `worker:slave:{processId}` - no separate mapping needed + +### TTL Configuration +```typescript +// Per-entry TTL using hSetEx for automatic cleanup +PROCESS_TTL = 45; // Process registration expires after 45 seconds +HEARTBEAT_RENEWAL_INTERVAL = 10; // Process heartbeats renew TTL every 10 seconds +MASTER_LOCK_TTL = 30; // Master lock expires after 30 seconds +``` + +### Data Persistence Strategy +Uses **per-entry TTL with hSetEx** for automatic cleanup: +- Process entries automatically expire if heartbeats stop +- No manual cleanup processes required +- Prevents memory leaks from crashed processes +- Self-healing system that maintains only active processes +- Slave information derived from processes with role='slave' - no separate storage needed +- Channel names derived directly from process ID - no mapping table required + +## Master Election Algorithm + +### Election Flow Diagram + +```mermaid +graph TB + subgraph "Election Process" + START[Process Starts] --> ATTEMPT[attemptElection] + ATTEMPT --> ACQUIRE{acquireMasterLock} + + ACQUIRE -->|Success| MASTER[becomeMaster] + ACQUIRE -->|Failed| SLAVE[becomeSlave] + + MASTER --> HEARTBEAT[startHeartbeat] + SLAVE --> REGISTER[registerAsSlave] + + HEARTBEAT --> RENEW{renewMasterLock} + RENEW -->|Success| CONTINUE[Continue as Master] + RENEW -->|Failed| STEPDOWN[Step Down → SLAVE] + + REGISTER --> MONITOR[Monitor Master] + MONITOR --> CHECK{Master Exists?} + CHECK -->|Yes| WAIT[Wait and Monitor] + CHECK -->|No| ATTEMPT + + STEPDOWN --> SLAVE + WAIT --> MONITOR + CONTINUE --> RENEW + end + + subgraph "Atomic Operations" + ACQUIRE --> LUA1[Lua Script: SET master NX + SET heartbeat] + RENEW --> LUA2[Lua Script: Check owner + PEXPIRE + SET heartbeat] + STEPDOWN --> LUA3[Lua Script: Check owner + DEL master + DEL heartbeat] + end +``` + +### Atomic Lock Operations + +#### Master Lock Acquisition +```lua +-- Atomic master lock acquisition with heartbeat +if redis.call("SET", KEYS[1], ARGV[1], "NX", "PX", ARGV[2]) then + redis.call("SET", KEYS[2], ARGV[3], "PX", ARGV[2]) + return 1 +else + return 0 +end +``` + +#### Master Lock Renewal +```lua +-- Atomic master lock renewal with heartbeat update +if redis.call("GET", KEYS[1]) == ARGV[1] then + redis.call("PEXPIRE", KEYS[1], ARGV[2]) + redis.call("SET", KEYS[2], ARGV[3], "PX", ARGV[2]) + return 1 +else + return 0 +end +``` + +#### Master Lock Release +```lua +-- Atomic master lock release +if redis.call("GET", KEYS[1]) == ARGV[1] then + redis.call("DEL", KEYS[1], KEYS[2]) + return 1 +else + return 0 +end +``` + +## Process Lifecycle Management + +### Process Registration Flow + +```mermaid +sequenceDiagram + participant P as Process + participant R as Redis + participant M as Master Process + + Note over P,M: Process Registration with TTL + + P->>+P: Generate UUID processId + P->>+P: Determine role (master/slave) + P->>+P: Assign channel name + + P->>+R: hSetEx(processes, processId, processInfo, {EX: 45}) + R-->>-P: Registration confirmed + + P->>+R: hSet(channels, processId, channelName) + R-->>-P: Channel mapping stored + + alt Process becomes master + P->>+R: set(master_process, processInfo) + R-->>-P: Master process registered + P->>+M: emit('master-acquired') + else Process becomes slave + P->>+R: hSet(slaves, nodeId, slaveInfo) + R-->>-P: Slave registered + P->>+M: emit('slave-registered', slaveInfo) + end + + Note over P,M: Heartbeat Loop (Every 10s) + + loop Every 10 seconds + P->>+P: updateProcessHeartbeat(processId) + P->>+R: hSetEx(processes, processId, updatedInfo, {EX: 45}) + Note over R: TTL renewed for 45 seconds + R-->>-P: Heartbeat recorded + end + + Note over P,M: Automatic Expiration (No heartbeat) + + R->>R: 45 seconds pass without heartbeat + R->>R: Process entry automatically expires + Note over R: No manual cleanup needed +``` + +### Master Election Scenarios + +#### Scenario 1: Initial Startup +```mermaid +sequenceDiagram + participant P1 as Process 1 + participant P2 as Process 2 + participant R as Redis + + Note over P1,R: First Process Startup + + P1->>+P1: attemptElection() + P1->>+R: Lua Script: SET master NX + R-->>-P1: Success (no existing master) + + P1->>+P1: becomeMaster() + P1->>+P1: emit('master-acquired') + P1->>+P1: startHeartbeat() every 10s + + Note over P1,R: Second Process Startup + + P2->>+P2: attemptElection() + P2->>+R: Lua Script: SET master NX + R-->>-P2: Failed (master exists) + + P2->>+P2: becomeSlave() + P2->>+R: hSet(slaves, nodeId, slaveInfo) + P2->>+P2: emit('election-completed', false) +``` + +#### Scenario 2: Master Failure and Failover +```mermaid +sequenceDiagram + participant P1 as Master Process + participant P2 as Slave Process 1 + participant P3 as Slave Process 2 + participant R as Redis + + Note over P1,R: Normal Operation + + P1->>+R: Heartbeat renewal every 10s + P2->>+P2: Monitor master existence every 5s + P3->>+P3: Monitor master existence every 5s + + Note over P1,R: Master Failure + + P1--XP1: Process crashes/network failure + + Note over R: Master lock expires after 30s + + R->>R: Master lock TTL expires + + Note over P2,R: Slave Detects Missing Master + + P2->>+R: checkMasterExists() Lua Script + R-->>-P2: Master not found or stale + + P2->>+P2: Random delay (0-2s) to reduce collisions + P2->>+R: attemptElection() - Lua Script: SET master NX + R-->>-P2: Success - became new master + + P2->>+P2: becomeMaster() + P2->>+P2: emit('master-acquired') + + Note over P3,R: Other Slave Detects New Master + + P3->>+R: checkMasterExists() + R-->>-P3: New master found + P3->>+P3: Continue as slave - no election needed +``` + +## TTL-Based Cleanup System + +### Per-Entry TTL Implementation + +```typescript +// Process registration with automatic TTL expiration +public async registerProcess(processInfo: ProcessInfo): Promise { + // Set process registration with 45 second TTL per entry + await redisClient.hSetEx( + this.processesKey, + { + [processInfo.processId]: JSON.stringify(processInfo) + }, + { + expiration: { + type: 'EX', + value: 45 // 45 second TTL per process entry + } + } + ); + + // Map process to channel (no TTL - cleaned up manually) + await redisClient.hSet( + this.processChannelsKey, + processInfo.processId, + processInfo.channelName + ); +} + +// Heartbeat renewal extends TTL automatically +public async updateProcessHeartbeat(processId: string): Promise { + const processData = await redisClient.hGet(this.processesKey, processId); + if (processData) { + const processInfo: ProcessInfo = JSON.parse(processData); + processInfo.lastSeen = new Date().toISOString(); + + // Update process and renew TTL on heartbeat (per-entry TTL) + await redisClient.hSetEx( + this.processesKey, + { + [processId]: JSON.stringify(processInfo) + }, + { + expiration: { + type: 'EX', + value: 45 // Renew 45 second TTL for this specific process entry + } + } + ); + } +} +``` + +### Cleanup Behavior + +```mermaid +graph TB + subgraph "TTL Cleanup Process" + REG[Process Registration] --> TTL[45s TTL Set] + TTL --> HB{Heartbeat Within 45s?} + + HB -->|Yes| RENEW[TTL Renewed to 45s] + HB -->|No| EXPIRE[Entry Automatically Expires] + + RENEW --> HB + EXPIRE --> GONE[Process Removed from Redis] + + GONE --> DETECT[Other Processes Detect Absence] + DETECT --> REBALANCE[Automatic Rebalancing] + end + + subgraph "Manual vs TTL Cleanup" + MANUAL[Manual Cleanup Process] + AUTOMATIC[TTL-Based Cleanup] + + MANUAL -.->|"❌ Complex"| ISSUES[Race Conditions
Memory Leaks
Stale Data] + AUTOMATIC -.->|"✅ Simple"| BENEFITS[Self-Healing
No Race Conditions
Guaranteed Cleanup] + end +``` + +## Event System Architecture + +### Event Emission Flow + +```mermaid +graph TD + subgraph "Election Events" + START[Election Started] --> ATTEMPT[Attempt Lock Acquisition] + ATTEMPT --> SUCCESS{Lock Acquired?} + + SUCCESS -->|Yes| MASTER[Become Master] + SUCCESS -->|No| SLAVE[Become Slave] + + MASTER --> MASTER_EVENT[emit('master-acquired')] + SLAVE --> SLAVE_EVENT[emit('election-completed', false)] + + MASTER_EVENT --> HEARTBEAT[Start Heartbeat Loop] + SLAVE_EVENT --> MONITOR[Start Master Monitoring] + end + + subgraph "Heartbeat Events" + HEARTBEAT --> RENEW{Renew Lock?} + RENEW -->|Success| CONTINUE[Continue as Master] + RENEW -->|Failed| LOST[emit('master-lost')] + + LOST --> STEPDOWN[Step Down to Slave] + STEPDOWN --> TRIGGER[Trigger New Election] + CONTINUE --> HEARTBEAT + end + + subgraph "Slave Management Events" + SLAVE_JOIN[New Slave Joins] --> SLAVE_REG[emit('slave-registered')] + SLAVE_TIMEOUT[Slave Heartbeat Timeout] --> SLAVE_REM[emit('slave-removed')] + + SLAVE_REG --> NOTIFY[Notify Dependent Services] + SLAVE_REM --> CLEANUP[Cleanup Assignments] + end +``` + +### Event Handler Integration + +```typescript +// Example: Camera module integration with MasterElection events +const masterElection = getMasterElection(); + +masterElection.on('master-acquired', () => { + // This process became master - start managing workers + masterSlaveWorkerCluster.becomeMaster(); + logger.info('Camera cluster: Became master, connecting to workers'); +}); + +masterElection.on('master-lost', () => { + // This process lost master status - become slave + masterSlaveWorkerCluster.becomeSlave(); + logger.info('Camera cluster: Became slave, disconnecting workers'); +}); + +masterElection.on('slave-registered', (slave: SlaveNode) => { + // New backend process joined - rebalance workload + masterSlaveWorkerCluster.handleSlaveJoined(slave); + logger.info(`Camera cluster: Slave ${slave.nodeId} joined`); +}); + +masterElection.on('slave-removed', (nodeId: string) => { + // Backend process left - reassign its workload + masterSlaveWorkerCluster.handleSlaveLeft(nodeId); + logger.info(`Camera cluster: Slave ${nodeId} removed`); +}); +``` + +## Process Coordination Patterns + +### Master Role Responsibilities + +```mermaid +graph TB + subgraph "Master Process Duties" + LOCK[Maintain Master Lock] --> HEARTBEAT[Send Heartbeats Every 10s] + HEARTBEAT --> MONITOR[Monitor All Slave Processes] + + MONITOR --> CLEANUP[Cleanup Stale Slave Entries] + CLEANUP --> BALANCE[Coordinate Resource Balancing] + + BALANCE --> WORKERS[Manage Worker Connections] + WORKERS --> ROUTE[Route Messages to Slaves] + + ROUTE --> STATUS[Provide Cluster Status] + STATUS --> LOCK + end + + subgraph "Master Failure Scenarios" + NETWORK[Network Partition] --> TIMEOUT[Lock Renewal Timeout] + CRASH[Process Crash] --> TIMEOUT + OVERLOAD[Resource Overload] --> TIMEOUT + + TIMEOUT --> EXPIRE[Master Lock Expires] + EXPIRE --> ELECTION[New Election Triggered] + ELECTION --> RECOVER[New Master Elected] + end +``` + +### Slave Role Responsibilities + +```mermaid +graph TB + subgraph "Slave Process Duties" + REGISTER[Register with Master Election] --> HEARTBEAT[Send Heartbeats Every 5s] + HEARTBEAT --> MONITOR[Monitor Master Existence] + + MONITOR --> PROCESS[Process Assigned Messages] + PROCESS --> REPORT[Report Status to Master] + + REPORT --> DETECT{Master Missing?} + DETECT -->|No| MONITOR + DETECT -->|Yes| ELECTION[Trigger Election] + + ELECTION --> ATTEMPT{Win Election?} + ATTEMPT -->|Yes| PROMOTE[Become Master] + ATTEMPT -->|No| CONTINUE[Continue as Slave] + + PROMOTE --> MASTER[Master Role Duties] + CONTINUE --> REGISTER + end +``` + +## Class Responsibilities Overview + +### Core Class Functions + +| Class | Primary Responsibility | Key Methods | Process Type | +|-------|----------------------|-------------|--------------| +| **MasterElection** | Distributed coordination and leadership election | • `start()` - Initialize election process
• `attemptElection()` - Try to acquire master lock
• `becomeMaster()` - Transition to master role
• `becomeSlave()` - Transition to slave role
• `waitForElectionComplete()` - Synchronous election waiting | Both Master & Slave | +| **Process Registry** | Process lifecycle management | • `registerProcess()` - Register with TTL
• `updateProcessHeartbeat()` - Renew TTL
• `getAllProcesses()` - Get active processes
• `getProcessesByRole()` - Filter by master/slave
• `unregisterProcess()` - Manual cleanup | Both Master & Slave | +| **Master Lock Manager** | Atomic lock operations | • `acquireMasterLock()` - Lua script lock acquisition
• `renewMasterLock()` - Lua script lock renewal
• `releaseMasterLock()` - Lua script lock release
• `checkMasterExists()` - Lua script master validation | Both Master & Slave | +| **Slave Management** | Slave registration and monitoring | • `registerAsSlave()` - Register as slave node
• `updateSlaveHeartbeat()` - Update slave status
• `cleanupStaleSlaves()` - Remove expired slaves
• `getSlaves()` - Get all registered slaves | Both Master & Slave | + +## Object Relationship Diagrams + +### Core Class Structure and Dependencies + +```mermaid +classDiagram + class MasterElection { + -nodeId: string + -identifier: string + -isMaster: boolean + -lockTtl: number + -heartbeatInterval: number + +start() + +stop() + +getIsMaster(): boolean + +getNodeId(): string + +waitForElectionComplete(): Promise~boolean~ + -attemptElection() + -acquireMasterLock(): Promise~boolean~ + -renewMasterLock(): Promise~boolean~ + -releaseMasterLock() + -becomeMaster() + -becomeSlave() + -checkMasterExists(): Promise~boolean~ + } + + class ProcessRegistry { + +registerProcess(processInfo) + +updateProcessHeartbeat(processId) + +getAllProcesses(): Promise~ProcessInfo[]~ + +getMasterProcess(): Promise~ProcessInfo~ + +getProcessesByRole(role): Promise~ProcessInfo[]~ + +unregisterProcess(processId) + +getProcessChannel(processId): Promise~string~ + } + + class SlaveManagement { + +registerAsSlave() + +unregisterFromSlaves() + +updateSlaveHeartbeat() + +getSlaves(): Promise~SlaveNode[]~ + +getSlave(nodeId): Promise~SlaveNode~ + +getSlaveCount(): Promise~number~ + -cleanupStaleSlaves() + -startSlaveManagement() + -stopSlaveManagement() + } + + class EventEmitter { + +on(event, listener) + +emit(event, ...args) + +once(event, listener) + +off(event, listener) + } + + MasterElection --|> EventEmitter : extends + MasterElection --* ProcessRegistry : contains + MasterElection --* SlaveManagement : contains + + MasterElection --> Redis : uses for coordination + ProcessRegistry --> Redis : uses hSetEx for TTL + SlaveManagement --> Redis : uses for slave state +``` + +### Redis Operations and Key Management + +```mermaid +graph TB + subgraph "Redis Key Structure" + MASTER[master-election:master
String - Current master ID with TTL] + HEARTBEAT[master-election:heartbeat
String - Master heartbeat timestamp] + MASTER_PROC[master-election:master_process
String - Master ProcessInfo JSON] + + PROCESSES[master-election:processes
Hash - ProcessInfo with per-entry TTL] + CHANNELS[master-election:channels
Hash - ProcessID → Channel mapping] + SLAVES[master-election:slaves
Hash - SlaveNode data] + end + + subgraph "Atomic Operations" + LUA1[Master Acquisition
SET master NX + SET heartbeat] + LUA2[Master Renewal
Check owner + PEXPIRE + SET heartbeat] + LUA3[Master Release
Check owner + DEL master + heartbeat] + LUA4[Master Check
GET master + GET heartbeat + validate TTL] + end + + subgraph "TTL Operations" + HSETEX1[Process Registration
hSetEx with 45s TTL per entry] + HSETEX2[Heartbeat Renewal
hSetEx renews TTL to 45s] + AUTO[Automatic Expiration
Redis removes expired entries] + end + + MASTER --> LUA1 + MASTER --> LUA2 + MASTER --> LUA3 + HEARTBEAT --> LUA1 + HEARTBEAT --> LUA2 + HEARTBEAT --> LUA4 + + PROCESSES --> HSETEX1 + PROCESSES --> HSETEX2 + PROCESSES --> AUTO +``` + +## Method Call Flow Analysis + +### Election and Role Transition Flow + +```mermaid +sequenceDiagram + participant App as Application + participant ME as MasterElection + participant R as Redis + participant Dep as Dependent Services + + Note over App,Dep: Election Initialization + + App->>+ME: start() + ME->>+ME: attemptElection() + ME->>+ME: emit('election-started') + + ME->>+R: Lua Script: acquireMasterLock() + + alt Lock acquired successfully + R-->>-ME: Success (1) + ME->>+ME: becomeMaster() + ME->>+ME: startHeartbeat() - every 10s + ME->>+ME: startSlaveManagement() + ME->>+Dep: emit('master-acquired') + ME->>+ME: emit('election-completed', true) + else Lock acquisition failed + R-->>-ME: Failed (0) + ME->>+ME: becomeSlave() + ME->>+R: hSet(slaves, nodeId, slaveInfo) + ME->>+ME: startPeriodicCheck() - every 5s + ME->>+Dep: emit('election-completed', false) + end + + Note over App,Dep: Heartbeat and Monitoring Loop + + loop Every 10 seconds (Master) / 5 seconds (Slave) + alt Process is Master + ME->>+R: Lua Script: renewMasterLock() + alt Renewal successful + R-->>-ME: Success (1) + ME->>+ME: Continue as master + else Renewal failed + R-->>-ME: Failed (0) + ME->>+ME: becomeSlave() + ME->>+Dep: emit('master-lost') + ME->>+ME: attemptElection() after delay + end + else Process is Slave + ME->>+R: Lua Script: checkMasterExists() + alt Master exists and healthy + R-->>-ME: Master found (1) + ME->>+ME: Continue monitoring + else No master or stale + R-->>-ME: No master (0) + ME->>+ME: attemptElection() with random delay + end + end + end +``` + +### Process Registration and TTL Management Flow + +```mermaid +sequenceDiagram + participant P as Process + participant ME as MasterElection + participant R as Redis + participant Auto as Redis TTL + + Note over P,Auto: Process Registration with TTL + + P->>+ME: registerProcess(processInfo) + + ME->>+R: hSetEx(processes, processId, processInfo, {EX: 45}) + Note over R: Entry set with 45 second TTL + R-->>-ME: Registration confirmed + + ME->>+R: hSet(channels, processId, channelName) + R-->>-ME: Channel mapping stored + + alt Process is master + ME->>+R: set(master_process, processInfo) + R-->>-ME: Master process info stored + end + + ME-->>-P: Registration complete + + Note over P,Auto: Heartbeat Loop (Every 10s) + + loop Every 10 seconds + P->>+ME: updateProcessHeartbeat(processId) + + ME->>+R: hGet(processes, processId) + R-->>-ME: Current process data + + ME->>+ME: Update lastSeen timestamp + + ME->>+R: hSetEx(processes, processId, updatedInfo, {EX: 45}) + Note over R: TTL renewed to 45 seconds + R-->>-ME: Heartbeat recorded + + ME-->>-P: Heartbeat updated + end + + Note over P,Auto: Automatic TTL Expiration (No heartbeat) + + Note over Auto: 45 seconds pass without heartbeat + Auto->>Auto: Process entry automatically expires + Auto->>R: Remove expired entry from hash + + Note over P,Auto: Other processes detect absence + + P->>+ME: getAllProcesses() + ME->>+R: hGetAll(processes) + R-->>-ME: Only active processes returned + Note over ME: Expired process not included + ME-->>-P: Updated process list +``` + +## System Architecture Diagrams + +### Master Election Cluster Architecture + +```mermaid +graph TB + subgraph "Backend Process Cluster" + M[Master Process
Elected Leader
🏆] + S1[Slave Process 1
Follower] + S2[Slave Process 2
Follower] + S3[Slave Process N
Follower] + end + + subgraph "Redis Coordination Layer" + R[(Redis Server)] + subgraph "Election Keys" + MK[master-election:master
Lock with TTL] + HK[master-election:heartbeat
Timestamp] + end + subgraph "Process Registry (TTL)" + PK[master-election:processes
Hash with per-entry TTL] + CK[master-election:channels
Process→Channel mapping] + end + subgraph "Slave Management" + SK[master-election:slaves
Slave registration data] + end + end + + subgraph "Dependent Services" + CAM[Camera Module
MasterSlaveWorkerCluster] + DS[Display Service
WebSocket Cluster] + OTHER[Other Services
...] + end + + M ===|Master Lock
Heartbeat Every 10s| MK + M ===|Timestamp Update| HK + M ===|TTL Registration
Heartbeat Renewal| PK + + S1 <-->|Monitor Master
Every 5s| R + S2 <-->|Monitor Master
Every 5s| R + S3 <-->|Monitor Master
Every 5s| R + + S1 ===|Slave Registration
Heartbeat Every 5s| SK + S2 ===|Slave Registration
Heartbeat Every 5s| SK + S3 ===|Slave Registration
Heartbeat Every 5s| SK + + M -.->|master-acquired
slave-registered
slave-removed| CAM + M -.->|Role transition events| DS + M -.->|Coordination events| OTHER + + S1 -.->|election-completed
master-lost| CAM + S2 -.->|Election events| DS + S3 -.->|Status events| OTHER +``` + +### TTL-Based Cleanup Architecture + +```mermaid +graph TB + subgraph "Process Lifecycle with TTL" + START[Process Starts] --> REG[Register with 45s TTL] + REG --> ACTIVE[Process Active] + + ACTIVE --> HB{Heartbeat?} + HB -->|Every 10s| RENEW[Renew TTL to 45s] + HB -->|Missed| COUNT[Count down TTL] + + RENEW --> ACTIVE + COUNT --> EXPIRE{TTL = 0?} + EXPIRE -->|No| COUNT + EXPIRE -->|Yes| CLEANUP[Redis Auto-Remove] + + CLEANUP --> DETECT[Other Processes Detect] + DETECT --> REBALANCE[Trigger Rebalancing] + end + + subgraph "Traditional Manual Cleanup vs TTL" + subgraph "❌ Manual Cleanup Problems" + RACE[Race Conditions] + LEAK[Memory Leaks] + STALE[Stale Data] + COMPLEX[Complex Logic] + end + + subgraph "✅ TTL-Based Benefits" + AUTO[Automatic Cleanup] + RELIABLE[Reliable Expiration] + SIMPLE[Simple Implementation] + SELF[Self-Healing] + end + end + + subgraph "TTL Management Operations" + HSETEX[hSetEx(key, field, value, {EX: 45})] + RENEWAL[Heartbeat renews TTL automatically] + EXPIRY[Redis removes expired entries] + + HSETEX --> RENEWAL + RENEWAL --> EXPIRY + EXPIRY --> HSETEX + end +``` + +### Election Timing and Coordination + +```mermaid +gantt + title Master Election Timeline + dateFormat X + axisFormat %s + + section Master Lock + Master Lock TTL (30s) :milestone, m1, 0, 0s + Lock Renewal (10s) :10, 20s + Lock Renewal (10s) :20, 30s + Lock Expires :milestone, m2, 30, 30s + + section Process TTL + Process Registration (45s) :milestone, p1, 0, 0s + Heartbeat Renewal (10s) :10, 20s + Heartbeat Renewal (10s) :20, 30s + Heartbeat Renewal (10s) :30, 40s + Process Expires :milestone, p2, 45, 45s + + section Election Events + Initial Election :milestone, e1, 0, 0s + Slave Monitoring (5s) :5, 10s + Slave Monitoring (5s) :10, 15s + Master Failure Detected :milestone, e2, 30, 30s + New Election Started :32, 35s + New Master Elected :milestone, e3, 35, 35s +``` + +## Event System Architecture + +### Event Flow and Dependencies + +```mermaid +graph TD + subgraph "MasterElection Events" + ES[election-started] --> EA{Election Attempt} + EA -->|Success| MA[master-acquired] + EA -->|Failed| EC[election-completed(false)] + + MA --> HB[Start Heartbeat Loop] + EC --> MON[Start Master Monitoring] + + HB --> RENEW{Heartbeat Success?} + RENEW -->|Success| CONT[Continue as Master] + RENEW -->|Failed| ML[master-lost] + + ML --> STEP[Step Down to Slave] + STEP --> MON + + CONT --> HB + MON --> CHECK{Master Missing?} + CHECK -->|Yes| ES + CHECK -->|No| MON + end + + subgraph "Slave Management Events" + SR[slave-registered] --> UP[Update Assignments] + SREM[slave-removed] --> CLEAN[Cleanup Assignments] + + UP --> NOTIFY[Notify Services] + CLEAN --> REBAL[Rebalance Load] + end + + subgraph "Error Handling Events" + ERR[error] --> LOG[Log Error Details] + LOG --> RECOVER[Attempt Recovery] + RECOVER --> ES + end + + subgraph "External Service Integration" + MA -.->|becomeMaster()| CAMERA[Camera Module] + ML -.->|becomeSlave()| CAMERA + SR -.->|slaveJoined()| CAMERA + SREM -.->|slaveLeft()| CAMERA + + MA -.->|Master role| DISPLAY[Display Service] + ML -.->|Slave role| DISPLAY + + MA -.->|Coordinate| OTHER[Other Services] + ML -.->|Follow| OTHER + end +``` + +### Event Sequence Patterns + +#### Master Failure and Recovery Pattern + +```mermaid +sequenceDiagram + participant M as Master Process + participant S1 as Slave 1 + participant S2 as Slave 2 + participant R as Redis + participant Svc as Dependent Services + + Note over M,Svc: Normal Operation + M->>R: Heartbeat renewal every 10s + S1->>R: Monitor master every 5s + S2->>R: Monitor master every 5s + + Note over M,Svc: Master Failure + M--XM: Process crashes + + Note over R: Master lock expires (30s) + R->>R: Lock TTL expires + + Note over S1,S2: Slaves detect master failure + S1->>R: checkMasterExists() → false + S2->>R: checkMasterExists() → false + + Note over S1,S2: Election race with random delay + S1->>S1: Random delay 1.2s + S2->>S2: Random delay 0.8s + + S2->>R: attemptElection() first + R->>S2: Success - became master + S2->>S2: emit('master-acquired') + S2->>Svc: becomeMaster() event + + S1->>R: attemptElection() second + R->>S1: Failed - master exists + S1->>S1: Continue as slave + + Note over S2,Svc: New master operational + S2->>R: Start heartbeat renewal + Svc->>S2: Acknowledge new master +``` + +## Configuration and Tuning + +### Timing Configuration + +```typescript +// MasterElection constructor parameters +interface MasterElectionConfig { + lockName: string = 'master-election'; // Redis key prefix + lockTtl: number = 30000; // Master lock TTL (30 seconds) + heartbeatInterval: number = 10000; // Master heartbeat interval (10 seconds) + checkInterval: number = 5000; // Slave monitoring interval (5 seconds) + identifier: string = 'cms-backend'; // Human-readable process identifier +} + +// TTL Configuration +const PROCESS_TTL_SECONDS = 45; // Process registration TTL +const SLAVE_TIMEOUT_MS = 15000; // Slave cleanup threshold (3x heartbeat) +const ELECTION_RANDOM_DELAY_MAX = 2000; // Max random delay to prevent collisions +``` + +### Redis Key Structure + +```typescript +// Election and coordination keys +const REDIS_KEYS = { + // Master election coordination + master: `${lockName}:master`, // Current master ID with TTL + heartbeat: `${lockName}:heartbeat`, // Master heartbeat timestamp + masterProcess: `${lockName}:master_process`, // Master ProcessInfo JSON + + // Process registry with TTL + processes: `${lockName}:processes`, // Hash: processId → ProcessInfo (TTL per entry) + channels: `${lockName}:channels`, // Hash: processId → channelName + + // Slave management + slaves: `${lockName}:slaves`, // Hash: nodeId → SlaveNode +}; + +// TTL settings +const TTL_CONFIG = { + masterLock: 30, // seconds - Master lock expiration + processEntry: 45, // seconds - Process registration TTL + heartbeatRenewal: 10, // seconds - How often to renew heartbeats + slaveMonitoring: 5, // seconds - How often slaves check master +}; +``` + +### Performance Characteristics + +#### Scalability Metrics +- **Election Speed**: < 100ms for uncontested election +- **Failover Time**: < 5 seconds from master failure to new election +- **Process Registration**: < 10ms per process registration +- **TTL Cleanup**: Automatic, no performance impact on application + +#### Resource Usage +- **Memory**: O(n) where n = number of backend processes +- **Redis Operations**: Atomic Lua scripts prevent race conditions +- **Network**: Minimal - only heartbeats and election attempts +- **CPU**: Negligible overhead for coordination operations + +#### Reliability Guarantees +- **Split-Brain Prevention**: Atomic Lua scripts ensure single master +- **Automatic Recovery**: TTL-based cleanup handles all failure scenarios +- **Event Consistency**: All role transitions emit events for service coordination +- **State Persistence**: Process registry survives Redis restarts + +## Public Interface Specification + +The MasterElection service provides a clean, event-driven interface for distributed coordination across backend processes. + +### Primary Interface: MasterElection Class + +#### Core Lifecycle Methods + +```typescript +/** + * Initialize and start the master election process + * @returns Promise - Resolves when election completes + */ +public async start(): Promise + +/** + * Stop master election and cleanup resources + * @returns Promise - Resolves when cleanup completes + */ +public async stop(): Promise + +/** + * Wait for election to complete with timeout + * @param timeoutMs - Maximum time to wait (default: 30000) + * @returns Promise - true if became master, false if slave + */ +public async waitForElectionComplete(timeoutMs: number = 30000): Promise +``` + +#### Status and Information Methods + +```typescript +/** + * Check if this process is currently the master + * @returns boolean - true if master, false if slave + */ +public getIsMaster(): boolean + +/** + * Get this process's unique node identifier + * @returns string - UUID-based node identifier + */ +public getNodeId(): string + +/** + * Get this process's human-readable identifier + * @returns string - Process identifier (e.g., 'cms-backend') + */ +public getIdentifier(): string + +/** + * Get or set process metadata for coordination + * @param metadata - Optional metadata to set + * @returns Record - Current metadata + */ +public setMetadata(metadata: Record): void +public getMetadata(): Record +``` + +#### Process Registry Methods + +```typescript +/** + * Register a process in the distributed registry with TTL + * @param processInfo - Process information including role and capabilities + * @returns Promise + */ +public async registerProcess(processInfo: ProcessInfo): Promise + +/** + * Update process heartbeat to renew TTL (45 seconds) + * @param processId - Process identifier to update + * @returns Promise + */ +public async updateProcessHeartbeat(processId: string): Promise + +/** + * Get all currently registered processes (auto-filtered by TTL) + * @returns Promise - Array of active processes + */ +public async getAllProcesses(): Promise + +/** + * Get current master process information + * @returns Promise - Master process or null if none + */ +public async getMasterProcess(): Promise + +/** + * Get processes filtered by role + * @param role - 'master' or 'slave' + * @returns Promise - Processes with specified role + */ +public async getProcessesByRole(role: 'master' | 'slave'): Promise +``` + +#### Slave Management Methods + +```typescript +/** + * Get all registered slave nodes + * @returns Promise - Array of active slaves + */ +public async getSlaves(): Promise + +/** + * Get specific slave node information + * @param nodeId - Slave node identifier + * @returns Promise - Slave info or null if not found + */ +public async getSlave(nodeId: string): Promise + +/** + * Get count of registered slave nodes + * @returns Promise - Number of active slaves + */ +public async getSlaveCount(): Promise +``` + +### Event System Interface + +#### Event Registration + +```typescript +// Type-safe event registration +masterElection.on('master-acquired', () => { + // This process became the master + console.log('Became master - start coordinating resources'); +}); + +masterElection.on('master-lost', () => { + // This process lost master status + console.log('Lost master status - step down to slave role'); +}); + +masterElection.on('election-completed', (isMaster: boolean) => { + // Election finished - role determined + console.log(`Election completed - role: ${isMaster ? 'MASTER' : 'SLAVE'}`); +}); + +masterElection.on('slave-registered', (slave: SlaveNode) => { + // New backend process joined cluster + console.log(`New slave joined: ${slave.nodeId}`); +}); + +masterElection.on('slave-removed', (nodeId: string) => { + // Backend process left cluster (TTL expired) + console.log(`Slave removed: ${nodeId}`); +}); + +masterElection.on('error', (error: Error) => { + // Election or coordination error occurred + console.error('Master election error:', error); +}); +``` + +#### Event Timing Guarantees + +- **master-acquired**: Emitted immediately after successful lock acquisition +- **master-lost**: Emitted immediately after failed lock renewal +- **election-completed**: Emitted after initial election resolves (master or slave) +- **slave-registered**: Emitted when new slave joins (master only) +- **slave-removed**: Emitted when slave TTL expires (master only) +- **error**: Emitted on Redis connection issues or election failures + +### Usage Patterns + +#### Basic Initialization and Coordination + +```typescript +import { initialize, getMasterElection } from '~/services/MasterElection'; + +// Initialize master election with custom settings +await initialize( + 'cms-cluster', // lockName - Redis key prefix + 30000, // lockTtl - Master lock TTL (30s) + 10000, // heartbeatInterval - Master heartbeat (10s) + 5000, // checkInterval - Slave monitoring (5s) + 'cms-backend-prod' // identifier - Human-readable name +); + +// Get election instance for event handling +const masterElection = getMasterElection(); + +// Wait for initial election to complete +const isMaster = await masterElection.waitForElectionComplete(); +console.log(`Process started as: ${isMaster ? 'MASTER' : 'SLAVE'}`); +``` + +#### Service Integration Pattern + +```typescript +// Camera module integration example +class CameraClusterService { + private masterElection: MasterElection; + + constructor() { + this.masterElection = getMasterElection(); + this.setupElectionHandlers(); + } + + private setupElectionHandlers() { + // Handle master role transitions + this.masterElection.on('master-acquired', () => { + this.becomeMaster(); + }); + + this.masterElection.on('master-lost', () => { + this.becomeSlave(); + }); + + // Handle cluster membership changes + this.masterElection.on('slave-registered', (slave) => { + this.handleSlaveJoined(slave); + }); + + this.masterElection.on('slave-removed', (nodeId) => { + this.handleSlaveLeft(nodeId); + }); + } + + private async becomeMaster() { + console.log('Camera service: Becoming master'); + + // Connect to all Python ML workers + await this.connectToAllWorkers(); + + // Start managing cluster assignments + this.startClusterManagement(); + + // Begin rebalancing subscriptions + this.startRebalancing(); + } + + private async becomeSlave() { + console.log('Camera service: Becoming slave'); + + // Disconnect from Python workers (master-only) + await this.disconnectFromWorkers(); + + // Stop cluster management + this.stopClusterManagement(); + + // Start listening for routed messages + this.startSlaveMessageHandling(); + } +} +``` + +#### Process Registration with Custom Capabilities + +```typescript +// Register this process with specific capabilities +await masterElection.registerProcess({ + processId: masterElection.getNodeId(), + nodeId: masterElection.getNodeId(), + role: masterElection.getIsMaster() ? 'master' : 'slave', + channelName: `worker:slave:${masterElection.getNodeId()}`, + lastSeen: new Date().toISOString(), + capabilities: { + canProcessDetections: true, // Can handle AI detection callbacks + maxSubscriptions: 100, // Maximum camera subscriptions + preferredWorkload: 80 // Preferred load percentage (0-100) + } +}); + +// Start heartbeat loop to maintain registration +setInterval(async () => { + await masterElection.updateProcessHeartbeat(masterElection.getNodeId()); +}, 10000); // Every 10 seconds +``` + +#### Cluster Monitoring and Status + +```typescript +// Monitor cluster status and health +async function monitorClusterHealth() { + // Get all active processes (TTL-filtered automatically) + const allProcesses = await masterElection.getAllProcesses(); + console.log(`Active processes: ${allProcesses.length}`); + + // Get current master + const masterProcess = await masterElection.getMasterProcess(); + if (masterProcess) { + console.log(`Master: ${masterProcess.processId} (${masterProcess.capabilities.maxSubscriptions} max subscriptions)`); + } + + // Get all slaves + const slaves = await masterElection.getSlaves(); + console.log(`Slaves: ${slaves.length}`); + slaves.forEach(slave => { + console.log(` Slave ${slave.nodeId}: last seen ${slave.lastSeen}`); + }); + + // Check if this process is master + if (masterElection.getIsMaster()) { + console.log('This process is the master - coordinating cluster'); + } else { + console.log('This process is a slave - following master'); + } +} + +// Run monitoring every 30 seconds +setInterval(monitorClusterHealth, 30000); +``` + +#### Graceful Shutdown Pattern + +```typescript +// Graceful shutdown with proper cleanup +process.on('SIGTERM', async () => { + console.log('Shutting down master election...'); + + try { + // Stop election and cleanup resources + await masterElection.stop(); + + // Master automatically releases lock + // Process TTL will expire naturally + // Slaves will detect and trigger new election + + console.log('Master election shutdown complete'); + } catch (error) { + console.error('Error during election shutdown:', error); + } + + process.exit(0); +}); +``` + +### Error Handling and Recovery + +#### Election Failure Scenarios + +```typescript +// Handle various failure modes +masterElection.on('error', (error) => { + console.error('Master election error:', error.message); + + // Common error types: + if (error.message.includes('Redis')) { + // Redis connection issues + console.log('Redis connectivity problem - will retry automatically'); + + } else if (error.message.includes('timeout')) { + // Election timeout + console.log('Election timeout - may indicate network issues'); + + } else if (error.message.includes('lock')) { + // Lock acquisition issues + console.log('Lock contention - normal during elections'); + } + + // Service continues running - election will retry automatically +}); + +// Handle network partitions +masterElection.on('master-lost', () => { + console.log('Lost master status - likely network partition or overload'); + + // Dependent services should gracefully step down + // New election will start automatically after random delay +}); +``` + +#### Recovery Guarantees + +- **Split-Brain Prevention**: Atomic Lua scripts ensure only one master exists +- **Automatic Failover**: New elections triggered immediately when master fails +- **TTL-Based Cleanup**: Processes automatically removed when heartbeats stop +- **State Recovery**: Process registry rebuilds automatically from active heartbeats +- **Event Consistency**: All role changes emit events for service coordination + +### Integration with Dependent Services + +The MasterElection service is designed to coordinate multiple backend services that need distributed leadership: + +#### Camera Module Integration +- Master: Connects to Python ML workers, manages subscriptions +- Slaves: Process routed detection messages, forward commands + +#### Display WebSocket Cluster +- Master: Manages WebSocket connection assignments across processes +- Slaves: Handle assigned display connections, route messages + +#### Database Migration Coordination +- Master: Executes database migrations and schema changes +- Slaves: Wait for master to complete before proceeding + +This specification provides a comprehensive understanding of the MasterElection service's distributed coordination capabilities and integration patterns for multi-process backend systems. \ No newline at end of file diff --git a/docs/WorkerConnection.md b/docs/WorkerConnection.md new file mode 100644 index 0000000..822c700 --- /dev/null +++ b/docs/WorkerConnection.md @@ -0,0 +1,1498 @@ +# Worker Connection Architecture Specification - Pure Declarative State Management + +## Overview + +The Camera Module implements a pure declarative architecture for managing connections to Python ML workers. This system uses the database as the single source of truth for desired subscription state, with automatic regeneration and reconciliation providing intelligent camera management, real-time object detection, and AI-powered content selection with automatic load balancing capabilities. + +**Key Architectural Principle**: Database mutations trigger complete state regeneration rather than incremental updates, ensuring consistency and eliminating complex state synchronization issues. + +## Architecture Components + +### Two-Cluster System + +The system consists of two distinct but coordinated clusters: + +1. **Backend Process Cluster**: Multiple CMS backend processes with leader election +2. **Worker Cluster**: Python ML workers for object detection processing + +### Master-Slave WebSocket Architecture + +- **Master Process**: Single elected backend process that maintains WebSocket connections to Python workers +- **Slave Processes**: All other backend processes that handle message routing and processing +- **Message Routing**: Master forwards worker messages to assigned slaves via Redis pub/sub channels +- **MasterElection Integration**: Automated master/slave role management with event-driven transitions +- **Seamless Scaling**: Backend processes can be added/removed without affecting WebSocket connections + +## Core Components + +### DetectorCluster +`cms-backend/modules/camera/services/DetectorCluster.ts` + +Primary interface for camera operations that abstracts the underlying distributed architecture. + +**Key Responsibilities:** +- Routes camera subscription requests through the cluster +- Manages detection callback registration and event emission +- Bridges CameraService with underlying MasterSlaveWorkerCluster +- Provides unified API regardless of master/slave status + +### MasterSlaveWorkerCluster +`cms-backend/modules/camera/services/MasterSlaveWorkerCluster.ts` + +Core distributed cluster implementation that handles declarative state management and worker assignment reconciliation. + +**Master Mode Responsibilities:** +- Maintains WebSocket connections to all Python workers +- Manages desired vs actual subscription state separation +- Implements intelligent global rebalancing algorithm +- Processes automatic reconciliation every 30 seconds +- Responds to slave join/leave events from MasterElection +- Generates fresh pre-signed model URLs for worker assignments + +**Slave Mode Responsibilities:** +- Submits desired subscription state changes to master +- Processes detection results routed from master +- Event-driven role transitions managed by MasterElection +- No direct worker management (delegated to master) + +### DetectorConnection +`cms-backend/modules/camera/services/DetectorConnection.ts` + +Individual WebSocket connection handler for Python workers. + +**Key Features:** +- Connection lifecycle management (connect, disconnect, reconnect) +- Exponential backoff reconnection with 10-second intervals +- Subscription state management and restoration after reconnection +- Real-time heartbeat monitoring with 10-second timeout +- Resource usage tracking (CPU, memory, GPU) + +## Data Structures + +### WorkerConnectionState +```typescript +interface WorkerConnectionState { + url: string; // Worker WebSocket URL + processId: string; // Backend process managing this worker + online: boolean; // Connection status + cpuUsage: number | null; // Worker CPU utilization + memoryUsage: number | null; // Worker memory usage + gpuUsage: number | null; // Worker GPU utilization + gpuMemoryUsage: number | null; // Worker GPU memory usage + subscriptionCount: number; // Active camera subscriptions + subscriptions: string[]; // List of subscription identifiers + lastHeartbeat: string; // Last heartbeat timestamp + connectedAt: string; // Connection established timestamp +} +``` + +### DesiredCameraSubscription +```typescript +interface DesiredCameraSubscription { + subscriptionIdentifier: string; // Format: ${displayId};${cameraId} + rtspUrl: string; // Camera RTSP stream URL + modelId: number; // AI model database ID + modelName: string; // AI model identifier + createdAt: string; // Subscription creation timestamp + + // Snapshot configuration + snapshotUrl?: string; // Optional snapshot endpoint URL + snapshotInterval?: number; // Snapshot interval in milliseconds + + // Image cropping parameters + cropX1?: number; // Crop region top-left X + cropY1?: number; // Crop region top-left Y + cropX2?: number; // Crop region bottom-right X + cropY2?: number; // Crop region bottom-right Y +} +``` + +### ActualCameraSubscription +```typescript +interface ActualCameraSubscription { + subscriptionIdentifier: string; // Format: ${displayId};${cameraId} + assignedWorkerUrl: string; // Worker handling this subscription + modelUrl: string; // AI model presigned URL (1hr TTL) + status: 'active' | 'pending' | 'failed' | 'recovering'; + assignedAt: string; // Worker assignment timestamp + lastSeen: string; // Last activity timestamp +} +``` + +### SlaveState +```typescript +interface SlaveState { + slaveId: string; // Unique slave identifier (process ID) + processId: string; // Backend process ID (same as slaveId) + online: boolean; // Always true (maintained by MasterElection) + workload: number; // Number of assigned workers (calculated) + lastSeen: string; // Last heartbeat from MasterElection + capabilities?: Record; // Metadata from MasterElection +} +``` + +### DetectorWorkerCommand +```typescript +interface DetectorWorkerCommand { + type: DetectorWorkerCommandType; + payload?: { + subscriptionIdentifier: string; + rtspUrl: string; + snapshotUrl?: string; + snapshotInterval?: number; + modelUrl: string; + modelName: string; + modelId: number; + cropX1?: number; + cropY1?: number; + cropX2?: number; + cropY2?: number; + }; +} + +enum DetectorWorkerCommandType { + SUBSCRIBE = "subscribe", + UNSUBSCRIBE = "unsubscribe", + REQUEST_STATE = "requestState", + PATCH_SESSION_RESULT = "patchSessionResult", + SET_SESSION_ID = "setSessionId" +} +``` + +### ImageDetectionResponse +```typescript +interface ImageDetectionResponse { + subscriptionIdentifier: string; + timestamp: Date; + data: { + detection: { + carModel?: string; + carBrand?: string; + carYear?: number; + bodyType?: string; + licensePlateText?: string; + licensePlateType?: string; + }; + modelId: number; + modelName: string; + }; +} +``` + +## Redis Data Architecture + +### Persistent Storage Keys +- `worker:connections` - Worker connection states and health metrics +- `worker:assignments` - Worker-to-slave assignment mappings +- `worker:desired_subscriptions` - Desired camera subscription state (user intent) +- `worker:actual_subscriptions` - Actual worker subscription assignments (system state) +- `master-election:slaves` - Slave registration and heartbeat (managed by MasterElection) + +### Communication Channels +- `worker:slave:{slaveId}` - Individual slave message routing channels +- `worker:messages:upstream` - Worker-to-master communication channel (currently unused) +- `worker:assignments:changed` - Assignment change broadcast notifications +- `worker:master:commands` - Database change notification channel (slaves → master) + +### Data Persistence Strategy +All Redis data uses **manual cleanup only** (no TTL) to ensure: +- Reliable state recovery after process restarts +- Consistent subscription persistence across failovers +- Predictable cleanup during planned maintenance +- Debug visibility into system state history + +## Pure Declarative Architecture + +### Concept Overview +The system implements a pure declarative approach where: +- **Database**: Single source of truth for desired state (Display+Camera+Playlist combinations) +- **Actual State**: What subscriptions are currently running on workers (stored in `worker:actual_subscriptions`) +- **Regeneration**: Master regenerates complete desired state from database on every change notification +- **Reconciliation**: Master continuously reconciles desired vs actual state via global rebalancing + +### Pure Declarative Benefits +- **Database as Truth**: Desired state always derived fresh from database, eliminating state synchronization issues +- **Zero Incremental Updates**: No complex state management, just "regenerate everything on change" +- **Automatic Recovery**: System heals itself by comparing database state vs actual worker state +- **Load Balancing**: Global optimization across all workers and subscriptions +- **Fault Tolerance**: Desired state survives all failures since it's always derived from database +- **Simplicity**: Database mutations just trigger regeneration - no complex command protocols + +### Pure Declarative Flow +```typescript +// Triggered by any database change +async handleDatabaseChange(changeType: string, entityId: string) { + // 1. Any process detects database change + await triggerSubscriptionUpdate(changeType, entityId); + + // 2. Master receives regeneration request + async handleMasterCommand(message) { + if (data.type === 'regenerate_subscriptions') { + await regenerateDesiredStateFromDatabase(); + } + } + + // 3. Master regenerates complete desired state from database + async regenerateDesiredStateFromDatabase() { + const activeDisplays = await db.display.findMany({ + where: { + AND: [ + { cameraIdentifier: { not: null } }, + { playlistId: { not: null } } + ] + }, + include: { camera: true, playlist: { include: { model: true } } } + }); + + // Generate fresh desired subscriptions from database + await storeDesiredSubscriptions(generateFromDisplays(activeDisplays)); + + // Trigger reconciliation + await rebalanceCameraSubscriptions(); + } + + // 4. Reconciliation (same VMware DRS algorithm) + async rebalanceCameraSubscriptions() { + const desired = await getDesiredSubscriptions(); // Fresh from database + const actual = await getActualSubscriptions(); // Current worker state + + // Find and fix differences using load balancing + await reconcileDifferences(desired, actual); + } +} + +// Intelligent worker selection (unchanged) +function findBestWorkerForLoad(workers, currentLoads) { + return workers + .map(worker => ({ + worker, + score: (currentLoads.get(worker.url) * 0.4) + // 40% load balance + (worker.cpuUsage * 0.35) + // 35% CPU usage + (worker.memoryUsage * 0.25) // 25% memory usage + })) + .sort((a, b) => a.score - b.score)[0].worker; // Lower score = better +} +``` + +### Simplified Reconciliation Flow +1. **Database Change**: Any process modifies database (Display, Camera, Playlist, Model) +2. **Trigger Notification**: Process sends `regenerate_subscriptions` to `worker:master:commands` +3. **Complete Regeneration**: Master queries database for all active Display+Camera+Playlist combinations +4. **Desired State Creation**: Master generates fresh desired subscriptions from database query results +5. **Diff Analysis**: Master compares fresh desired state vs current actual state on workers +6. **Global Reconciliation**: Master applies intelligent load balancing algorithm to reconcile differences +7. **Worker Commands**: Master sends subscription/unsubscription commands to workers +8. **State Update**: Master updates actual subscription state in Redis + +### Key Simplifications vs Previous Architecture +- **No Incremental State Management**: No complex tracking of individual subscription changes +- **No State Synchronization Issues**: Desired state always freshly derived from database +- **No Complex Command Protocols**: Only one command type: `regenerate_subscriptions` +- **No Partial Update Bugs**: Complete regeneration eliminates edge cases and race conditions +- **Zero Database-Redis Divergence**: Database is always the authoritative source +- **Simpler Service Layer**: Services just update database + trigger, no subscription logic + +## Class Responsibilities Overview + +### Core Class Functions + +| Class | Primary Responsibility | Key Functions | Process Type | +|-------|----------------------|---------------|--------------| +| **DetectorCluster** | Public API facade and event management | • `subscribeToCamera()` - Legacy interface (triggers regeneration)
• `addDetectionListener()` - Callback registration
• `getState()` - Cluster monitoring
• Event emission to external services | Both Master & Slave | +| **MasterSlaveWorkerCluster** | Pure declarative cluster coordination | **Master**: `regenerateDesiredStateFromDatabase()`, `rebalanceCameraSubscriptions()`, `connectToAllWorkers()`
**Slave**: Minimal role - just routes detection messages
**Both**: `handleDetectionMessage()` for callbacks | Both (different roles) | +| **DetectorConnection** | Individual worker WebSocket management | • `initialize()` - WebSocket connection setup
• `subscribeToCamera()` - Send subscription to worker
• `handleImageDetectionResponse()` - Process AI results
• `resubscribeAll()` - Restore subscriptions after reconnect | Master Only | +| **CameraService** | Database operations + trigger notifications | • `addCamera()` - Database create + trigger regeneration
• `updateCamera()` - Database update + trigger regeneration
• `removeCamera()` - Database delete + trigger regeneration | Both Master & Slave | +| **DisplayService** | Database operations + trigger notifications | • `registerDisplay()` - Database create + trigger regeneration
• `updateDisplay()` - Database update + trigger regeneration
• `deleteDisplay()` - Database delete + trigger regeneration | Both Master & Slave | +| **SubscriptionTrigger** | Simple notification system | • `triggerSubscriptionUpdate()` - Send regeneration request to master | Both Master & Slave | + +## Object Relationship Diagrams + +### Core Class Structure and Methods + +```mermaid +classDiagram + class CameraService { + +addCamera(identifier, rtspUrl) + +removeCamera(identifier) + +resubscribeCamera(identifier) + +getCameras() + +updateCamera(...) + -processDetection(data) + } + + class DetectorCluster { + +initialize() + +subscribeToCamera(...) + +unsubscribeFromCamera(subscriptionId) + +unsubscribeFromAllWithCameraID(cameraId) + +getState() + +addDetectionListener(subscriptionId, callback) + +addGlobalDetectionListener(callback) + -handleWorkerDetection(data) + } + + class MasterSlaveWorkerCluster { + +initialize() + +subscribeToCamera(...) + +storeCameraSubscription(subscription) + +getClusterState() + +shutdown() + -connectToAllWorkers() [MASTER] + -rebalanceCameraSubscriptions() [MASTER] + -triggerRebalancing() [MASTER] + -becomeMaster() + -becomeSlave() + -setupMasterElectionListeners() + } + + class DetectorConnection { + +initialize() + +subscribeToCamera(...) + +unsubscribeFromCamera(subscriptionId) + +getCameraImage(cameraId) + +setSessionId(displayId, sessionId) + +getState() + -connect() + -resubscribeAll() + -handleImageDetectionResponse(data) + -scheduleReconnect() + } + + CameraService --> DetectorCluster : "subscribeToCamera()\ngetState()" + DetectorCluster --> MasterSlaveWorkerCluster : "initialize()\nstoreCameraSubscription()" + MasterSlaveWorkerCluster --> DetectorConnection : "[MASTER] creates connections" +``` + +### Direct Function Call Relationships + +```mermaid +graph TD + API[API Routes] --> CS[CameraService] + CS --> |subscribeToCamera
getState
unsubscribeFromAllWithCameraID| DC[DetectorCluster] + DC --> |initialize
storeCameraSubscription
getClusterState
subscribeToCamera| MSC[MasterSlaveWorkerCluster] + + subgraph "Master Process Only" + MSC --> |connectToAllWorkers
creates connections| CONN[DetectorConnection] + CONN --> |WebSocket calls| PW[Python ML Worker] + end + + ME[MasterElection] --> |getIsMaster
getNodeId
getSlaves| MSC + WL[WorkerLogger] --> |attachToDetectorCluster| DC + + classDef masterOnly fill:#ffcccc + classDef external fill:#ffffcc + + class CONN masterOnly + class PW external + class API external +``` + +### Event-Driven Communication + +```mermaid +graph LR + subgraph "Internal Events" + MSC[MasterSlaveWorkerCluster] -.-> |emit detection| DC[DetectorCluster] + MSC -.-> |emit worker:online
emit worker:offline| DC + DC -.-> |emit worker:detection_result
emit worker:online
emit worker:offline| CS[CameraService] + DC -.-> |emit events| WL[WorkerLogger] + ME[MasterElection] -.-> |master-acquired
master-lost
slave-registered
slave-removed| MSC + end + + subgraph "Callback System" + CS -.-> |callback registration| DC + DC -.-> |detection callbacks| CS + end + + subgraph "WebSocket Events (Master Only)" + CONN[DetectorConnection] -.-> |handleWorkerMessage
handleWorkerOnline
handleWorkerOffline| MSC + PW[Python ML Worker] -.-> |IMAGE_DETECTION
STATE_REPORT| CONN + end + + classDef events fill:#e6f3ff + classDef callbacks fill:#fff2e6 + classDef websocket fill:#ffe6e6 + + class MSC,DC,CS,WL events + class CONN,PW websocket +``` + +### Redis Communication Patterns + +```mermaid +graph TB + subgraph "Master Process" + M[Master MasterSlaveWorkerCluster] + end + + subgraph "Slave Processes" + S1[Slave Process 1] + S2[Slave Process 2] + end + + subgraph "Redis Channels" + SC1[worker:slave:slave1] + SC2[worker:slave:slave2] + MC[worker:master:commands] + AC[worker:assignments:changed] + end + + subgraph "Redis Storage" + WC[worker:connections] + WA[worker:assignments] + WS[worker:slaves] + CS[worker:camera_subscriptions] + end + + M --> |publish detection routing| SC1 + M --> |publish detection routing| SC2 + M --> |publish assignments| AC + M --> |hSet/hGet state| WC + M --> |hSet/hGet assignments| WA + M --> |hSet/hGet subscriptions| CS + + S1 --> |publish commands| MC + S2 --> |publish commands| MC + S1 --> |hSet registration| WS + S2 --> |hSet registration| WS + + SC1 --> |subscribe| S1 + SC2 --> |subscribe| S2 + MC --> |subscribe| M + AC --> |subscribe all| S1 + AC --> |subscribe all| S2 +``` + +## Method Call Flow Analysis + +### Camera Subscription Flow (External Request → Worker) + +```mermaid +sequenceDiagram + participant API as API Routes + participant CS as CameraService + participant DB as Database + participant ST as SubscriptionTrigger + participant R as Redis + participant MSC as MasterSlaveCluster + participant CONN as DetectorConnection + participant W as Python Worker + + Note over API,W: Pure Declarative Flow + API->>+CS: POST /api/camera + CS->>+DB: db.cameraEntity.create({...}) + DB-->>-CS: Camera created + CS->>+ST: triggerSubscriptionUpdate('camera.created', id) + ST->>+R: publish(worker:master:commands, {type: 'regenerate_subscriptions', ...}) + + Note over R,MSC: Only Master Processes Commands + R->>+MSC: Master receives regeneration request + MSC->>+MSC: regenerateDesiredStateFromDatabase() + MSC->>+DB: Query all Display+Camera+Playlist combinations + DB-->>-MSC: Active display configurations + MSC->>+MSC: Generate fresh desired subscriptions + MSC->>+R: Store desired state in Redis + MSC->>+MSC: rebalanceCameraSubscriptions() + MSC->>+MSC: findBestWorkerForSubscription() + MSC->>+CONN: subscribeToCamera(subscriptionId, rtspUrl, ...) + CONN->>+W: WebSocket: {type: "subscribe", payload: {...}} + W-->>-CONN: WebSocket: {type: "stateReport", ...} + CONN->>-MSC: handleWorkerOnline(workerUrl) + MSC->>-R: Update actual subscription state + + Note over W,CS: Detection Processing (unchanged) + W->>CONN: Detection results + CONN->>MSC: Route to assigned slave + MSC->>CS: Detection callback + CS-->>-API: Camera added successfully +``` + +### Detection Processing Flow (Worker → External Callback) + +```mermaid +sequenceDiagram + participant W as Python Worker + participant CONN as DetectorConnection + participant MSC as MasterSlaveCluster + participant R as Redis + participant DC as DetectorCluster + participant CS as CameraService + + Note over W,CS: AI Detection Result Processing + W->>+CONN: WebSocket: {type: "imageDetection", subscriptionIdentifier, data} + CONN->>+MSC: handleWorkerMessage(ImageDetectionResponse) + + Note over MSC: Master finds assigned slave + MSC->>+MSC: findWorkerForSubscription(subscriptionId) + MSC->>+R: hGet(worker:assignments, workerUrl) + MSC->>+R: publish(worker:slave:{slaveId}, {type: 'detection', ...}) + + Note over R: Redis routes to assigned slave + R-->>+MSC: Slave receives detection message + MSC->>+MSC: handleDetectionMessage(message) + MSC->>+DC: emit('detection', detectionData) + + Note over DC: Process detection and trigger callbacks + DC->>+DC: handleWorkerDetection(data) + DC->>+DC: detectionListeners.get(subscriptionId).forEach(callback) + DC->>+CS: callback(detectionData) + DC->>+DC: emit('worker:detection_result', {url, cameraId, detections}) + + Note over CS: External service processes detection + CS->>+CS: processDetection(data) + CS-->>CS: updateAnalytics(), triggerDecisionTrees() +``` + +### Master Election and Failover Flow + +```mermaid +sequenceDiagram + participant ME as MasterElection + participant MSC1 as MasterSlaveCluster (Process 1) + participant MSC2 as MasterSlaveCluster (Process 2) + participant R as Redis + participant W1 as Python Worker 1 + participant W2 as Python Worker 2 + + Note over ME,W2: Master Failover Scenario + + %% Initial state + ME->>+MSC1: emit('master-acquired') + MSC1->>+MSC1: becomeMaster() + ME->>+MSC2: emit('master-lost') + MSC2->>+MSC2: becomeSlave() + + ME->>+R: Automatic slave registration + MSC1->>+W1: WebSocket connection (Master) + MSC1->>+W2: WebSocket connection (Master) + + Note over MSC1: Original master fails + MSC1--xMSC1: Process crash/network failure + + %% MasterElection detects failure and triggers new election + ME->>+ME: Detect failed master, trigger election + ME->>+MSC2: emit('master-acquired') + MSC2->>+MSC2: becomeMaster() + + %% Master recovery process + MSC2->>+MSC2: connectToAllWorkers() + MSC2->>+W1: WebSocket reconnection + MSC2->>+W2: WebSocket reconnection + + MSC2->>+MSC2: healClusterAssignments() + MSC2->>+R: hGetAll(worker:camera_subscriptions) + MSC2->>+MSC2: rebalanceCameraSubscriptions() + + %% Restore subscriptions + MSC2->>+W1: WebSocket: SUBSCRIBE commands + MSC2->>+W2: WebSocket: SUBSCRIBE commands + + Note over MSC2,W2: New master operational - slave registration handled by MasterElection +``` + +## System Architecture Diagrams + +### Master-Slave Cluster Architecture + +```mermaid +graph TB + subgraph "Backend Process Cluster" + M[Master Process
NodeJS Backend] + S1[Slave Process 1
NodeJS Backend] + S2[Slave Process 2
NodeJS Backend] + S3[Slave Process N
NodeJS Backend] + end + + subgraph "Python Worker Cluster" + W1[Python ML Worker 1
WebSocket Server] + W2[Python ML Worker 2
WebSocket Server] + W3[Python ML Worker N
WebSocket Server] + end + + subgraph "Redis Coordination Layer" + R[(Redis)] + R --- C1[worker:slave:* channels] + R --- C2[worker:connections state] + R --- C3[worker:assignments mapping] + R --- C4[worker:camera_subscriptions] + end + + M ===|WebSocket Connections
Only Master| W1 + M ===|WebSocket Connections
Only Master| W2 + M ===|WebSocket Connections
Only Master| W3 + + M <-->|Pub/Sub Messages| R + S1 <-->|Pub/Sub Messages| R + S2 <-->|Pub/Sub Messages| R + S3 <-->|Pub/Sub Messages| R + + M -.->|Route Messages| S1 + M -.->|Route Messages| S2 + M -.->|Route Messages| S3 +``` + +### Data Flow Architecture + +```mermaid +sequenceDiagram + participant CS as CameraService + participant DC as DetectorCluster + participant MS as MasterSlaveCluster + participant R as Redis + participant W as Python Worker + participant S as Slave Process + + Note over CS,S: Camera Subscription Flow + + CS->>DC: subscribeToCamera(cameraId, rtspUrl, modelUrl, ...) + DC->>MS: storeCameraSubscription({...}) + + alt Master Process + MS->>MS: findBestWorkerForSubscription() + MS->>R: hSet(camera_subscriptions, subscriptionId, {...}) + MS->>W: WebSocket: SUBSCRIBE command + W->>MS: STATE_REPORT (subscription confirmed) + MS->>R: publish(worker:slave:{slaveId}, detection_message) + else Slave Process + MS->>R: publish(worker:master:commands, subscribe_command) + Note over MS: Routes to master for execution + end + + Note over CS,S: Detection Processing Flow + + W->>MS: WebSocket: IMAGE_DETECTION response + MS->>MS: findSlaveForWorker(workerUrl) + MS->>R: publish(worker:slave:{slaveId}, detection_data) + R->>S: Redis pub/sub delivery + S->>DC: emit('detection', detectionData) + DC->>CS: callback(detectionData) +``` + +### Subscription Lifecycle Management + +```mermaid +stateDiagram-v2 + [*] --> Pending: Camera Subscription Request + + Pending --> Active: Worker accepts subscription + Pending --> Failed: Worker rejects/unavailable + Pending --> Recovering: Assignment change needed + + Active --> Recovering: Worker goes offline + Active --> [*]: Unsubscribe request + + Recovering --> Active: Reassigned to online worker + Recovering --> Failed: No workers available + Recovering --> [*]: Subscription expired + + Failed --> Recovering: Worker becomes available + Failed --> [*]: Max retries exceeded + + note right of Recovering + Automatic rebalancing every 30s + Master detects offline workers + Reassigns to healthy workers + end note +``` + +### Worker Connection State Machine + +```mermaid +stateDiagram-v2 + [*] --> Connecting: initialize() + + Connecting --> Online: WebSocket connected + STATE_REPORT received + Connecting --> Reconnecting: Connection failed + + Online --> Offline: Heartbeat timeout (10s) + Online --> Reconnecting: WebSocket error/close + Online --> [*]: close() called + + Offline --> Reconnecting: Scheduled reconnect (10s) + Offline --> [*]: close() called + + Reconnecting --> Online: Reconnection successful + Reconnecting --> Reconnecting: Reconnection failed (retry) + Reconnecting --> [*]: close() called + + note right of Online + - Sends heartbeat every 2s + - Processes subscriptions + - Reports resource usage + - Handles detection results + end note +``` + +### Redis Channel Communication Flow + +```mermaid +graph LR + subgraph "Master Process" + M[Master] + WS1[WebSocket to Worker 1] + WS2[WebSocket to Worker 2] + end + + subgraph "Slave Processes" + S1[Slave 1] + S2[Slave 2] + end + + subgraph "Redis Channels" + CH1[worker:slave:slave1] + CH2[worker:slave:slave2] + CH3[worker:messages:upstream] + CH4[worker:assignments:changed] + end + + WS1 -->|Detection Data| M + WS2 -->|Detection Data| M + + M -->|Route by Assignment| CH1 + M -->|Route by Assignment| CH2 + + CH1 -->|Subscribed| S1 + CH2 -->|Subscribed| S2 + + S1 -->|Commands/Responses| CH3 + S2 -->|Commands/Responses| CH3 + CH3 -->|Subscribed| M + + M -->|Assignment Updates| CH4 + CH4 -->|Subscribed| S1 + CH4 -->|Subscribed| S2 +``` + +### Detailed Message Flow by Channel + +```mermaid +graph TB + subgraph "Python ML Workers" + W1[Worker 1
ws://worker1:8000] + W2[Worker 2
ws://worker2:8000] + W3[Worker N
ws://workerN:8000] + end + + subgraph "Master Process (Only One)" + M[Master Backend Process] + subgraph "Master Managed Data" + WC1[WebSocket Connection Pool] + AS[Assignment State] + SUB[Subscription Manager] + end + end + + subgraph "Redis Channels & Storage" + subgraph "Individual Slave Channels" + SC1["worker:slave:slave-uuid-1"] + SC2["worker:slave:slave-uuid-2"] + SC3["worker:slave:slave-uuid-N"] + end + + subgraph "Master Coordination Channels" + MC["worker:master:commands"] + ACH["worker:assignments:changed"] + UPC["worker:messages:upstream"] + SEC["worker:subscription:events"] + end + + subgraph "Persistent Storage" + WCS["worker:connections
(Worker Health States)"] + WAS["worker:assignments
(Worker→Slave Mapping)"] + WSS["worker:slaves
(Slave Registration)"] + CSS["worker:camera_subscriptions
(Subscription Persistence)"] + end + end + + subgraph "Slave Processes" + S1[Slave Process 1
slave-uuid-1] + S2[Slave Process 2
slave-uuid-2] + S3[Slave Process N
slave-uuid-N] + end + + %% WebSocket Communications (Master Only) + W1 -.->|"WebSocket Messages:
• IMAGE_DETECTION
• STATE_REPORT
• PATCH_SESSION"| WC1 + W2 -.->|"WebSocket Messages:
• IMAGE_DETECTION
• STATE_REPORT
• PATCH_SESSION"| WC1 + W3 -.->|"WebSocket Messages:
• IMAGE_DETECTION
• STATE_REPORT
• PATCH_SESSION"| WC1 + + WC1 -.->|"WebSocket Commands:
• SUBSCRIBE
• UNSUBSCRIBE
• REQUEST_STATE
• SET_SESSION_ID"| W1 + WC1 -.->|"WebSocket Commands:
• SUBSCRIBE
• UNSUBSCRIBE
• REQUEST_STATE
• SET_SESSION_ID"| W2 + WC1 -.->|"WebSocket Commands:
• SUBSCRIBE
• UNSUBSCRIBE
• REQUEST_STATE
• SET_SESSION_ID"| W3 + + %% Master Redis Operations + M -->|"hSet() operations:
• Worker states
• Assignments
• Subscriptions"| WCS + M -->|"hSet() operations:
• Worker→Slave mapping
• Load balancing data"| WAS + M -->|"hSet() operations:
• Subscription details
• Assignment tracking"| CSS + + %% Master to Slave Routing + M -->|"Detection Routing:
{type: 'detection',
workerUrl: string,
data: ImageDetectionResponse,
timestamp: string}"| SC1 + M -->|"Detection Routing:
{type: 'detection',
workerUrl: string,
data: ImageDetectionResponse,
timestamp: string}"| SC2 + M -->|"Detection Routing:
{type: 'detection',
workerUrl: string,
data: ImageDetectionResponse,
timestamp: string}"| SC3 + + M -->|"Assignment Updates:
{type: 'assignments_updated',
assignments: Record,
timestamp: string}"| ACH + + %% Slave to Master Communication + S1 -->|"Slave Commands:
{type: 'subscribe_camera',
subscriptionIdentifier: string,
rtspUrl: string,
modelUrl: string,
modelId: number,
snapshotUrl?: string,
cropX1?: number, ...}"| MC + S2 -->|"Slave Commands:
{type: 'subscribe_camera',
subscriptionIdentifier: string,
rtspUrl: string,
modelUrl: string,
modelId: number,
snapshotUrl?: string,
cropX1?: number, ...}"| MC + S3 -->|"Slave Commands:
{type: 'subscribe_camera',
subscriptionIdentifier: string,
rtspUrl: string,
modelUrl: string,
modelId: number,
snapshotUrl?: string,
cropX1?: number, ...}"| MC + + %% Slave Registration and Heartbeats + S1 -->|"hSet() Slave Registration:
{slaveId: string,
processId: string,
online: boolean,
workload: number,
lastSeen: string,
capabilities: {...}}"| WSS + S2 -->|"hSet() Slave Registration:
{slaveId: string,
processId: string,
online: boolean,
workload: number,
lastSeen: string,
capabilities: {...}}"| WSS + S3 -->|"hSet() Slave Registration:
{slaveId: string,
processId: string,
online: boolean,
workload: number,
lastSeen: string,
capabilities: {...}}"| WSS + + %% Channel Subscriptions + SC1 -->|"Subscribed"| S1 + SC2 -->|"Subscribed"| S2 + SC3 -->|"Subscribed"| S3 + + MC -->|"Subscribed"| M + ACH -->|"Subscribed (All Slaves)"| S1 + ACH -->|"Subscribed (All Slaves)"| S2 + ACH -->|"Subscribed (All Slaves)"| S3 + + style M fill:#ff9999 + style WC1 fill:#ffcc99 + style AS fill:#ffcc99 + style SUB fill:#ffcc99 + style S1 fill:#99ccff + style S2 fill:#99ccff + style S3 fill:#99ccff +``` + +### Channel Message Specification + +| Channel Name | Direction | Message Type | Sender | Receiver | Payload Structure | Purpose | +|--------------|-----------|--------------|---------|-----------|-------------------|---------| +| `worker:slave:{slaveId}` | Master→Slave | `detection` | Master Process | Assigned Slave | `{type: 'detection', workerUrl: string, data: ImageDetectionResponse, timestamp: string}` | Route AI detection results from workers to processing slaves | +| `worker:master:commands` | Slave→Master | `regenerate_subscriptions` | Any Process | Master Process | `{type: 'regenerate_subscriptions', reason: string, triggeredBy: string, timestamp: string}` | Notify master that database changed and subscriptions need regeneration | +| `worker:assignments:changed` | Master→All Slaves | `assignments_updated` | Master Process | All Slave Processes | `{type: 'assignments_updated', assignments: Record, timestamp: string}` | Broadcast worker-to-slave assignment changes for rebalancing | +| `worker:messages:upstream` | Slave→Master | Various | Any Slave Process | Master Process | `{type: string, slaveId: string, data: any, timestamp: string}` | General slave-to-master communication (currently unused) | + +### Redis Hash Storage Specification + +| Redis Key | Data Type | Content | Update Pattern | Cleanup Strategy | +|-----------|-----------|---------|----------------|-------------------| +| `worker:connections` | Hash Map | `{[workerUrl]: JSON.stringify(WorkerConnectionState)}` | Master updates every 2s | Manual cleanup only | +| `worker:assignments` | Hash Map | `{[workerUrl]: slaveId}` | Master updates on rebalancing | Manual cleanup only | +| `worker:camera_subscriptions` | Hash Map | `{[subscriptionId]: JSON.stringify(CameraSubscription)}` | Master on subscription changes | Manual cleanup only | +| `master-election:slaves` | Hash Map | `{[nodeId]: JSON.stringify(SlaveNode)}` | MasterElection service manages | TTL-based cleanup | + +### WebSocket Message Protocol + +| Direction | Message Type | JSON Structure | Trigger | Response Expected | +|-----------|--------------|----------------|---------|-------------------| +| Backend→Worker | `SUBSCRIBE` | `{type: "subscribe", payload: {subscriptionIdentifier, rtspUrl, snapshotUrl?, snapshotInterval?, modelUrl, modelName, modelId, cropX1?, cropY1?, cropX2?, cropY2?}}` | Camera subscription request | STATE_REPORT confirmation | +| Backend→Worker | `UNSUBSCRIBE` | `{type: "unsubscribe", payload: {subscriptionIdentifier}}` | Camera unsubscription | STATE_REPORT confirmation | +| Backend→Worker | `REQUEST_STATE` | `{type: "requestState"}` | Health check or monitoring | STATE_REPORT response | +| Backend→Worker | `SET_SESSION_ID` | `{type: "setSessionId", payload: {displayIdentifier, sessionId}}` | Associate session with display | None | +| Backend→Worker | `PATCH_SESSION_RESULT` | `{type: "patchSessionResult", payload: {sessionId, success, message?}}` | Session update response | None | +| Worker→Backend | `IMAGE_DETECTION` | `{type: "imageDetection", subscriptionIdentifier, timestamp, data: {detection: {carModel?, carBrand?, carYear?, bodyType?, licensePlateText?, licensePlateType?}, modelId, modelName}}` | AI detection result | None | +| Worker→Backend | `STATE_REPORT` | `{type: "stateReport", cpuUsage, memoryUsage, gpuUsage?, gpuMemoryUsage?, cameraConnections: [{subscriptionIdentifier, modelId, modelName, online, cropX?, cropY?}]}` | Periodic health report (every 2s) | None | +| Worker→Backend | `PATCH_SESSION` | `{type: "patchSession", sessionId, data: any}` | Session data update from ML processing | PATCH_SESSION_RESULT | + +## Event System Architecture + +### Event Flow Hierarchy + +```mermaid +graph TD + subgraph "Service Layer" + CS[CameraService] + end + + subgraph "Cluster Layer" + DC[DetectorCluster] + DC --> DCE[Detection Events] + DC --> WOE[Worker Online Events] + DC --> WOFE[Worker Offline Events] + end + + subgraph "Worker Management Layer" + MS[MasterSlaveWorkerCluster] + MS --> DE[detection] + MS --> WC[worker:connected] + MS --> WD[worker:disconnected] + MS --> WSE[worker:websocket_error] + MS --> WON[worker:online] + MS --> WOFF[worker:offline] + MS --> WSR[worker:state_report] + end + + subgraph "Connection Layer" + DConn[DetectorConnection] + DConn --> IMG[IMAGE_DETECTION] + DConn --> STATE[STATE_REPORT] + DConn --> PATCH[PATCH_SESSION] + end + + DConn --> MS + MS --> DC + DC --> CS + + IMG -.-> DE + STATE -.-> WSR + WC -.-> WOE + WD -.-> WOFE +``` + +### Message Types and Routing + +#### WebSocket Message Types (Python Worker → Backend) +- `IMAGE_DETECTION`: AI detection results from camera streams +- `STATE_REPORT`: Worker health, resource usage, and subscription status +- `PATCH_SESSION`: Session data updates from worker processing + +#### Redis Channel Message Types +- `detection`: Detection results routed from master to assigned slave +- `command_response`: Command acknowledgment and status updates +- `heartbeat`: Worker and slave health monitoring messages +- `assignments_updated`: Worker-to-slave assignment change notifications + +#### Internal Event Types +- `worker:online`: Worker connection established and ready +- `worker:offline`: Worker connection lost or health check failed +- `worker:connected`: WebSocket connection opened (not necessarily ready) +- `worker:disconnected`: WebSocket connection closed +- `worker:websocket_error`: WebSocket communication errors +- `worker:detection_result`: Processed detection with metadata +- `worker:state_report`: Worker resource and subscription status + +## Subscription Management + +### Camera Subscription Flow + +1. **Registration Phase** + - `CameraService.subscribeToCamera()` → `DetectorCluster.subscribeToCamera()` + - Master process finds optimal worker using load balancing algorithm + - Subscription stored in Redis with full configuration including crop parameters + - Master sends WebSocket SUBSCRIBE command to assigned worker + +2. **Processing Phase** + - Python worker establishes RTSP connection to camera + - Worker performs AI inference on video stream frames + - Detection results sent back via WebSocket with subscription identifier + - Master routes results to appropriate slave based on worker assignments + +3. **Rebalancing Phase** + - Master monitors worker health every 30 seconds + - Orphaned subscriptions (offline workers) automatically detected + - Load balancing algorithm reassigns cameras to healthy workers + - Fresh model URLs generated to handle S3 presigned URL expiration + +### Load Balancing Algorithm + +```typescript +// Simplified load balancing logic +function findBestWorkerForSubscription(onlineWorkers, allSubscriptions) { + return onlineWorkers + .sort((a, b) => { + const loadA = getSubscriptionCount(a.url); + const loadB = getSubscriptionCount(b.url); + if (loadA !== loadB) { + return loadA - loadB; // Prefer lower load + } + return (a.cpuUsage || 0) - (b.cpuUsage || 0); // Then prefer lower CPU + })[0]; +} +``` + +### Automatic Failover Process + +1. **Detection**: Master detects worker offline via missed heartbeats (10s timeout) +2. **Identification**: System identifies all camera subscriptions assigned to offline worker +3. **Reassignment**: Load balancer selects optimal replacement worker +4. **Migration**: Subscription updated in Redis with new worker assignment +5. **Resubscription**: Master sends SUBSCRIBE command to new worker with fresh model URL +6. **Verification**: New worker confirms subscription and begins processing + +## Resource Management + +### Connection Pooling +- Master maintains persistent WebSocket connections to all configured workers +- Connection sharing across all backend processes reduces resource overhead +- Automatic reconnection with exponential backoff prevents connection storms + +### Memory Management +- Redis data uses manual cleanup to prevent accidental state loss +- Subscription callbacks stored in local memory with automatic cleanup on unsubscribe +- Worker resource usage tracked in real-time to prevent overload + +### CPU and GPU Monitoring +- Workers report resource usage every 2 seconds via STATE_REPORT messages +- Load balancing algorithm considers CPU usage when assigning new subscriptions +- GPU utilization tracked for ML model optimization and capacity planning + +## Error Handling + +### Connection Error Recovery +- **Exponential Backoff**: 10-second fixed interval reconnection attempts +- **Circuit Breaker**: Automatic failover prevents overwhelming failed workers +- **Graceful Degradation**: System continues operating with available workers + +### Master Election Failover +- **Leadership Transfer**: New master elected via Redis-based coordination +- **State Recovery**: Worker connections and subscriptions restored from Redis persistence +- **Seamless Transition**: No subscription loss during master failover process + +### Monitoring and Observability + +#### Structured Logging Topics +- `detector-cluster`: High-level cluster operations and state changes +- `master-slave-worker-cluster`: Worker assignment and rebalancing operations +- `DetectorConnection`: WebSocket connection events and message processing + +#### Monitoring Information +- Subscription identifier format: `${displayId};${cameraId}` for traceability +- Worker assignment tracking with process ID and timestamp correlation +- Redis pub/sub message routing with structured logging +- Heartbeat and health check timing with millisecond precision + +## Configuration Parameters + +### Timing Configuration +```typescript +const WORKER_TIMEOUT_MS = 10000; // Worker heartbeat timeout +const SLAVE_HEARTBEAT_INTERVAL = 5000; // Slave heartbeat frequency +const SLAVE_TIMEOUT = 15000; // Slave registration timeout +const REBALANCE_INTERVAL = 30000; // Automatic rebalancing frequency +const STATE_UPDATE_INTERVAL = 2000; // Worker state update frequency +const RECONNECT_DELAY = 10000; // WebSocket reconnection delay +``` + +### Environment Variables +```bash +DETECTOR_WORKERS=ws://worker1:8000,ws://worker2:8000 # Python worker URLs +REDIS_HOST=localhost # Redis coordination server +REDIS_PORT=6379 # Redis server port +REDIS_PASSWORD=secure_password # Redis authentication +DETECT_DEBUG=true # Enable detailed structured logging +``` + +## Performance Characteristics + +### Scalability Metrics +- **Horizontal Scaling**: Add backend processes without WebSocket connection changes +- **Worker Scaling**: Python ML workers scale independently of backend processes +- **Redis Optimization**: Efficient pub/sub routing with minimal memory overhead + +### Throughput Capabilities +- **Camera Subscriptions**: Support for 100+ simultaneous camera streams per worker +- **Detection Processing**: Sub-second AI inference with real-time result delivery +- **Message Routing**: Sub-millisecond Redis pub/sub message delivery + +### Resource Efficiency +- **Connection Multiplexing**: Single WebSocket per worker shared across all processes +- **Memory Usage**: Lightweight subscription state with callback cleanup +- **Network Optimization**: Binary WebSocket frames with JSON payload compression + +## Public Interface Specification + +The distributed worker cluster exposes a clean, simplified interface to external services like CameraService, hiding the complexity of the underlying master-slave architecture. All interactions go through the `DetectorCluster` class, which serves as the primary facade. + +### Primary Interface: DetectorCluster + +The `DetectorCluster` class in `/services/DetectorCluster.ts` provides the main public interface that external services interact with. It abstracts away the distributed architecture complexity and provides consistent behavior regardless of whether the current process is a master or slave. + +#### Core Interface Methods + +##### Camera Subscription Management + +```typescript +/** + * Subscribe to a camera stream for AI detection processing + * @param subscriptionIdentifier - Unique identifier format: "${displayId};${cameraId}" + * @param rtspUrl - RTSP stream URL for the camera + * @param modelUrl - Pre-signed S3 URL for AI model (1hr TTL) + * @param modelId - Database ID of the AI model + * @param modelName - Human-readable model identifier + * @param callback - Function called when detection results are received + * @param snapshotUrl - Optional HTTP endpoint for camera snapshots + * @param snapshotInterval - Optional snapshot capture interval in milliseconds + * @param cropX1, cropY1, cropX2, cropY2 - Optional image crop coordinates + * @returns Promise - Always returns true (errors thrown as exceptions) + */ +public async subscribeToCamera( + subscriptionIdentifier: string, + rtspUrl: string, + modelUrl: string, + modelId: number, + modelName: string, + callback: Function, + snapshotUrl?: string, + snapshotInterval?: number, + cropX1?: number, + cropY1?: number, + cropX2?: number, + cropY2?: number +): Promise +``` + +**Behavior:** +- **Master Process**: Stores subscription in Redis, assigns to optimal worker, sends WebSocket command +- **Slave Process**: Routes subscription request to master via Redis pub/sub +- **Callback Registration**: Stores callback locally for detection result processing +- **Persistence**: All subscription details stored in Redis for failover recovery +- **Load Balancing**: Automatically selects best available worker based on CPU and subscription load + +```typescript +/** + * Unsubscribe from a specific camera stream + * @param subscriptionIdentifier - The subscription to remove + * @returns Promise - Success status + */ +public async unsubscribeFromCamera(subscriptionIdentifier: string): Promise +``` + +**Behavior:** +- Removes local callback listeners immediately +- Subscription cleanup handled automatically by cluster rebalancing +- Safe to call multiple times (idempotent operation) + +```typescript +/** + * Remove all subscriptions for a specific camera across all displays + * @param cameraIdentifier - The camera ID to unsubscribe from all displays + * @returns Promise + */ +public async unsubscribeFromAllWithCameraID(cameraIdentifier: string): Promise +``` + +**Behavior:** +- Finds all subscription identifiers matching pattern `*;${cameraIdentifier}` +- Removes all local callbacks for matched subscriptions +- Cluster automatically handles worker-side cleanup + +##### Event Registration and Callbacks + +```typescript +/** + * Register a callback for detection results from a specific subscription + * @param subscriptionIdentifier - Target subscription + * @param callback - Function to call with detection data + */ +public addDetectionListener(subscriptionIdentifier: string, callback: Function): void + +/** + * Register a global callback for all detection results + * @param callback - Function to call with any detection data + */ +public addGlobalDetectionListener(callback: Function): void +``` + +**Detection Callback Signature:** +```typescript +type DetectionCallback = (data: { + subscriptionIdentifier: string; + timestamp: Date; + data: { + detection: { + carModel?: string; + carBrand?: string; + carYear?: number; + bodyType?: string; + licensePlateText?: string; + licensePlateType?: string; + }; + modelId: number; + modelName: string; + }; +}) => void; +``` + +##### Cluster State Management + +```typescript +/** + * Get comprehensive cluster state for monitoring and status reporting + * @returns Promise + */ +public async getState(): Promise + +/** + * Legacy method - rebalancing now happens automatically + * @returns Promise - Always returns true + */ +public async rebalanceWorkers(): Promise +``` + +**DetectorClusterState Interface:** +```typescript +interface DetectorClusterState { + processId: string; // Current process identifier + isMaster: boolean; // Whether this process is the master + slaveId: string; // This process's slave identifier + totalWorkers: number; // Number of Python ML workers + totalSlaves: number; // Number of backend slave processes + workers: WorkerState[]; // Detailed worker health and status + slaves: SlaveInfo[]; // Slave process information + assignments: Record; // workerUrl -> slaveId mapping +} +``` + +##### Session Management (Future Implementation) + +```typescript +/** + * Associate a session ID with a camera subscription for tracking + * @param subscriptionIdentifier - Target subscription + * @param sessionId - Session ID to associate (null to clear) + * @returns Promise - Success status + */ +public async setSessionId(subscriptionIdentifier: string, sessionId: number | null): Promise + +/** + * Get current camera image via worker REST API + * @param cameraIdentifier - Camera to capture from + * @returns Promise - JPEG image data + */ +public async getCameraImage(cameraIdentifier: string): Promise +``` + +**Note:** These methods are currently not fully implemented in master-slave mode. + +### Event System Interface + +The cluster emits events that external services can listen to for system monitoring and integration: + +#### Emitted Events + +```typescript +// Detection result processed +detectorCluster.on('worker:detection_result', (event: { + url: string; // Worker URL (always 'cluster-managed') + cameraId: string; // Subscription identifier + detections: number; // Number of objects detected (0 or 1) +}) => void); + +// Worker status changes +detectorCluster.on('worker:online', (event: { url: string }) => void); +detectorCluster.on('worker:offline', (event: { url: string }) => void); + +// Connection events +detectorCluster.on('worker:connecting', (event: { url: string }) => void); +detectorCluster.on('worker:disconnected', (event: { url: string, reason: string }) => void); +detectorCluster.on('worker:websocket_error', (event: { url: string, error: string }) => void); +``` + +### Usage Examples + +#### Basic Camera Subscription (CameraService Integration) + +```typescript +import { detectorCluster } from '~/modules/camera/services/CameraService'; + +// Subscribe to camera with AI detection +const success = await detectorCluster.subscribeToCamera( + `display-123;camera-456`, // subscriptionIdentifier + 'rtsp://192.168.1.100:554/stream1', // rtspUrl + 'https://s3.bucket.com/model.onnx', // modelUrl (pre-signed) + 42, // modelId + 'vehicle-detection-v2', // modelName + (detectionData) => { // callback + console.log('Detection:', detectionData.data.detection); + // Process car model, license plate, etc. + }, + 'http://192.168.1.100/snapshot.jpg', // snapshotUrl (optional) + 5000, // snapshotInterval (optional) + 100, 50, 800, 600 // crop coordinates (optional) +); +``` + +#### Event Monitoring Integration + +```typescript +// Monitor worker health +detectorCluster.on('worker:online', (event) => { + console.log(`Worker ${event.url} came online`); + // Update dashboard, send notifications, etc. +}); + +detectorCluster.on('worker:offline', (event) => { + console.log(`Worker ${event.url} went offline`); + // Alert administrators, trigger failover procedures +}); + +// Monitor detection activity +detectorCluster.on('worker:detection_result', (event) => { + if (event.detections > 0) { + console.log(`Camera ${event.cameraId} detected objects`); + // Trigger content changes, log analytics, etc. + } +}); +``` + +#### Cluster State Monitoring + +```typescript +// Get comprehensive cluster status +const state = await detectorCluster.getState(); + +console.log(`Process ${state.processId} is ${state.isMaster ? 'MASTER' : 'SLAVE'}`); +console.log(`Cluster: ${state.totalWorkers} workers, ${state.totalSlaves} slaves`); + +// Monitor worker health +state.workers.forEach(worker => { + console.log(`Worker ${worker.url}: ${worker.online ? 'ONLINE' : 'OFFLINE'}`); + console.log(` CPU: ${worker.cpuUsage}%, Memory: ${worker.memoryUsage}%`); + console.log(` Subscriptions: ${worker.subscriptionCount}`); +}); + +// Check assignments +Object.entries(state.assignments).forEach(([workerUrl, slaveId]) => { + console.log(`Worker ${workerUrl} assigned to slave ${slaveId}`); +}); +``` + +#### Bulk Camera Management + +```typescript +// Remove all subscriptions for a camera being deleted +await detectorCluster.unsubscribeFromAllWithCameraID('camera-456'); + +// Re-subscribe camera to all displays after configuration change +const displays = await getDisplaysForCamera('camera-456'); +for (const display of displays) { + await detectorCluster.subscribeToCamera( + `${display.id};camera-456`, + camera.rtspUrl, + freshModelUrl, + modelId, + modelName, + createDetectionHandler(display.id, camera.id), + camera.snapshotUrl, + camera.snapshotInterval, + display.cropX1, display.cropY1, + display.cropX2, display.cropY2 + ); +} +``` + +### Error Handling Interface + +The cluster interface follows consistent error handling patterns: + +#### Exception Types + +```typescript +// Subscription errors +try { + await detectorCluster.subscribeToCamera(...); +} catch (error) { + // Possible errors: + // - "No workers available for assignment" + // - "Invalid subscription identifier format" + // - "Model URL expired or inaccessible" + // - Redis connection errors +} + +// State retrieval errors +try { + const state = await detectorCluster.getState(); +} catch (error) { + // Returns safe default state on errors + // Logs detailed error information +} +``` + +#### Graceful Degradation + +- **No Workers Available**: Subscriptions stored in Redis, will activate when workers come online +- **Master Process Failure**: New master elected, all subscriptions restored from Redis +- **Redis Connection Issues**: Local callbacks continue working, subscriptions restored when connection recovers +- **Invalid Parameters**: Clear error messages with parameter validation + +### Integration Patterns + +#### Service Layer Integration + +```typescript +// CameraService.ts example +export class CameraService { + constructor() { + // Initialize cluster connection + detectorCluster.initialize(); + + // Set up global detection processing + detectorCluster.addGlobalDetectionListener(this.processDetection.bind(this)); + } + + async subscribeCamera(displayId: string, camera: CameraEntity) { + const subscriptionId = `${displayId};${camera.cameraIdentifier}`; + + return await detectorCluster.subscribeToCamera( + subscriptionId, + camera.rtspUrl, + await this.getModelUrl(camera.modelId), + camera.modelId, + camera.modelName, + (data) => this.handleDetection(displayId, camera.id, data), + camera.snapshotUrl, + camera.snapshotInterval, + camera.cropX1, camera.cropY1, + camera.cropX2, camera.cropY2 + ); + } + + private processDetection(data: ImageDetectionResponse) { + // Global detection processing logic + this.updateAnalytics(data); + this.triggerDecisionTrees(data); + } +} +``` + +### Interface Guarantees and Contracts + +#### Reliability Guarantees + +- **At-Least-Once Detection Delivery**: Detection callbacks will be called at least once per detection +- **Subscription Persistence**: Subscriptions survive process restarts and master failovers +- **Automatic Reconnection**: Workers automatically reconnect with exponential backoff +- **Load Balancing**: New subscriptions automatically assigned to least loaded workers + +#### Performance Characteristics + +- **Subscription Latency**: < 100ms for new camera subscriptions +- **Detection Latency**: < 50ms from worker to callback (excluding AI processing time) +- **State Query Performance**: < 10ms for cluster state retrieval +- **Memory Usage**: O(n) where n = number of active subscriptions + +#### Thread Safety + +- **Callback Execution**: All callbacks executed on main event loop (Node.js single-threaded) +- **Concurrent Subscriptions**: Multiple simultaneous subscriptions handled safely +- **State Consistency**: Redis operations use atomic transactions where needed + +This interface specification provides external services with a clear understanding of how to integrate with the distributed worker cluster while maintaining abstraction from the underlying complexity. + +## Architecture Evolution: From Complex to Pure Declarative + +### Previous Architecture Limitations (Addressed) +- **Complex State Synchronization**: Incremental updates between database, Redis desired state, and worker actual state created synchronization complexity +- **Command Protocol Complexity**: Multiple command types (`subscribe_camera`, `unsubscribe_camera`) with complex payloads and error handling +- **State Divergence**: Database and Redis desired state could diverge, causing inconsistent behavior +- **Partial Update Complexity**: Complex logic for handling individual subscription changes led to edge cases and race conditions +- **Service Layer Complexity**: Camera/Display services contained complex subscription management logic + +### Current Pure Declarative Architecture Benefits +- **Single Source of Truth**: Database is the only source for desired state - no secondary state stores to synchronize +- **Zero State Divergence**: Desired state is always freshly derived from database queries, eliminating synchronization complexity +- **Simplified Protocol**: Only one command type (`regenerate_subscriptions`) with minimal payload +- **Consistent State Management**: Complete regeneration eliminates all edge cases and partial update complexity +- **Service Layer Simplicity**: Services just update database + trigger regeneration - no subscription logic +- **Operational Resilience**: System is self-healing and predictable - any database change triggers complete reconciliation + +### Declarative Architecture Benefits +- **Global Optimization**: Every regeneration considers all subscriptions globally for optimal load balancing +- **Automatic Recovery**: System automatically heals from any inconsistent state by regenerating from database +- **Resource Efficiency**: Workers assigned based on real-time CPU/memory metrics with load balancing +- **Fault Tolerance**: Complete state recovery from database after any failure (process crashes, network interruptions, etc.) + +### Performance Characteristics +- **Regeneration Speed**: Database queries are fast (~10ms) even with hundreds of displays +- **Reconciliation Efficiency**: Only changed subscriptions are actually modified on workers +- **Memory Efficiency**: No persistent state storage outside of database and current worker assignments +- **Network Efficiency**: Minimal command protocol reduces Redis pub/sub overhead + +This pure declarative architecture provides the reliability and simplicity of container orchestration-style declarative resource management while maintaining the performance and scalability needed for real-time camera processing systems. \ No newline at end of file diff --git a/requirements.base.txt b/requirements.base.txt index b8af923..e7a302f 100644 --- a/requirements.base.txt +++ b/requirements.base.txt @@ -1,12 +1,12 @@ -torch -torchvision -ultralytics -opencv-python -scipy -filterpy -psycopg2-binary -lap>=0.5.12 -pynvml -PyTurboJPEG -PyNvVideoCodec -cupy-cuda12x \ No newline at end of file +ultralytics>=8.3.0 +opencv-python>=4.6.0 +scipy>=1.9.0 +filterpy>=1.4.0 +psycopg2-binary>=2.9.0 +easydict +loguru +pyzmq +gitpython +gdown +lap +pynvml \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 2afeb0e..baddeb5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,5 @@ -fastapi +fastapi[standard] uvicorn websockets -fastapi[standard] redis -urllib3<2.0.0 -numpy -requests -watchdog \ No newline at end of file +urllib3<2.0.0 \ No newline at end of file diff --git a/archive/siwatsystem/database.py b/siwatsystem/database.py similarity index 87% rename from archive/siwatsystem/database.py rename to siwatsystem/database.py index 6340986..5bcbf1d 100644 --- a/archive/siwatsystem/database.py +++ b/siwatsystem/database.py @@ -80,37 +80,50 @@ class DatabaseManager: try: cur = self.connection.cursor() - # Build the UPDATE query dynamically + # Build the INSERT and UPDATE query dynamically + insert_placeholders = [] + insert_values = [key_value] # Start with key_value + set_clauses = [] - values = [] + update_values = [] for field, value in fields.items(): if value == "NOW()": + # Special handling for NOW() + insert_placeholders.append("NOW()") set_clauses.append(f"{field} = NOW()") else: + insert_placeholders.append("%s") + insert_values.append(value) set_clauses.append(f"{field} = %s") - values.append(value) + update_values.append(value) # Add schema prefix if table doesn't already have it full_table_name = table if '.' in table else f"gas_station_1.{table}" + # Build the complete query query = f""" INSERT INTO {full_table_name} ({key_field}, {', '.join(fields.keys())}) - VALUES (%s, {', '.join(['%s'] * len(fields))}) + VALUES (%s, {', '.join(insert_placeholders)}) ON CONFLICT ({key_field}) DO UPDATE SET {', '.join(set_clauses)} """ - # Add key_value to the beginning of values list - all_values = [key_value] + list(fields.values()) + values + # Combine values for the query: insert_values + update_values + all_values = insert_values + update_values + + logger.debug(f"SQL Query: {query}") + logger.debug(f"Values: {all_values}") cur.execute(query, all_values) self.connection.commit() cur.close() - logger.info(f"Updated {table} for {key_field}={key_value}") + logger.info(f"✅ Updated {table} for {key_field}={key_value} with fields: {fields}") return True except Exception as e: - logger.error(f"Failed to execute update on {table}: {e}") + logger.error(f"❌ Failed to execute update on {table}: {e}") + logger.debug(f"Query: {query if 'query' in locals() else 'Query not built'}") + logger.debug(f"Values: {all_values if 'all_values' in locals() else 'Values not prepared'}") if self.connection: self.connection.rollback() return False diff --git a/siwatsystem/model_registry.py b/siwatsystem/model_registry.py new file mode 100644 index 0000000..95daf3b --- /dev/null +++ b/siwatsystem/model_registry.py @@ -0,0 +1,242 @@ +""" +Shared Model Registry for Memory Optimization + +This module implements a global shared model registry to prevent duplicate model loading +in memory when multiple cameras use the same model. This significantly reduces RAM and +GPU VRAM usage by ensuring only one instance of each unique model is loaded. + +Key Features: +- Thread-safe model loading and access +- Reference counting for proper cleanup +- Automatic model lifecycle management +- Maintains compatibility with existing pipeline system +""" + +import os +import threading +import logging +from typing import Dict, Any, Optional, Set +import torch +from ultralytics import YOLO + +# Create a logger for this module +logger = logging.getLogger("detector_worker.model_registry") + +class ModelRegistry: + """ + Singleton class for managing shared YOLO models across multiple cameras. + + This registry ensures that each unique model is loaded only once in memory, + dramatically reducing RAM and GPU VRAM usage when multiple cameras use the + same model. + """ + + _instance = None + _lock = threading.Lock() + + def __new__(cls): + if cls._instance is None: + with cls._lock: + if cls._instance is None: + cls._instance = super(ModelRegistry, cls).__new__(cls) + cls._instance._initialized = False + return cls._instance + + def __init__(self): + if self._initialized: + return + + self._initialized = True + + # Thread-safe storage for loaded models + self._models: Dict[str, YOLO] = {} # modelId -> YOLO model instance + self._model_files: Dict[str, str] = {} # modelId -> file path + self._reference_counts: Dict[str, int] = {} # modelId -> reference count + self._model_lock = threading.RLock() # Reentrant lock for nested calls + + logger.info("🏭 Shared Model Registry initialized - ready for memory-optimized model loading") + + def get_model(self, model_id: str, model_file_path: str) -> YOLO: + """ + Get or load a YOLO model. Returns shared instance if already loaded. + + Args: + model_id: Unique identifier for the model + model_file_path: Path to the model file + + Returns: + YOLO model instance (shared across all callers) + """ + with self._model_lock: + if model_id in self._models: + # Model already loaded - increment reference count and return + self._reference_counts[model_id] += 1 + logger.info(f"📖 Model '{model_id}' reused (ref_count: {self._reference_counts[model_id]}) - SAVED MEMORY!") + return self._models[model_id] + + # Model not loaded yet - load it + logger.info(f"🔄 Loading NEW model '{model_id}' from {model_file_path}") + + if not os.path.exists(model_file_path): + raise FileNotFoundError(f"Model file {model_file_path} not found") + + try: + # Load the YOLO model + model = YOLO(model_file_path) + + # Move to GPU if available + if torch.cuda.is_available(): + logger.info(f"🚀 CUDA available. Moving model '{model_id}' to GPU VRAM") + model.to("cuda") + else: + logger.info(f"💻 CUDA not available. Using CPU for model '{model_id}'") + + # Store in registry + self._models[model_id] = model + self._model_files[model_id] = model_file_path + self._reference_counts[model_id] = 1 + + logger.info(f"✅ Model '{model_id}' loaded and registered (ref_count: 1)") + self._log_registry_status() + + return model + + except Exception as e: + logger.error(f"❌ Failed to load model '{model_id}' from {model_file_path}: {e}") + raise + + def release_model(self, model_id: str) -> None: + """ + Release a reference to a model. If reference count reaches zero, + the model may be unloaded to free memory. + + Args: + model_id: Unique identifier for the model to release + """ + with self._model_lock: + if model_id not in self._reference_counts: + logger.warning(f"⚠️ Attempted to release unknown model '{model_id}'") + return + + self._reference_counts[model_id] -= 1 + logger.info(f"📉 Model '{model_id}' reference count decreased to {self._reference_counts[model_id]}") + + # For now, keep models in memory even when ref count reaches 0 + # This prevents reload overhead if the same model is needed again soon + # In the future, we could implement LRU eviction policy + # if self._reference_counts[model_id] <= 0: + # logger.info(f"💤 Model '{model_id}' has 0 references but keeping in memory for reuse") + # Optionally: self._unload_model(model_id) + + def _unload_model(self, model_id: str) -> None: + """ + Internal method to unload a model from memory. + Currently not used to prevent reload overhead. + """ + with self._model_lock: + if model_id in self._models: + logger.info(f"🗑️ Unloading model '{model_id}' from memory") + + # Clear GPU memory if model was on GPU + model = self._models[model_id] + if hasattr(model, 'model') and hasattr(model.model, 'cuda'): + try: + # Move model to CPU before deletion to free GPU memory + model.to('cpu') + except Exception as e: + logger.warning(f"⚠️ Failed to move model '{model_id}' to CPU: {e}") + + # Remove from registry + del self._models[model_id] + del self._model_files[model_id] + del self._reference_counts[model_id] + + # Force garbage collection + import gc + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + logger.info(f"✅ Model '{model_id}' unloaded and memory freed") + self._log_registry_status() + + def get_registry_status(self) -> Dict[str, Any]: + """ + Get current status of the model registry. + + Returns: + Dictionary with registry statistics + """ + with self._model_lock: + return { + "total_models": len(self._models), + "models": { + model_id: { + "file_path": self._model_files[model_id], + "reference_count": self._reference_counts[model_id] + } + for model_id in self._models + }, + "total_references": sum(self._reference_counts.values()) + } + + def _log_registry_status(self) -> None: + """Log current registry status for debugging.""" + status = self.get_registry_status() + logger.info(f"📊 Model Registry Status: {status['total_models']} unique models, {status['total_references']} total references") + for model_id, info in status['models'].items(): + logger.debug(f" 📋 '{model_id}': refs={info['reference_count']}, file={os.path.basename(info['file_path'])}") + + def cleanup_all(self) -> None: + """ + Clean up all models from the registry. Used during shutdown. + """ + with self._model_lock: + model_ids = list(self._models.keys()) + logger.info(f"🧹 Cleaning up {len(model_ids)} models from registry") + + for model_id in model_ids: + self._unload_model(model_id) + + logger.info("✅ Model registry cleanup complete") + + +# Global singleton instance +_registry = ModelRegistry() + +def get_shared_model(model_id: str, model_file_path: str) -> YOLO: + """ + Convenience function to get a shared model instance. + + Args: + model_id: Unique identifier for the model + model_file_path: Path to the model file + + Returns: + YOLO model instance (shared across all callers) + """ + return _registry.get_model(model_id, model_file_path) + +def release_shared_model(model_id: str) -> None: + """ + Convenience function to release a shared model reference. + + Args: + model_id: Unique identifier for the model to release + """ + _registry.release_model(model_id) + +def get_registry_status() -> Dict[str, Any]: + """ + Convenience function to get registry status. + + Returns: + Dictionary with registry statistics + """ + return _registry.get_registry_status() + +def cleanup_registry() -> None: + """ + Convenience function to cleanup the entire registry. + """ + _registry.cleanup_all() \ No newline at end of file diff --git a/siwatsystem/mpta_manager.py b/siwatsystem/mpta_manager.py new file mode 100644 index 0000000..1abda3f --- /dev/null +++ b/siwatsystem/mpta_manager.py @@ -0,0 +1,375 @@ +""" +Shared MPTA Manager for Disk Space Optimization + +This module implements shared MPTA file management to prevent duplicate downloads +and extractions when multiple cameras use the same model. MPTA files are stored +in modelId-based directories and shared across all cameras using that model. + +Key Features: +- Thread-safe MPTA downloading and extraction +- ModelId-based directory structure: models/{modelId}/ +- Reference counting for proper cleanup +- Eliminates duplicate MPTA downloads +- Maintains compatibility with existing pipeline system +""" + +import os +import threading +import logging +import shutil +import requests +from typing import Dict, Set, Optional +from urllib.parse import urlparse +from .pympta import load_pipeline_from_zip + +# Create a logger for this module +logger = logging.getLogger("detector_worker.mpta_manager") + +class MPTAManager: + """ + Singleton class for managing shared MPTA files across multiple cameras. + + This manager ensures that each unique modelId is downloaded and extracted + only once, dramatically reducing disk usage and download time when multiple + cameras use the same model. + """ + + _instance = None + _lock = threading.Lock() + + def __new__(cls): + if cls._instance is None: + with cls._lock: + if cls._instance is None: + cls._instance = super(MPTAManager, cls).__new__(cls) + cls._instance._initialized = False + return cls._instance + + def __init__(self): + if self._initialized: + return + + self._initialized = True + + # Thread-safe storage for MPTA management + self._model_paths: Dict[int, str] = {} # modelId -> shared_extraction_path + self._mpta_file_paths: Dict[int, str] = {} # modelId -> local_mpta_file_path + self._reference_counts: Dict[int, int] = {} # modelId -> reference count + self._download_locks: Dict[int, threading.Lock] = {} # modelId -> download lock + self._cameras_using_model: Dict[int, Set[str]] = {} # modelId -> set of camera_ids + self._manager_lock = threading.RLock() # Reentrant lock for nested calls + + logger.info("🏭 Shared MPTA Manager initialized - ready for disk-optimized MPTA management") + + def get_or_download_mpta(self, model_id: int, model_url: str, camera_id: str) -> Optional[tuple[str, str]]: + """ + Get or download an MPTA file. Returns (extraction_path, mpta_file_path) if successful. + + Args: + model_id: Unique identifier for the model + model_url: URL to download the MPTA file from + camera_id: Identifier for the requesting camera + + Returns: + Tuple of (extraction_path, mpta_file_path), or None if failed + """ + with self._manager_lock: + # Track camera usage + if model_id not in self._cameras_using_model: + self._cameras_using_model[model_id] = set() + self._cameras_using_model[model_id].add(camera_id) + + # Check if model directory already exists on disk (from previous sessions) + if model_id not in self._model_paths: + potential_path = f"models/{model_id}" + if os.path.exists(potential_path) and os.path.isdir(potential_path): + # Directory exists from previous session, find the MPTA file + mpta_files = [f for f in os.listdir(potential_path) if f.endswith('.mpta')] + if mpta_files: + # Use the first .mpta file found + mpta_file_path = os.path.join(potential_path, mpta_files[0]) + self._model_paths[model_id] = potential_path + self._mpta_file_paths[model_id] = mpta_file_path + self._reference_counts[model_id] = 0 # Will be incremented below + logger.info(f"📂 Found existing MPTA modelId {model_id} from previous session") + + # Check if already available + if model_id in self._model_paths: + shared_path = self._model_paths[model_id] + mpta_file_path = self._mpta_file_paths.get(model_id) + if os.path.exists(shared_path) and mpta_file_path and os.path.exists(mpta_file_path): + self._reference_counts[model_id] += 1 + logger.info(f"📂 MPTA modelId {model_id} reused for camera {camera_id} (ref_count: {self._reference_counts[model_id]}) - SAVED DOWNLOAD!") + return (shared_path, mpta_file_path) + else: + # Path was deleted externally, clean up our records + logger.warning(f"⚠️ MPTA path for modelId {model_id} was deleted externally, will re-download") + del self._model_paths[model_id] + self._mpta_file_paths.pop(model_id, None) + self._reference_counts.pop(model_id, 0) + + # Need to download - get or create download lock for this modelId + if model_id not in self._download_locks: + self._download_locks[model_id] = threading.Lock() + + # Download with model-specific lock (released _manager_lock to allow other models) + download_lock = self._download_locks[model_id] + with download_lock: + # Double-check after acquiring download lock + with self._manager_lock: + if model_id in self._model_paths and os.path.exists(self._model_paths[model_id]): + mpta_file_path = self._mpta_file_paths.get(model_id) + if mpta_file_path and os.path.exists(mpta_file_path): + self._reference_counts[model_id] += 1 + logger.info(f"📂 MPTA modelId {model_id} became available during wait (ref_count: {self._reference_counts[model_id]})") + return (self._model_paths[model_id], mpta_file_path) + + # Actually download and extract + shared_path = f"models/{model_id}" + logger.info(f"🔄 Downloading NEW MPTA for modelId {model_id} from {model_url}") + + try: + # Ensure directory exists + os.makedirs(shared_path, exist_ok=True) + + # Download MPTA file + mpta_filename = self._extract_filename_from_url(model_url) or f"model_{model_id}.mpta" + local_mpta_path = os.path.join(shared_path, mpta_filename) + + if not self._download_file(model_url, local_mpta_path): + logger.error(f"❌ Failed to download MPTA for modelId {model_id}") + return None + + # Extract MPTA + pipeline_tree = load_pipeline_from_zip(local_mpta_path, shared_path) + if pipeline_tree is None: + logger.error(f"❌ Failed to extract MPTA for modelId {model_id}") + return None + + # Success - register in manager + with self._manager_lock: + self._model_paths[model_id] = shared_path + self._mpta_file_paths[model_id] = local_mpta_path + self._reference_counts[model_id] = 1 + + logger.info(f"✅ MPTA modelId {model_id} downloaded and registered (ref_count: 1)") + self._log_manager_status() + + return (shared_path, local_mpta_path) + + except Exception as e: + logger.error(f"❌ Error downloading/extracting MPTA for modelId {model_id}: {e}") + # Clean up partial download + if os.path.exists(shared_path): + shutil.rmtree(shared_path, ignore_errors=True) + return None + + def release_mpta(self, model_id: int, camera_id: str) -> None: + """ + Release a reference to an MPTA. If reference count reaches zero, + the MPTA directory may be cleaned up to free disk space. + + Args: + model_id: Unique identifier for the model to release + camera_id: Identifier for the camera releasing the reference + """ + with self._manager_lock: + if model_id not in self._reference_counts: + logger.warning(f"⚠️ Attempted to release unknown MPTA modelId {model_id} for camera {camera_id}") + return + + # Remove camera from usage tracking + if model_id in self._cameras_using_model: + self._cameras_using_model[model_id].discard(camera_id) + + self._reference_counts[model_id] -= 1 + logger.info(f"📉 MPTA modelId {model_id} reference count decreased to {self._reference_counts[model_id]} (released by {camera_id})") + + # Clean up if no more references + # if self._reference_counts[model_id] <= 0: + # self._cleanup_mpta(model_id) + + def _cleanup_mpta(self, model_id: int) -> None: + """ + Internal method to clean up an MPTA directory and free disk space. + """ + if model_id in self._model_paths: + shared_path = self._model_paths[model_id] + + try: + if os.path.exists(shared_path): + shutil.rmtree(shared_path) + logger.info(f"🗑️ Cleaned up MPTA directory: {shared_path}") + + # Remove from tracking + del self._model_paths[model_id] + self._mpta_file_paths.pop(model_id, None) + del self._reference_counts[model_id] + self._cameras_using_model.pop(model_id, None) + + # Clean up download lock (optional, could keep for future use) + self._download_locks.pop(model_id, None) + + logger.info(f"✅ MPTA modelId {model_id} fully cleaned up and disk space freed") + self._log_manager_status() + + except Exception as e: + logger.error(f"❌ Error cleaning up MPTA modelId {model_id}: {e}") + + def get_shared_path(self, model_id: int) -> Optional[str]: + """ + Get the shared extraction path for a modelId without downloading. + + Args: + model_id: Model identifier to look up + + Returns: + Shared path if available, None otherwise + """ + with self._manager_lock: + return self._model_paths.get(model_id) + + def get_manager_status(self) -> Dict: + """ + Get current status of the MPTA manager. + + Returns: + Dictionary with manager statistics + """ + with self._manager_lock: + return { + "total_mpta_models": len(self._model_paths), + "models": { + str(model_id): { + "shared_path": path, + "reference_count": self._reference_counts.get(model_id, 0), + "cameras_using": list(self._cameras_using_model.get(model_id, set())) + } + for model_id, path in self._model_paths.items() + }, + "total_references": sum(self._reference_counts.values()), + "active_downloads": len(self._download_locks) + } + + def _log_manager_status(self) -> None: + """Log current manager status for debugging.""" + status = self.get_manager_status() + logger.info(f"📊 MPTA Manager Status: {status['total_mpta_models']} unique models, {status['total_references']} total references") + for model_id, info in status['models'].items(): + cameras_str = ','.join(info['cameras_using'][:3]) # Show first 3 cameras + if len(info['cameras_using']) > 3: + cameras_str += f"+{len(info['cameras_using'])-3} more" + logger.debug(f" 📋 ModelId {model_id}: refs={info['reference_count']}, cameras=[{cameras_str}]") + + def cleanup_all(self) -> None: + """ + Clean up all MPTA directories. Used during shutdown. + """ + with self._manager_lock: + model_ids = list(self._model_paths.keys()) + logger.info(f"🧹 Cleaning up {len(model_ids)} MPTA directories") + + for model_id in model_ids: + self._cleanup_mpta(model_id) + + # Clear all tracking data + self._download_locks.clear() + logger.info("✅ MPTA manager cleanup complete") + + def _download_file(self, url: str, local_path: str) -> bool: + """ + Download a file from URL to local path with progress logging. + + Args: + url: URL to download from + local_path: Local path to save to + + Returns: + True if successful, False otherwise + """ + try: + logger.info(f"⬇️ Starting download from {url}") + + response = requests.get(url, stream=True) + response.raise_for_status() + + total_size = int(response.headers.get('content-length', 0)) + if total_size > 0: + logger.info(f"📦 File size: {total_size / 1024 / 1024:.2f} MB") + + downloaded = 0 + last_logged_progress = 0 + with open(local_path, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + downloaded += len(chunk) + + if total_size > 0: + progress = int((downloaded / total_size) * 100) + # Log at 10% intervals (10%, 20%, 30%, etc.) + if progress >= last_logged_progress + 10 and progress <= 100: + logger.debug(f"Download progress: {progress}%") + last_logged_progress = progress + + logger.info(f"✅ Successfully downloaded to {local_path}") + return True + + except Exception as e: + logger.error(f"❌ Download failed: {e}") + # Clean up partial file + if os.path.exists(local_path): + os.remove(local_path) + return False + + def _extract_filename_from_url(self, url: str) -> Optional[str]: + """Extract filename from URL.""" + try: + parsed = urlparse(url) + filename = os.path.basename(parsed.path) + return filename if filename else None + except Exception: + return None + + +# Global singleton instance +_mpta_manager = MPTAManager() + +def get_or_download_mpta(model_id: int, model_url: str, camera_id: str) -> Optional[tuple[str, str]]: + """ + Convenience function to get or download a shared MPTA. + + Args: + model_id: Unique identifier for the model + model_url: URL to download the MPTA file from + camera_id: Identifier for the requesting camera + + Returns: + Tuple of (extraction_path, mpta_file_path), or None if failed + """ + return _mpta_manager.get_or_download_mpta(model_id, model_url, camera_id) + +def release_mpta(model_id: int, camera_id: str) -> None: + """ + Convenience function to release a shared MPTA reference. + + Args: + model_id: Unique identifier for the model to release + camera_id: Identifier for the camera releasing the reference + """ + _mpta_manager.release_mpta(model_id, camera_id) + +def get_mpta_manager_status() -> Dict: + """ + Convenience function to get MPTA manager status. + + Returns: + Dictionary with manager statistics + """ + return _mpta_manager.get_manager_status() + +def cleanup_mpta_manager() -> None: + """ + Convenience function to cleanup the entire MPTA manager. + """ + _mpta_manager.cleanup_all() \ No newline at end of file diff --git a/siwatsystem/pympta.py b/siwatsystem/pympta.py new file mode 100644 index 0000000..1d81596 --- /dev/null +++ b/siwatsystem/pympta.py @@ -0,0 +1,1849 @@ +import os +import json +import logging +import torch +import cv2 +import zipfile +import shutil +import traceback +import redis +import time +import uuid +import concurrent.futures +from ultralytics import YOLO +from urllib.parse import urlparse +from .database import DatabaseManager +from .model_registry import get_shared_model, release_shared_model +from datetime import datetime + +# Create a logger specifically for this module +logger = logging.getLogger("detector_worker.pympta") + +# Global camera-aware stability tracking +# Structure: {camera_id: {model_id: {"track_stability_counters": {track_id: count}, "stable_tracks": set(), "session_state": {...}}}} +_camera_stability_tracking = {} + +# Session timeout configuration (waiting for backend sessionId) +_session_timeout_seconds = 15 + +def validate_redis_config(redis_config: dict) -> bool: + """Validate Redis configuration parameters.""" + required_fields = ["host", "port"] + for field in required_fields: + if field not in redis_config: + logger.error(f"Missing required Redis config field: {field}") + return False + + if not isinstance(redis_config["port"], int) or redis_config["port"] <= 0: + logger.error(f"Invalid Redis port: {redis_config['port']}") + return False + + return True + +def validate_postgresql_config(pg_config: dict) -> bool: + """Validate PostgreSQL configuration parameters.""" + required_fields = ["host", "port", "database", "username", "password"] + for field in required_fields: + if field not in pg_config: + logger.error(f"Missing required PostgreSQL config field: {field}") + return False + + if not isinstance(pg_config["port"], int) or pg_config["port"] <= 0: + logger.error(f"Invalid PostgreSQL port: {pg_config['port']}") + return False + + return True + +def crop_region_by_class(frame, regions_dict, class_name): + """Crop a specific region from frame based on detected class.""" + if class_name not in regions_dict: + logger.warning(f"Class '{class_name}' not found in detected regions") + return None + + bbox = regions_dict[class_name]['bbox'] + x1, y1, x2, y2 = bbox + + # Diagnostic logging for crop issues + frame_h, frame_w = frame.shape[:2] + logger.debug(f"CROP DEBUG: Frame dimensions: {frame_w}x{frame_h}") + logger.debug(f"CROP DEBUG: Original bbox: {bbox}") + logger.debug(f"CROP DEBUG: Bbox dimensions: {x2-x1}x{y2-y1}") + + # Check if bbox is within frame bounds + if x1 < 0 or y1 < 0 or x2 > frame_w or y2 > frame_h: + logger.warning(f"CROP DEBUG: Bbox extends beyond frame! Clipping...") + x1, y1 = max(0, x1), max(0, y1) + x2, y2 = min(frame_w, x2), min(frame_h, y2) + logger.debug(f"CROP DEBUG: Clipped bbox: ({x1}, {y1}, {x2}, {y2})") + + cropped = frame[y1:y2, x1:x2] + + if cropped.size == 0: + logger.warning(f"Empty crop for class '{class_name}' with bbox {bbox}") + return None + + logger.debug(f"CROP DEBUG: Successful crop shape: {cropped.shape}") + return cropped + +def format_action_context(base_context, additional_context=None): + """Format action context with dynamic values.""" + context = {**base_context} + if additional_context: + context.update(additional_context) + return context + +def load_pipeline_node(node_config: dict, mpta_dir: str, redis_client, db_manager=None) -> dict: + # Recursively load a model node from configuration. + model_path = os.path.join(mpta_dir, node_config["modelFile"]) + if not os.path.exists(model_path): + logger.error(f"Model file {model_path} not found. Current directory: {os.getcwd()}") + logger.error(f"Directory content: {os.listdir(os.path.dirname(model_path))}") + raise FileNotFoundError(f"Model file {model_path} not found.") + + # Use shared model registry to prevent duplicate loading + model_id = node_config['modelId'] + logger.info(f"Getting shared model for node {model_id} from {model_path}") + model = get_shared_model(model_id, model_path) + + # Prepare trigger class indices for optimization + trigger_classes = node_config.get("triggerClasses", []) + trigger_class_indices = None + if trigger_classes and hasattr(model, "names"): + # Convert class names to indices for the model + trigger_class_indices = [i for i, name in model.names.items() + if name in trigger_classes] + logger.debug(f"Converted trigger classes to indices: {trigger_class_indices}") + + # Extract stability threshold from main pipeline config (not tracking config) + tracking_config = node_config.get("tracking", {"enabled": True, "reidConfigPath": "botsort.yaml"}) + stability_threshold = node_config.get("stabilityThreshold", 4) # Read from main config, default to 4 + + node = { + "modelId": node_config["modelId"], + "modelFile": node_config["modelFile"], + "triggerClasses": trigger_classes, + "triggerClassIndices": trigger_class_indices, + "classMapping": node_config.get("classMapping", {}), + "crop": node_config.get("crop", False), + "cropClass": node_config.get("cropClass"), + "minConfidence": node_config.get("minConfidence", None), + "frontalMinConfidence": node_config.get("frontalMinConfidence", None), + "minBboxAreaRatio": node_config.get("minBboxAreaRatio", 0.0), + "multiClass": node_config.get("multiClass", False), + "expectedClasses": node_config.get("expectedClasses", []), + "parallel": node_config.get("parallel", False), + "actions": node_config.get("actions", []), + "parallelActions": node_config.get("parallelActions", []), + "tracking": tracking_config, + "stabilityThreshold": stability_threshold, + "model": model, + "branches": [], + "redis_client": redis_client, + "db_manager": db_manager + } + logger.debug(f"Configured node {node_config['modelId']} with trigger classes: {node['triggerClasses']}") + for child in node_config.get("branches", []): + logger.debug(f"Loading branch for parent node {node_config['modelId']}") + node["branches"].append(load_pipeline_node(child, mpta_dir, redis_client, db_manager)) + return node + +def load_pipeline_from_zip(zip_source: str, target_dir: str) -> dict: + logger.info(f"Attempting to load pipeline from {zip_source} to {target_dir}") + os.makedirs(target_dir, exist_ok=True) + zip_path = os.path.join(target_dir, "pipeline.mpta") + + # Parse the source; only local files are supported here. + parsed = urlparse(zip_source) + if parsed.scheme in ("", "file"): + local_path = parsed.path if parsed.scheme == "file" else zip_source + logger.debug(f"Checking if local file exists: {local_path}") + if os.path.exists(local_path): + try: + shutil.copy(local_path, zip_path) + logger.info(f"Copied local .mpta file from {local_path} to {zip_path}") + except Exception as e: + logger.error(f"Failed to copy local .mpta file from {local_path}: {str(e)}", exc_info=True) + return None + else: + logger.error(f"Local file {local_path} does not exist. Current directory: {os.getcwd()}") + # List all subdirectories of models directory to help debugging + if os.path.exists("models"): + logger.error(f"Content of models directory: {os.listdir('models')}") + for root, dirs, files in os.walk("models"): + logger.error(f"Directory {root} contains subdirs: {dirs} and files: {files}") + else: + logger.error("The models directory doesn't exist") + return None + else: + logger.error(f"HTTP download functionality has been moved. Use a local file path here. Received: {zip_source}") + return None + + try: + if not os.path.exists(zip_path): + logger.error(f"Zip file not found at expected location: {zip_path}") + return None + + logger.debug(f"Extracting .mpta file from {zip_path} to {target_dir}") + # Extract contents and track the directories created + extracted_dirs = [] + with zipfile.ZipFile(zip_path, "r") as zip_ref: + file_list = zip_ref.namelist() + logger.debug(f"Files in .mpta archive: {file_list}") + + # Extract and track the top-level directories + for file_path in file_list: + parts = file_path.split('/') + if len(parts) > 1: + top_dir = parts[0] + if top_dir and top_dir not in extracted_dirs: + extracted_dirs.append(top_dir) + + # Now extract the files + zip_ref.extractall(target_dir) + + logger.info(f"Successfully extracted .mpta file to {target_dir}") + logger.debug(f"Extracted directories: {extracted_dirs}") + + # Check what was actually created after extraction + actual_dirs = [d for d in os.listdir(target_dir) if os.path.isdir(os.path.join(target_dir, d))] + logger.debug(f"Actual directories created: {actual_dirs}") + except zipfile.BadZipFile as e: + logger.error(f"Bad zip file {zip_path}: {str(e)}", exc_info=True) + return None + except Exception as e: + logger.error(f"Failed to extract .mpta file {zip_path}: {str(e)}", exc_info=True) + return None + finally: + if os.path.exists(zip_path): + os.remove(zip_path) + logger.debug(f"Removed temporary zip file: {zip_path}") + + # Use the first extracted directory if it exists, otherwise use the expected name + pipeline_name = os.path.basename(zip_source) + pipeline_name = os.path.splitext(pipeline_name)[0] + + # Find the directory with pipeline.json + mpta_dir = None + # First try the expected directory name + expected_dir = os.path.join(target_dir, pipeline_name) + if os.path.exists(expected_dir) and os.path.exists(os.path.join(expected_dir, "pipeline.json")): + mpta_dir = expected_dir + logger.debug(f"Found pipeline.json in the expected directory: {mpta_dir}") + else: + # Look through all subdirectories for pipeline.json + for subdir in actual_dirs: + potential_dir = os.path.join(target_dir, subdir) + if os.path.exists(os.path.join(potential_dir, "pipeline.json")): + mpta_dir = potential_dir + logger.info(f"Found pipeline.json in directory: {mpta_dir} (different from expected: {expected_dir})") + break + + if not mpta_dir: + logger.error(f"Could not find pipeline.json in any extracted directory. Directory content: {os.listdir(target_dir)}") + return None + + pipeline_json_path = os.path.join(mpta_dir, "pipeline.json") + if not os.path.exists(pipeline_json_path): + logger.error(f"pipeline.json not found in the .mpta file. Files in directory: {os.listdir(mpta_dir)}") + return None + + try: + with open(pipeline_json_path, "r") as f: + pipeline_config = json.load(f) + logger.info(f"Successfully loaded pipeline configuration from {pipeline_json_path}") + logger.debug(f"Pipeline config: {json.dumps(pipeline_config, indent=2)}") + + # Establish Redis connection if configured + redis_client = None + if "redis" in pipeline_config: + redis_config = pipeline_config["redis"] + if not validate_redis_config(redis_config): + logger.error("Invalid Redis configuration, skipping Redis connection") + else: + try: + redis_client = redis.Redis( + host=redis_config["host"], + port=redis_config["port"], + password=redis_config.get("password"), + db=redis_config.get("db", 0), + decode_responses=True + ) + redis_client.ping() + logger.info(f"Successfully connected to Redis at {redis_config['host']}:{redis_config['port']}") + except redis.exceptions.ConnectionError as e: + logger.error(f"Failed to connect to Redis: {e}") + redis_client = None + + # Establish PostgreSQL connection if configured + db_manager = None + if "postgresql" in pipeline_config: + pg_config = pipeline_config["postgresql"] + if not validate_postgresql_config(pg_config): + logger.error("Invalid PostgreSQL configuration, skipping database connection") + else: + try: + db_manager = DatabaseManager(pg_config) + if db_manager.connect(): + logger.info(f"Successfully connected to PostgreSQL at {pg_config['host']}:{pg_config['port']}") + else: + logger.error("Failed to connect to PostgreSQL") + db_manager = None + except Exception as e: + logger.error(f"Error initializing PostgreSQL connection: {e}") + db_manager = None + + return load_pipeline_node(pipeline_config["pipeline"], mpta_dir, redis_client, db_manager) + except json.JSONDecodeError as e: + logger.error(f"Error parsing pipeline.json: {str(e)}", exc_info=True) + return None + except KeyError as e: + logger.error(f"Missing key in pipeline.json: {str(e)}", exc_info=True) + return None + except Exception as e: + logger.error(f"Error loading pipeline.json: {str(e)}", exc_info=True) + return None + +def execute_actions(node, frame, detection_result, regions_dict=None): + if not node["redis_client"] or not node["actions"]: + return + + # Create a dynamic context for this detection event + from datetime import datetime + action_context = { + **detection_result, + "timestamp_ms": int(time.time() * 1000), + "uuid": str(uuid.uuid4()), + "timestamp": datetime.now().strftime("%Y-%m-%dT%H-%M-%S"), + "filename": f"{uuid.uuid4()}.jpg" + } + + for action in node["actions"]: + try: + if action["type"] == "redis_save_image": + key = action["key"].format(**action_context) + + # Check if we need to crop a specific region + region_name = action.get("region") + image_to_save = frame + + if region_name and regions_dict: + cropped_image = crop_region_by_class(frame, regions_dict, region_name) + if cropped_image is not None: + image_to_save = cropped_image + logger.debug(f"Cropped region '{region_name}' for redis_save_image") + else: + logger.warning(f"Could not crop region '{region_name}', saving full frame instead") + + # Encode image with specified format and quality (default to JPEG) + img_format = action.get("format", "jpeg").lower() + quality = action.get("quality", 90) + + if img_format == "jpeg": + encode_params = [cv2.IMWRITE_JPEG_QUALITY, quality] + success, buffer = cv2.imencode('.jpg', image_to_save, encode_params) + elif img_format == "png": + success, buffer = cv2.imencode('.png', image_to_save) + else: + success, buffer = cv2.imencode('.jpg', image_to_save, [cv2.IMWRITE_JPEG_QUALITY, quality]) + + if not success: + logger.error(f"Failed to encode image for redis_save_image") + continue + + expire_seconds = action.get("expire_seconds") + if expire_seconds: + node["redis_client"].setex(key, expire_seconds, buffer.tobytes()) + logger.info(f"Saved image to Redis with key: {key} (expires in {expire_seconds}s)") + else: + node["redis_client"].set(key, buffer.tobytes()) + logger.info(f"Saved image to Redis with key: {key}") + action_context["image_key"] = key + elif action["type"] == "redis_publish": + channel = action["channel"] + try: + # Handle JSON message format by creating it programmatically + message_template = action["message"] + + # Check if the message is JSON-like (starts and ends with braces) + if message_template.strip().startswith('{') and message_template.strip().endswith('}'): + # Create JSON data programmatically to avoid formatting issues + json_data = {} + + # Add common fields + json_data["event"] = "frontal_detected" + json_data["display_id"] = action_context.get("display_id", "unknown") + json_data["session_id"] = action_context.get("session_id") + json_data["timestamp"] = action_context.get("timestamp", "") + json_data["image_key"] = action_context.get("image_key", "") + + # Convert to JSON string + message = json.dumps(json_data) + else: + # Use regular string formatting for non-JSON messages + message = message_template.format(**action_context) + + # Publish to Redis + if not node["redis_client"]: + logger.error("Redis client is None, cannot publish message") + continue + + # Test Redis connection + try: + node["redis_client"].ping() + logger.debug("Redis connection is active") + except Exception as ping_error: + logger.error(f"Redis connection test failed: {ping_error}") + continue + + result = node["redis_client"].publish(channel, message) + logger.info(f"Published message to Redis channel '{channel}': {message}") + logger.info(f"Redis publish result (subscribers count): {result}") + + # Additional debug info + if result == 0: + logger.warning(f"No subscribers listening to channel '{channel}'") + else: + logger.info(f"Message delivered to {result} subscriber(s)") + + except KeyError as e: + logger.error(f"Missing key in redis_publish message template: {e}") + logger.debug(f"Available context keys: {list(action_context.keys())}") + except Exception as e: + logger.error(f"Error in redis_publish action: {e}") + logger.debug(f"Message template: {action['message']}") + logger.debug(f"Available context keys: {list(action_context.keys())}") + import traceback + logger.debug(f"Full traceback: {traceback.format_exc()}") + except Exception as e: + logger.error(f"Error executing action {action['type']}: {e}") + +def execute_parallel_actions(node, frame, detection_result, regions_dict): + """Execute parallel actions after all required branches have completed.""" + if not node.get("parallelActions"): + return + + logger.debug("Executing parallel actions...") + branch_results = detection_result.get("branch_results", {}) + + for action in node["parallelActions"]: + try: + action_type = action.get("type") + logger.debug(f"Processing parallel action: {action_type}") + + if action_type == "postgresql_update_combined": + # Check if all required branches have completed + wait_for_branches = action.get("waitForBranches", []) + missing_branches = [branch for branch in wait_for_branches if branch not in branch_results] + + if missing_branches: + logger.warning(f"Cannot execute postgresql_update_combined: missing branch results for {missing_branches}") + continue + + logger.info(f"All required branches completed: {wait_for_branches}") + + # Execute the database update + execute_postgresql_update_combined(node, action, detection_result, branch_results) + else: + logger.warning(f"Unknown parallel action type: {action_type}") + + except Exception as e: + logger.error(f"Error executing parallel action {action.get('type', 'unknown')}: {e}") + import traceback + logger.debug(f"Full traceback: {traceback.format_exc()}") + +def execute_postgresql_update_combined(node, action, detection_result, branch_results): + """Execute a PostgreSQL update with combined branch results.""" + if not node.get("db_manager"): + logger.error("No database manager available for postgresql_update_combined action") + return + + try: + table = action["table"] + key_field = action["key_field"] + key_value_template = action["key_value"] + fields = action["fields"] + + # Create context for key value formatting + action_context = {**detection_result} + key_value = key_value_template.format(**action_context) + + logger.info(f"Executing database update: table={table}, {key_field}={key_value}") + logger.debug(f"Available branch results: {list(branch_results.keys())}") + + # Process field mappings + mapped_fields = {} + for db_field, value_template in fields.items(): + try: + mapped_value = resolve_field_mapping(value_template, branch_results, action_context) + if mapped_value is not None: + mapped_fields[db_field] = mapped_value + logger.info(f"Mapped field: {db_field} = {mapped_value}") + else: + logger.warning(f"Could not resolve field mapping for {db_field}: {value_template}") + logger.debug(f"Available branch results: {branch_results}") + except Exception as e: + logger.error(f"Error mapping field {db_field} with template '{value_template}': {e}") + import traceback + logger.debug(f"Field mapping error traceback: {traceback.format_exc()}") + + if not mapped_fields: + logger.warning("No fields mapped successfully, skipping database update") + logger.debug(f"Branch results available: {branch_results}") + logger.debug(f"Field templates: {fields}") + return + + # Add updated_at field automatically + mapped_fields["updated_at"] = "NOW()" + + # Execute the database update + logger.info(f"Attempting database update with fields: {mapped_fields}") + success = node["db_manager"].execute_update(table, key_field, key_value, mapped_fields) + + if success: + logger.info(f"✅ Successfully updated database: {table} with {len(mapped_fields)} fields") + logger.info(f"Updated fields: {mapped_fields}") + else: + logger.error(f"❌ Failed to update database: {table}") + logger.error(f"Attempted update with: {key_field}={key_value}, fields={mapped_fields}") + + except KeyError as e: + logger.error(f"Missing required field in postgresql_update_combined action: {e}") + logger.debug(f"Action config: {action}") + except Exception as e: + logger.error(f"Error in postgresql_update_combined action: {e}") + import traceback + logger.debug(f"Full traceback: {traceback.format_exc()}") + +def resolve_field_mapping(value_template, branch_results, action_context): + """Resolve field mapping templates like {car_brand_cls_v1.brand}.""" + try: + logger.debug(f"Resolving field mapping: '{value_template}'") + logger.debug(f"Available branch results: {list(branch_results.keys())}") + + # Handle simple context variables first (non-branch references) + if not '.' in value_template: + result = value_template.format(**action_context) + logger.debug(f"Simple template resolved: '{value_template}' -> '{result}'") + return result + + # Handle branch result references like {model_id.field} + import re + branch_refs = re.findall(r'\{([^}]+\.[^}]+)\}', value_template) + logger.debug(f"Found branch references: {branch_refs}") + + resolved_template = value_template + for ref in branch_refs: + try: + model_id, field_name = ref.split('.', 1) + logger.debug(f"Processing branch reference: model_id='{model_id}', field_name='{field_name}'") + + if model_id in branch_results: + branch_data = branch_results[model_id] + logger.debug(f"Branch '{model_id}' data: {branch_data}") + + if field_name in branch_data: + field_value = branch_data[field_name] + resolved_template = resolved_template.replace(f'{{{ref}}}', str(field_value)) + logger.info(f"✅ Resolved {ref} to '{field_value}'") + else: + logger.warning(f"Field '{field_name}' not found in branch '{model_id}' results.") + logger.debug(f"Available fields in '{model_id}': {list(branch_data.keys())}") + + # Try alternative field names based on the class result and model type + if isinstance(branch_data, dict): + fallback_value = None + + # First, try the exact field name + if field_name in branch_data: + fallback_value = branch_data[field_name] + # Then try 'class' field as fallback + elif 'class' in branch_data: + fallback_value = branch_data['class'] + logger.info(f"Using 'class' field as fallback for '{field_name}': '{fallback_value}'") + # For brand models, also check if the class name exists as a key + elif field_name == 'brand' and branch_data.get('class') in branch_data: + fallback_value = branch_data[branch_data['class']] + logger.info(f"Found brand value using class name as key: '{fallback_value}'") + # For body_type models, also check if the class name exists as a key + elif field_name == 'body_type' and branch_data.get('class') in branch_data: + fallback_value = branch_data[branch_data['class']] + logger.info(f"Found body_type value using class name as key: '{fallback_value}'") + + if fallback_value is not None: + resolved_template = resolved_template.replace(f'{{{ref}}}', str(fallback_value)) + logger.info(f"✅ Resolved {ref} to '{fallback_value}' (using fallback)") + else: + logger.error(f"No suitable field found for '{field_name}' in branch '{model_id}'") + logger.debug(f"Branch data structure: {branch_data}") + return None + else: + logger.error(f"Branch data for '{model_id}' is not a dictionary: {type(branch_data)}") + return None + else: + logger.warning(f"Branch '{model_id}' not found in results. Available branches: {list(branch_results.keys())}") + return None + except ValueError as e: + logger.error(f"Invalid branch reference format: {ref}") + return None + + # Format any remaining simple variables + try: + final_value = resolved_template.format(**action_context) + logger.debug(f"Final resolved value: '{final_value}'") + return final_value + except KeyError as e: + logger.warning(f"Could not resolve context variable in template: {e}") + return resolved_template + + except Exception as e: + logger.error(f"Error resolving field mapping '{value_template}': {e}") + import traceback + logger.debug(f"Field mapping error traceback: {traceback.format_exc()}") + return None + +def run_detection_with_tracking(frame, node, context=None): + """ + Structured function for running YOLO detection with BoT-SORT tracking. + Now includes track ID-based validation requiring N consecutive frames of the same track ID. + + Args: + frame: Input frame/image + node: Pipeline node configuration with model and settings + context: Optional context information (camera info, session data, etc.) + + Returns: + tuple: (all_detections, regions_dict, track_validation_result) where: + - all_detections: List of all detection objects + - regions_dict: Dict mapping class names to highest confidence detections + - track_validation_result: Dict with validation status and stable tracks + + Configuration options in node: + - model: YOLO model instance + - triggerClassIndices: List of class indices to detect (None for all classes) + - minConfidence: Minimum confidence threshold + - multiClass: Whether to enable multi-class detection mode + - expectedClasses: List of expected class names for multi-class validation + - tracking: Dict with tracking configuration + - enabled: Boolean to enable/disable tracking + - method: Tracking method ("botsort") + - reidConfig: Path to ReID config file + - stabilityThreshold: Number of consecutive frames required for validation + """ + try: + # Extract tracking configuration + tracking_config = node.get("tracking", {}) + tracking_enabled = tracking_config.get("enabled", True) + reid_config_path = tracking_config.get("reidConfig", tracking_config.get("reidConfigPath", "botsort.yaml")) + stability_threshold = tracking_config.get("stabilityThreshold", node.get("stabilityThreshold", 4)) + + # Check if we need to reset tracker after cooldown + camera_id = context.get("camera_id", "unknown") if context else "unknown" + model_id = node.get("modelId", "unknown") + stability_data = get_camera_stability_data(camera_id, model_id) + session_state = stability_data["session_state"] + + if session_state.get("reset_tracker_on_resume", False): + # Reset YOLO tracker to get fresh track IDs + if hasattr(node["model"], 'trackers') and node["model"].trackers: + node["model"].trackers.clear() # Clear tracker state + logger.info(f"Camera {camera_id}: 🔄 Reset YOLO tracker - new cars will get fresh track IDs") + session_state["reset_tracker_on_resume"] = False # Clear the flag + + # Tracking zones removed - process all detections + + # Prepare class filtering + trigger_class_indices = node.get("triggerClassIndices") + class_filter = {"classes": trigger_class_indices} if trigger_class_indices else {} + + logger.debug(f"Running detection for {node['modelId']} - tracking: {tracking_enabled}, stability_threshold: {stability_threshold}, classes: {node.get('triggerClasses', 'all')}") + + # Use predict for detection-only models (frontal detection), track for main detection models + model_id = node.get("modelId", "") + use_tracking = tracking_enabled and not ("frontal" in model_id.lower() or "detection" in model_id.lower()) + + if use_tracking: + # Use tracking for main detection models (yolo11m, etc.) + logger.debug(f"Using tracking for {model_id}") + res = node["model"].track( + frame, + stream=False, + persist=True, + **class_filter + )[0] + else: + # Use detection only for frontal detection and other detection-only models + logger.debug(f"Using prediction only for {model_id}") + res = node["model"].predict( + frame, + stream=False, + **class_filter + )[0] + + # Process detection results + candidate_detections = [] + # Use frontalMinConfidence for frontal detection models, otherwise use minConfidence + model_id = node.get("modelId", "") + if "frontal" in model_id.lower() and "frontalMinConfidence" in node: + min_confidence = node.get("frontalMinConfidence", 0.0) + logger.debug(f"Using frontalMinConfidence={min_confidence} for {model_id}") + else: + min_confidence = node.get("minConfidence", 0.0) + + if res.boxes is None or len(res.boxes) == 0: + logger.debug(f"🚫 Camera {camera_id}: YOLO returned no detections") + + # Update stability tracking even when no detection (to reset counters) + camera_id = context.get("camera_id", "unknown") if context else "unknown" + model_id = node.get("modelId", "unknown") + track_validation_result = update_single_track_stability(node, None, camera_id, frame.shape, stability_threshold, context) + + # Store validation state in context for pipeline decisions + if context is not None: + context["track_validation_result"] = track_validation_result + + return [], {}, track_validation_result + + logger.debug(f"🔍 Camera {camera_id}: YOLO detected {len(res.boxes)} raw objects - processing with tracking...") + + # First pass: collect all valid detections + logger.debug(f"🔍 Camera {camera_id}: === DETECTION ANALYSIS ===") + for i, box in enumerate(res.boxes): + # Extract detection data + conf = float(box.cpu().conf[0]) + cls_id = int(box.cpu().cls[0]) + class_name = node["model"].names[cls_id] + + # Extract bounding box + xy = box.cpu().xyxy[0] + x1, y1, x2, y2 = map(int, xy) + bbox = (x1, y1, x2, y2) + + # Extract tracking ID if available + track_id = None + if hasattr(box, "id") and box.id is not None: + track_id = int(box.id.item()) + + logger.debug(f"🔍 Camera {camera_id}: Detection {i+1}: class='{class_name}' conf={conf:.3f} track_id={track_id} bbox={bbox}") + + # Apply confidence filtering + if conf < min_confidence: + logger.debug(f"❌ Camera {camera_id}: Detection {i+1} REJECTED - confidence {conf:.3f} < {min_confidence}") + continue + + # Tracking zone validation removed - process all detections + + # Calculate bbox area + x1, y1, x2, y2 = bbox + bbox_area = (x2 - x1) * (y2 - y1) + + # Create detection object + detection = { + "class": class_name, + "confidence": conf, + "id": track_id, + "bbox": bbox, + "bbox_area": bbox_area, + "class_id": cls_id + } + + candidate_detections.append(detection) + logger.debug(f"✅ Camera {camera_id}: Detection {i+1} ACCEPTED as candidate: {class_name} (conf={conf:.3f}, track_id={track_id})") + + # Second pass: select only the highest confidence detection overall + if not candidate_detections: + logger.debug(f"🚫 Camera {camera_id}: No valid candidates after filtering - no car will be tracked") + + # Update stability tracking even when no detection (to reset counters) + camera_id = context.get("camera_id", "unknown") if context else "unknown" + model_id = node.get("modelId", "unknown") + track_validation_result = update_single_track_stability(node, None, camera_id, frame.shape, stability_threshold, context) + + # Store validation state in context for pipeline decisions + if context is not None: + context["track_validation_result"] = track_validation_result + + return [], {}, track_validation_result + + logger.debug(f"🏆 Camera {camera_id}: === SELECTING HIGHEST CONFIDENCE CAR ===") + for i, detection in enumerate(candidate_detections): + logger.debug(f"🏆 Camera {camera_id}: Candidate {i+1}: {detection['class']} conf={detection['confidence']:.3f} track_id={detection['id']}") + + # Show all candidate detections before selection + logger.debug(f"Found {len(candidate_detections)} candidate detections:") + for i, det in enumerate(candidate_detections): + logger.debug(f"Candidate {i+1}: {det['class']} conf={det['confidence']:.3f} bbox={det['bbox']}") + + # Find the single highest confidence detection across all detected classes + best_detection = max(candidate_detections, key=lambda x: x["confidence"]) + original_class = best_detection["class"] + track_id = best_detection["id"] + + logger.info(f"🎯 Camera {camera_id}: SELECTED WINNER: {original_class} (conf={best_detection['confidence']:.3f}, track_id={track_id}, bbox={best_detection['bbox']})") + + # Show which cars were NOT selected + for detection in candidate_detections: + if detection != best_detection: + logger.debug(f"🚫 Camera {camera_id}: NOT SELECTED: {detection['class']} (conf={detection['confidence']:.3f}, track_id={detection['id']}) - lower confidence") + + # Apply class mapping if configured + mapped_class = original_class + class_mapping = node.get("classMapping", {}) + if original_class in class_mapping: + mapped_class = class_mapping[original_class] + logger.info(f"Class mapping applied: {original_class} → {mapped_class}") + # Update the detection object with mapped class + best_detection["class"] = mapped_class + best_detection["original_class"] = original_class # Keep original for reference + + # Keep only the single best detection with mapped class + all_detections = [best_detection] + regions_dict = { + mapped_class: { + "bbox": best_detection["bbox"], + "confidence": best_detection["confidence"], + "detection": best_detection, + "track_id": track_id + } + } + + # Multi-class validation + if node.get("multiClass", False) and node.get("expectedClasses"): + expected_classes = node["expectedClasses"] + detected_classes = list(regions_dict.keys()) + + logger.debug(f"Multi-class validation: expected={expected_classes}, detected={detected_classes}") + + # Check for required classes (flexible - at least one must match) + matching_classes = [cls for cls in expected_classes if cls in detected_classes] + if not matching_classes: + logger.warning(f"Multi-class validation failed: no expected classes detected") + return [], {} + + logger.info(f"Multi-class validation passed: {matching_classes} detected") + + logger.info(f"✅ Camera {camera_id}: DETECTION COMPLETE - tracking single car: track_id={track_id}, conf={best_detection['confidence']:.3f}") + logger.debug(f"📊 Camera {camera_id}: Detection summary: {len(res.boxes)} raw → {len(candidate_detections)} candidates → 1 selected") + + # Debug: Save vehicle crop for debugging (disabled for production) + # if node.get("modelId") in ["yolo11n", "yolo11m"] and regions_dict: + # try: + # import datetime + # os.makedirs("temp_debug", exist_ok=True) + # timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S_%f")[:-3] + # + # for class_name, region_data in regions_dict.items(): + # bbox = region_data['bbox'] + # x1, y1, x2, y2 = bbox + # cropped = frame[y1:y2, x1:x2] + # if cropped.size > 0: + # model_name = node.get("modelId", "yolo") + # debug_path = f"temp_debug/{model_name}_{class_name}_crop_{timestamp}.jpg" + # cv2.imwrite(debug_path, cropped) + # logger.debug(f"Saved {model_name} {class_name} crop to {debug_path}") + # except Exception as e: + # logger.error(f"Failed to save {node.get('modelId', 'yolo')} crop: {e}") + + # Update track-based stability tracking for the single selected car + camera_id = context.get("camera_id", "unknown") if context else "unknown" + model_id = node.get("modelId", "unknown") + + # Update stability tracking for the single best detection + track_validation_result = update_single_track_stability(node, best_detection, camera_id, frame.shape, stability_threshold, context) + + # Store validation state in context for pipeline decisions + if context is not None: + context["track_validation_result"] = track_validation_result + + return all_detections, regions_dict, track_validation_result + + except Exception as e: + logger.error(f"Error in detection_with_tracking for {node.get('modelId', 'unknown')}: {e}") + logger.debug(f"Detection error traceback: {traceback.format_exc()}") + return [], {}, {"validation_complete": False, "stable_tracks": [], "current_tracks": []} + + +def get_camera_stability_data(camera_id, model_id): + """Get or create stability tracking data for a specific camera and model.""" + global _camera_stability_tracking + + if camera_id not in _camera_stability_tracking: + _camera_stability_tracking[camera_id] = {} + + if model_id not in _camera_stability_tracking[camera_id]: + logger.warning(f"🔄 Camera {camera_id}: Creating NEW stability data for {model_id} - this will reset any cooldown!") + _camera_stability_tracking[camera_id][model_id] = { + "track_stability_counters": {}, # Track ID -> consecutive frame count + "stable_tracks": set(), # Set of track IDs that have reached stability threshold + "session_state": { + "active": True, + "waiting_for_backend_session": False, + "wait_start_time": 0.0, + "reset_tracker_on_resume": False + } + # Removed obsolete occupancy_state - app.py handles all mode transitions now + } + + return _camera_stability_tracking[camera_id][model_id] + +def reset_camera_stability_tracking(camera_id, model_id): + """Reset all stability tracking data for a specific camera and model.""" + if camera_id in _camera_stability_tracking and model_id in _camera_stability_tracking[camera_id]: + stability_data = _camera_stability_tracking[camera_id][model_id] + + # Clear all tracking data + track_counters = stability_data["track_stability_counters"] + stable_tracks = stability_data["stable_tracks"] + session_state = stability_data["session_state"] + + old_counters = dict(track_counters) + old_stable = list(stable_tracks) + + track_counters.clear() + stable_tracks.clear() + + # IMPORTANT: Set flag to reset YOLO tracker on next detection run + # This will ensure track IDs start fresh (1, 2, 3...) instead of continuing from old IDs + session_state["reset_tracker_on_resume"] = True + + logger.info(f"🧹 Camera {camera_id}: CLEARED stability tracking - old_counters={old_counters}, old_stable={old_stable}") + logger.info(f"🔄 Camera {camera_id}: YOLO tracker will be reset on next detection - fresh track IDs will start from 1") + else: + logger.debug(f"🧹 Camera {camera_id}: No stability tracking data to clear for model {model_id}") + +def update_single_track_stability(node, detection, camera_id, frame_shape=None, stability_threshold=4, context=None): + """Update track stability validation for a single highest confidence car.""" + model_id = node.get("modelId", "unknown") + + # Branch nodes should not do validation - only main pipeline should + is_branch_node = node.get("cropClass") is not None or node.get("parallel") is True + if is_branch_node: + logger.debug(f"⏭️ Camera {camera_id}: Skipping validation for branch node {model_id} - validation only done at main pipeline level") + return {"validation_complete": False, "branch_node": True, "stable_tracks": [], "current_tracks": []} + + # Check current mode - VALIDATION COUNTERS should increment in both validation_detecting and full_pipeline modes + current_mode = context.get("current_mode", "unknown") if context else "unknown" + is_validation_mode = (current_mode in ["validation_detecting", "full_pipeline"]) + + # Get camera-specific stability data + stability_data = get_camera_stability_data(camera_id, model_id) + track_counters = stability_data["track_stability_counters"] + stable_tracks = stability_data["stable_tracks"] + + current_track_id = detection.get("id") if detection else None + + # ─── Bbox Area Validation for Stability ─── + # Only count detections where bbox area is >=20% of frame (close cars only) + if detection and frame_shape is not None: + bbox = detection.get("bbox", [0, 0, 0, 0]) + if bbox and len(bbox) >= 4: + x1, y1, x2, y2 = bbox + bbox_area = (x2 - x1) * (y2 - y1) + frame_height, frame_width = frame_shape[:2] + frame_area = frame_height * frame_width + bbox_area_ratio = bbox_area / frame_area if frame_area > 0 else 0 + + # Require bbox to be at least 20% of frame area + min_area_ratio = 0.2 + + if bbox_area_ratio < min_area_ratio: + logger.info(f"🚫 Camera {camera_id}: Track {current_track_id} REJECTED for stability - bbox area {bbox_area_ratio:.1%} < {min_area_ratio:.0%} (too small/distant)") + # Completely reset - remove track entirely (same as trackId change) + if current_track_id and current_track_id in track_counters: + old_count = track_counters.pop(current_track_id, 0) # Remove completely + stable_tracks.discard(current_track_id) # Remove from stable + logger.info(f"🔄 Camera {camera_id}: COMPLETELY RESET track {current_track_id} counter from {old_count} to 0 (reason: bbox too small)") + return {"validation_complete": False, "stable_tracks": list(stable_tracks), "current_tracks": [], "bbox_too_small": True} + else: + logger.debug(f"✅ Camera {camera_id}: Track {current_track_id} bbox area {bbox_area_ratio:.1%} >= {min_area_ratio:.0%} - acceptable for stability") + + # ═══ MODE-AWARE TRACK VALIDATION ═══ + logger.debug(f"📋 Camera {camera_id}: === TRACK VALIDATION ANALYSIS ===") + logger.debug(f"📋 Camera {camera_id}: Current mode: {current_mode} (validation_mode={is_validation_mode})") + logger.debug(f"📋 Camera {camera_id}: Current track_id: {current_track_id} (assigned by YOLO tracking - not sequential)") + logger.debug(f"📋 Camera {camera_id}: Existing counters: {dict(track_counters)}") + logger.debug(f"📋 Camera {camera_id}: Stable tracks: {list(stable_tracks)}") + + # IMPORTANT: Only modify validation counters during validation_detecting mode + if not is_validation_mode: + logger.debug(f"🚫 Camera {camera_id}: NOT in validation mode - skipping counter modifications") + return { + "validation_complete": False, + "stable_tracks": list(stable_tracks), + "current_tracks": [current_track_id] if current_track_id is not None else [] + } + + if current_track_id is not None: + # Check if this is a different track than we were tracking + previous_track_ids = list(track_counters.keys()) + + # VALIDATION MODE: Reset counter if different track OR if track was previously stable + should_reset = ( + len(previous_track_ids) == 0 or # No previous tracking + current_track_id not in previous_track_ids or # Different track ID + current_track_id in stable_tracks # Track was stable - start fresh validation + ) + + logger.debug(f"📋 Camera {camera_id}: Previous track_ids: {previous_track_ids}") + logger.debug(f"📋 Camera {camera_id}: Track {current_track_id} was stable: {current_track_id in stable_tracks}") + logger.debug(f"📋 Camera {camera_id}: Should reset counters: {should_reset}") + + if should_reset: + # Clear all previous tracking - fresh validation needed + if previous_track_ids: + for old_track_id in previous_track_ids: + old_count = track_counters.pop(old_track_id, 0) + stable_tracks.discard(old_track_id) + logger.info(f"🔄 Camera {camera_id}: VALIDATION RESET - track {old_track_id} counter from {old_count} to 0 (reason: {'stable_track_restart' if current_track_id == old_track_id else 'different_track'})") + + # Start fresh validation for this track + old_count = track_counters.get(current_track_id, 0) # Store old count for logging + track_counters[current_track_id] = 1 + current_count = 1 + logger.info(f"🆕 Camera {camera_id}: FRESH VALIDATION - Track {current_track_id} starting at 1/{stability_threshold}") + else: + # Continue validation for same track + old_count = track_counters.get(current_track_id, 0) + track_counters[current_track_id] = old_count + 1 + current_count = track_counters[current_track_id] + + logger.debug(f"🔢 Camera {camera_id}: Track {current_track_id} counter: {old_count} → {current_count}") + logger.info(f"🔍 Camera {camera_id}: Track ID {current_track_id} validation {current_count}/{stability_threshold}") + + # Check if track has reached stability threshold + logger.debug(f"📊 Camera {camera_id}: Checking stability: {current_count} >= {stability_threshold}? {current_count >= stability_threshold}") + logger.debug(f"📊 Camera {camera_id}: Already stable: {current_track_id in stable_tracks}") + + if current_count >= stability_threshold and current_track_id not in stable_tracks: + stable_tracks.add(current_track_id) + logger.info(f"✅ Camera {camera_id}: Track ID {current_track_id} STABLE after {current_count} consecutive frames") + logger.info(f"🎯 Camera {camera_id}: TRACK VALIDATION COMPLETE") + logger.debug(f"🎯 Camera {camera_id}: Stable tracks now: {list(stable_tracks)}") + return { + "validation_complete": True, + "send_none_detection": True, + "stable_tracks": [current_track_id], + "newly_stable_tracks": [current_track_id], + "current_tracks": [current_track_id] + } + elif current_count >= stability_threshold: + logger.debug(f"📊 Camera {camera_id}: Track {current_track_id} already stable - not re-adding") + else: + # No car detected - ALWAYS clear all tracking and reset counters + logger.debug(f"🚫 Camera {camera_id}: NO CAR DETECTED - clearing all tracking") + if track_counters or stable_tracks: + logger.debug(f"🚫 Camera {camera_id}: Existing state before reset: counters={dict(track_counters)}, stable={list(stable_tracks)}") + for track_id in list(track_counters.keys()): + old_count = track_counters.pop(track_id, 0) + logger.info(f"🔄 Camera {camera_id}: No car detected - RESET track {track_id} counter from {old_count} to 0") + track_counters.clear() # Ensure complete reset + stable_tracks.clear() # Clear all stable tracks + logger.info(f"✅ Camera {camera_id}: RESET TO VALIDATION PHASE - All counters and stable tracks cleared") + else: + logger.debug(f"🚫 Camera {camera_id}: No existing counters to clear") + logger.debug(f"Camera {camera_id}: VALIDATION - no car detected (all counters reset)") + + # Final return - validation not complete + result = { + "validation_complete": False, + "stable_tracks": list(stable_tracks), + "current_tracks": [current_track_id] if current_track_id is not None else [] + } + + logger.debug(f"📋 Camera {camera_id}: Track stability result: {result}") + logger.debug(f"📋 Camera {camera_id}: Final counters: {dict(track_counters)}") + logger.debug(f"📋 Camera {camera_id}: Final stable tracks: {list(stable_tracks)}") + + return result + +# Keep the old function for backward compatibility but mark as deprecated +def update_track_stability_validation(node, detections, camera_id, frame_shape=None, stability_threshold=4): + """DEPRECATED: Use update_single_track_stability instead.""" + logger.warning(f"update_track_stability_validation called for camera {camera_id} - this function is deprecated, use update_single_track_stability instead") + if detections: + best_detection = max(detections, key=lambda x: x.get("confidence", 0)) + return update_single_track_stability(node, best_detection, camera_id, frame_shape, stability_threshold, None) + else: + return update_single_track_stability(node, None, camera_id, frame_shape, stability_threshold, None) + +def update_detection_stability(node, detections, camera_id, frame_shape=None): + """Legacy detection-based stability counter - DEPRECATED.""" + # This function is deprecated in favor of track-based validation only + logger.warning(f"update_detection_stability called for camera {camera_id} - this function is deprecated, use track-based validation instead") + return {"validation_complete": False, "valid_detections": 0, "deprecated": True} + +def update_track_stability(node, detections, camera_id, frame_shape=None): + """DEPRECATED: This function is obsolete and should not be used.""" + logger.warning(f"update_track_stability called for camera {camera_id} - this function is deprecated and obsolete") + return {"phase": "validation", "absence_counter": 0, "deprecated": True} + +def check_stable_tracks(camera_id, model_id, regions_dict): + """Check if any stable tracks match the detected classes for a specific camera.""" + # Get camera-specific stability data + stability_data = get_camera_stability_data(camera_id, model_id) + stable_tracks = stability_data["stable_tracks"] + + if not stable_tracks: + return False, [] + + # Check for track-based stability + stable_detections = [] + + for class_name, region_data in regions_dict.items(): + detection = region_data.get("detection", {}) + track_id = detection.get("id") + + if track_id is not None and track_id in stable_tracks: + stable_detections.append((class_name, track_id)) + logger.debug(f"Camera {camera_id}: Found stable detection: {class_name} with stable track ID {track_id}") + + has_stable_tracks = len(stable_detections) > 0 + return has_stable_tracks, stable_detections + +def reset_tracking_state(camera_id, model_id, reason="session ended"): + """Reset tracking state after session completion or timeout.""" + stability_data = get_camera_stability_data(camera_id, model_id) + session_state = stability_data["session_state"] + + # Clear all tracking data for fresh start + stability_data["track_stability_counters"].clear() + stability_data["stable_tracks"].clear() + session_state["active"] = True + session_state["waiting_for_backend_session"] = False + session_state["wait_start_time"] = 0.0 + session_state["reset_tracker_on_resume"] = True + + logger.info(f"Camera {camera_id}: 🔄 Reset tracking state - {reason}") + logger.info(f"Camera {camera_id}: 🧹 Cleared stability counters and stable tracks for fresh session") + +def is_camera_active(camera_id, model_id): + """Check if camera should be processing detections.""" + stability_data = get_camera_stability_data(camera_id, model_id) + session_state = stability_data["session_state"] + + # Check if waiting for backend sessionId has timed out + if session_state.get("waiting_for_backend_session", False): + current_time = time.time() + wait_start_time = session_state.get("wait_start_time", 0) + elapsed_time = current_time - wait_start_time + + if elapsed_time >= _session_timeout_seconds: + logger.warning(f"Camera {camera_id}: Backend sessionId timeout ({_session_timeout_seconds}s) - resetting tracking") + reset_tracking_state(camera_id, model_id, "backend sessionId timeout") + return True + else: + remaining_time = _session_timeout_seconds - elapsed_time + logger.debug(f"Camera {camera_id}: Still waiting for backend sessionId - {remaining_time:.1f}s remaining") + return False + + return session_state.get("active", True) + +def cleanup_pipeline_node(node: dict): + """Clean up a pipeline node and release its model reference.""" + if node and "modelId" in node: + model_id = node["modelId"] + logger.info(f"🧹 Cleaning up pipeline node: {model_id}") + release_shared_model(model_id) + + # Recursively clean up branches + for branch in node.get("branches", []): + cleanup_pipeline_node(branch) + +def cleanup_camera_stability(camera_id): + """Clean up stability tracking data when a camera is disconnected.""" + global _camera_stability_tracking + if camera_id in _camera_stability_tracking: + del _camera_stability_tracking[camera_id] + logger.info(f"Cleaned up stability tracking data for camera {camera_id}") + +def occupancy_detector(camera_id, model_id, enable=True): + """ + Temporary function to stop model inference after pipeline completion. + + Args: + camera_id (str): Camera identifier + model_id (str): Model identifier + enable (bool): True to enable occupancy mode (stop model after pipeline), False to disable + + When enabled: + - Model stops inference after completing full pipeline + - Backend sessionId handling continues in background + + Note: This is a temporary function that will be changed in the future. + """ + stability_data = get_camera_stability_data(camera_id, model_id) + session_state = stability_data["session_state"] + + if enable: + session_state["occupancy_mode"] = True + session_state["occupancy_enabled_at"] = time.time() + # Occupancy mode logging removed - not used in enhanced lightweight mode + else: + session_state["occupancy_mode"] = False + session_state.pop("occupancy_enabled_at", None) + # Occupancy mode logging removed - not used in enhanced lightweight mode + + return session_state.get("occupancy_mode", False) + +def validate_pipeline_execution(node, regions_dict): + """ + Pre-validate that all required branches will execute successfully before + committing to Redis actions and database records. + + Returns: + - (True, []) if pipeline can execute completely + - (False, missing_branches) if some required branches won't execute + """ + # Get all branches that parallel actions are waiting for + required_branches = set() + + for action in node.get("parallelActions", []): + if action.get("type") == "postgresql_update_combined": + wait_for_branches = action.get("waitForBranches", []) + required_branches.update(wait_for_branches) + + if not required_branches: + # No parallel actions requiring specific branches + logger.debug("No parallel actions with waitForBranches - validation passes") + return True, [] + + logger.debug(f"Pre-validation: checking if required branches {list(required_branches)} will execute") + + # Check each required branch + missing_branches = [] + + for branch in node.get("branches", []): + branch_id = branch["modelId"] + + if branch_id not in required_branches: + continue # This branch is not required by parallel actions + + # Check if this branch would be triggered + trigger_classes = branch.get("triggerClasses", []) + min_conf = branch.get("minConfidence", 0) + + branch_triggered = False + for det_class in regions_dict: + det_confidence = regions_dict[det_class]["confidence"] + + if (det_class in trigger_classes and det_confidence >= min_conf): + branch_triggered = True + logger.debug(f"Pre-validation: branch {branch_id} WILL be triggered by {det_class} (conf={det_confidence:.3f} >= {min_conf})") + break + + if not branch_triggered: + missing_branches.append(branch_id) + logger.warning(f"Pre-validation: branch {branch_id} will NOT be triggered - no matching classes or insufficient confidence") + logger.debug(f" Required: {trigger_classes} with min_conf={min_conf}") + logger.debug(f" Available: {[(cls, regions_dict[cls]['confidence']) for cls in regions_dict]}") + + if missing_branches: + logger.error(f"Pipeline pre-validation FAILED: required branches {missing_branches} will not execute") + return False, missing_branches + else: + logger.info(f"Pipeline pre-validation PASSED: all required branches {list(required_branches)} will execute") + return True, [] + +def run_lightweight_detection_with_validation(frame, node: dict, min_confidence=0.7, min_bbox_area_ratio=0.3): + """ + Run lightweight detection with validation rules for session ID triggering. + Returns detection info only if it passes validation thresholds. + """ + logger.debug(f"Running lightweight detection with validation: {node['modelId']} (conf>={min_confidence}, bbox_area>={min_bbox_area_ratio})") + + try: + # Run basic detection only - no branches, no actions + model = node["model"] + trigger_classes = node.get("triggerClasses", []) + trigger_class_indices = node.get("triggerClassIndices") + + # Run YOLO inference + res = model(frame, verbose=False) + + best_detection = None + frame_height, frame_width = frame.shape[:2] + frame_area = frame_height * frame_width + + for r in res: + boxes = r.boxes + if boxes is None or len(boxes) == 0: + continue + + for box in boxes: + # Extract detection info + xyxy = box.xyxy[0].cpu().numpy() + conf = box.conf[0].cpu().numpy() + cls_id = int(box.cls[0].cpu().numpy()) + class_name = model.names[cls_id] + + # Apply confidence threshold + if conf < min_confidence: + continue + + # Apply trigger class filtering if specified + if trigger_class_indices and cls_id not in trigger_class_indices: + continue + if trigger_classes and class_name not in trigger_classes: + continue + + # Calculate bbox area ratio + x1, y1, x2, y2 = xyxy + bbox_area = (x2 - x1) * (y2 - y1) + bbox_area_ratio = bbox_area / frame_area if frame_area > 0 else 0 + + # Apply bbox area threshold + if bbox_area_ratio < min_bbox_area_ratio: + logger.debug(f"Detection filtered out: bbox_area_ratio={bbox_area_ratio:.3f} < {min_bbox_area_ratio}") + continue + + # Validation passed + if not best_detection or conf > best_detection["confidence"]: + best_detection = { + "class": class_name, + "confidence": float(conf), + "bbox": [int(x) for x in xyxy], + "bbox_area_ratio": float(bbox_area_ratio), + "validation_passed": True + } + + if best_detection: + logger.debug(f"Validation PASSED: {best_detection['class']} (conf: {best_detection['confidence']:.3f}, area: {best_detection['bbox_area_ratio']:.3f})") + return best_detection + else: + logger.debug(f"Validation FAILED: No detection meets criteria (conf>={min_confidence}, area>={min_bbox_area_ratio})") + return {"validation_passed": False} + + except Exception as e: + logger.error(f"Error in lightweight detection with validation: {str(e)}", exc_info=True) + return {"validation_passed": False} + +def run_lightweight_detection(frame, node: dict): + """ + Run lightweight detection for car presence validation only. + Returns basic detection info without running branches or external actions. + """ + logger.debug(f"Running lightweight detection: {node['modelId']}") + + try: + # Run basic detection only - no branches, no actions + model = node["model"] + min_confidence = node.get("minConfidence", 0.5) + trigger_classes = node.get("triggerClasses", []) + trigger_class_indices = node.get("triggerClassIndices") + + # Run YOLO inference + res = model(frame, verbose=False) + + car_detected = False + best_detection = None + + for r in res: + boxes = r.boxes + if boxes is None or len(boxes) == 0: + continue + + for box in boxes: + # Extract detection info + xyxy = box.xyxy[0].cpu().numpy() + conf = box.conf[0].cpu().numpy() + cls_id = int(box.cls[0].cpu().numpy()) + class_name = model.names[cls_id] + + # Apply confidence threshold + if conf < min_confidence: + continue + + # Apply trigger class filtering if specified + if trigger_class_indices and cls_id not in trigger_class_indices: + continue + if trigger_classes and class_name not in trigger_classes: + continue + + # Car detected + car_detected = True + if not best_detection or conf > best_detection["confidence"]: + best_detection = { + "class": class_name, + "confidence": float(conf), + "bbox": [int(x) for x in xyxy] + } + + logger.debug(f"Lightweight detection result: car_detected={car_detected}") + if best_detection: + logger.debug(f"Best detection: {best_detection['class']} (conf: {best_detection['confidence']:.3f})") + + return { + "car_detected": car_detected, + "best_detection": best_detection + } + + except Exception as e: + logger.error(f"Error in lightweight detection: {str(e)}", exc_info=True) + return {"car_detected": False, "best_detection": None} + +def run_pipeline(frame, node: dict, return_bbox: bool=False, context=None, validated_detection=None): + """ + Enhanced pipeline that supports: + - Multi-class detection (detecting multiple classes simultaneously) + - Parallel branch processing + - Region-based actions and cropping + - Context passing for session/camera information + """ + try: + # Extract backend sessionId from context at the start of function + backend_session_id = context.get("backend_session_id") if context else None + camera_id = context.get("camera_id", "unknown") if context else "unknown" + model_id = node.get("modelId", "unknown") + + if backend_session_id: + logger.info(f"🔑 PIPELINE USING BACKEND SESSION_ID: {backend_session_id} for camera {camera_id}") + + task = getattr(node["model"], "task", None) + + # ─── Classification stage ─────────────────────────────────── + if task == "classify": + results = node["model"].predict(frame, stream=False) + if not results: + return (None, None) if return_bbox else None + + r = results[0] + probs = r.probs + if probs is None: + return (None, None) if return_bbox else None + + top1_idx = int(probs.top1) + top1_conf = float(probs.top1conf) + class_name = node["model"].names[top1_idx] + + det = { + "class": class_name, + "confidence": top1_conf, + "id": None, + class_name: class_name # Add class name as key for backward compatibility + } + + # Add specific field mappings for database operations based on model type + model_id = node.get("modelId", "").lower() + if "brand" in model_id or "brand_cls" in model_id: + det["brand"] = class_name + elif "bodytype" in model_id or "body" in model_id: + det["body_type"] = class_name + elif "color" in model_id: + det["color"] = class_name + + execute_actions(node, frame, det, context.get("regions_dict") if context else None) + return (det, None) if return_bbox else det + + # ─── Occupancy mode check (stop future frames after pipeline completion) ─────────────────────────────────────── + # Old occupancy mode logic removed - now using two-phase detection system + + # ─── Session management check ─────────────────────────────────────── + if not is_camera_active(camera_id, model_id): + logger.debug(f"⏰ Camera {camera_id}: Waiting for backend sessionId, sending 'none' detection") + none_detection = { + "class": "none", + "confidence": 1.0, + "bbox": [0, 0, 0, 0], + "branch_results": {} + } + return (none_detection, [0, 0, 0, 0]) if return_bbox else none_detection + + # ─── Detection stage - Use validated detection if provided (full_pipeline mode) ─── + if validated_detection: + track_id = validated_detection.get('track_id') + logger.info(f"🔄 PIPELINE: Using validated detection from validation phase - track_id={track_id}") + # Convert validated detection back to all_detections format for branch processing + all_detections = [validated_detection] + # Create regions_dict based on validated detection class with proper structure + class_name = validated_detection.get("class", "car") + regions_dict = { + class_name: { + "confidence": validated_detection.get("confidence"), + "bbox": validated_detection.get("bbox", [0, 0, 0, 0]), + "detection": validated_detection + } + } + # Bypass track validation completely - force pipeline execution + track_validation_result = { + "validation_complete": True, + "stable_tracks": ["cached"], # Use dummy stable track to force pipeline execution + "current_tracks": ["cached"], + "bypass_validation": True + } + else: + # Normal detection stage - Using structured detection function + all_detections, regions_dict, track_validation_result = run_detection_with_tracking(frame, node, context) + + # ─── Apply largest bbox area selection for full pipeline mode ─── + # When we have stable tracks and multiple detections, select the largest bbox area one + stable_tracks = track_validation_result.get("stable_tracks", []) + if stable_tracks and len(all_detections) > 1: + logger.info(f"🔍 PIPELINE: Full pipeline mode - selecting largest bbox area from {len(all_detections)} detections") + + # Select detection with largest bbox area + largest_detection = max(all_detections, key=lambda x: x.get("bbox_area", 0)) + logger.info(f"🎯 PIPELINE: Selected largest bbox area detection: conf={largest_detection.get('confidence', 0):.3f}, area={largest_detection.get('bbox_area', 0):.0f}") + + # Update all_detections to only contain the largest bbox area detection + all_detections = [largest_detection] + + # Update regions_dict to reflect the selected detection + class_name = largest_detection.get("class", "car") + regions_dict = { + class_name: { + "confidence": largest_detection.get("confidence"), + "bbox": largest_detection.get("bbox", [0, 0, 0, 0]), + "detection": largest_detection + } + } + + logger.debug(f"🔄 PIPELINE: Updated regions_dict for largest bbox area selection: {list(regions_dict.keys())}") + elif stable_tracks: + logger.debug(f"🔄 PIPELINE: Full pipeline mode - single detection, no area selection needed") + else: + logger.debug(f"🔄 PIPELINE: No stable tracks yet, proceeding with confidence-based detection") + + # Debug: Save crops for debugging (disabled for production) + # if regions_dict: + # try: + # import datetime + # os.makedirs("temp_debug", exist_ok=True) + # timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + # model_id = node.get("modelId", "unknown") + # + # # Save vehicle crop from yolo model (any vehicle: car, truck, bus, motorcycle) + # if model_id in ["yolo11n", "yolo11m"]: + # # Look for any vehicle class in regions_dict + # vehicle_classes = ["car", "truck", "bus", "motorcycle"] + # found_vehicle = None + # for vehicle_class in vehicle_classes: + # if vehicle_class in regions_dict: + # found_vehicle = vehicle_class + # break + # + # if found_vehicle: + # bbox = regions_dict[found_vehicle]['bbox'] + # x1, y1, x2, y2 = bbox + # cropped = frame[y1:y2, x1:x2] + # if cropped.size > 0: + # debug_path = f"temp_debug/{found_vehicle}_crop_{timestamp}.jpg" + # cv2.imwrite(debug_path, cropped) + # logger.debug(f"Saved {found_vehicle} crop to {debug_path}") + # + # # Save frontal crop from frontal_detection_v1 + # elif model_id == "frontal_detection_v1" and "frontal" in regions_dict: + # bbox = regions_dict["frontal"]['bbox'] + # x1, y1, x2, y2 = bbox + # cropped = frame[y1:y2, x1:x2] + # if cropped.size > 0: + # debug_path = f"temp_debug/frontal_crop_{timestamp}.jpg" + # cv2.imwrite(debug_path, cropped) + # logger.debug(f"Saved frontal crop to {debug_path}") + # + # except Exception as e: + # logger.error(f"Failed to save crops: {e}") + + if not all_detections: + logger.debug("No detections from structured detection function - sending 'none' detection") + none_detection = { + "class": "none", + "confidence": 1.0, + "bbox": [0, 0, 0, 0], + "branch_results": {} + } + return (none_detection, [0, 0, 0, 0]) if return_bbox else none_detection + + # Extract bounding boxes for compatibility + all_boxes = [det["bbox"] for det in all_detections] + + # ─── Track-Based Validation System: Using Track ID Stability ──────────────────────── + tracking_config = node.get("tracking", {}) + stability_threshold = tracking_config.get("stabilityThreshold", node.get("stabilityThreshold", 1)) + + camera_id = context.get("camera_id", "unknown") if context else "unknown" + + if stability_threshold > 1 and tracking_config.get("enabled", True): + # Note: Old occupancy state system removed - app.py handles all mode transitions now + # Track validation is handled by update_single_track_stability function + model_id = node.get("modelId", "unknown") + + # Simplified: just check if we have stable tracks from track validation + current_phase = "validation" # Always validation phase in simplified system + absence_counter = 0 + max_absence_frames = 3 + + if current_phase == "validation": + # ═══ TRACK VALIDATION PHASE ═══ + # Check if this is a branch node - branches should execute regardless of main validation state + is_branch_node = node.get("cropClass") is not None or node.get("parallel") is True + + if is_branch_node: + # This is a branch node - allow normal execution regardless of main pipeline validation + logger.debug(f"🔍 Camera {camera_id}: Branch node {model_id} executing during track validation phase") + else: + # Main pipeline node during track validation - check for stable tracks + stable_tracks = track_validation_result.get("stable_tracks", []) + + if not stable_tracks: + # No stable tracks yet - return detection without branches until track validation completes + if all_detections: + # Return the best detection but skip branches during validation + primary_detection = max(all_detections, key=lambda x: x["confidence"]) + logger.debug(f"🔍 Camera {camera_id}: TRACK VALIDATION PHASE - returning detection without branches (stable_tracks: {len(stable_tracks)}, sessionId: {backend_session_id or 'none'})") + else: + # No detection - return none + primary_detection = {"class": "none", "confidence": 0.0, "bbox": [0, 0, 0, 0]} + logger.debug(f"🔍 Camera {camera_id}: TRACK VALIDATION PHASE - no detection found (sessionId: {backend_session_id or 'none'})") + + primary_bbox = primary_detection.get("bbox", [0, 0, 0, 0]) + return (primary_detection, primary_bbox) if return_bbox else primary_detection + else: + # We have stable tracks - validation is complete, proceed with pipeline + logger.info(f"🎯 Camera {camera_id}: STABLE TRACKS DETECTED - proceeding with full pipeline (tracks: {stable_tracks})") + + # Note: Old waiting_for_session and occupancy phases removed + # app.py lightweight mode handles all state transitions now + + # ─── Pre-validate pipeline execution (only proceed if we have stable tracks for main pipeline) ──────────────────────── + is_branch_node = node.get("cropClass") is not None or node.get("parallel") is True + + if not is_branch_node and stability_threshold > 1 and tracking_config.get("enabled", True): + # Main pipeline node with tracking - check for stable tracks before proceeding + stable_tracks = track_validation_result.get("stable_tracks", []) + if not stable_tracks: + logger.debug(f"🔒 Camera {camera_id}: Main pipeline requires stable tracks - none found, skipping pipeline execution") + none_detection = {"class": "none", "confidence": 1.0, "bbox": [0, 0, 0, 0], "awaiting_stable_tracks": True} + return (none_detection, [0, 0, 0, 0]) if return_bbox else none_detection + + pipeline_valid, missing_branches = validate_pipeline_execution(node, regions_dict) + + if not pipeline_valid: + logger.error(f"Pipeline execution validation FAILED - required branches {missing_branches} cannot execute") + logger.error("Aborting pipeline: no Redis actions or database records will be created") + return (None, None) if return_bbox else None + + # ─── Execute actions with region information ──────────────── + detection_result = { + "detections": all_detections, + "regions": regions_dict, + **(context or {}) + } + + # ─── Database operations will be handled when backend sessionId is received ──── + + if node.get("db_manager") and regions_dict: + detected_classes = list(regions_dict.keys()) + logger.debug(f"Valid detections found: {detected_classes}") + + if backend_session_id: + # Backend sessionId is available, proceed with database operations + from datetime import datetime + display_id = detection_result.get("display_id", "unknown") + timestamp = datetime.now().strftime("%Y-%m-%dT%H-%M-%S") + + inserted_session_id = node["db_manager"].insert_initial_detection( + display_id=display_id, + captured_timestamp=timestamp, + session_id=backend_session_id + ) + + if inserted_session_id: + detection_result["session_id"] = inserted_session_id + detection_result["timestamp"] = timestamp + logger.info(f"💾 DATABASE RECORD CREATED with backend session_id: {inserted_session_id}") + logger.debug(f"Database record: display_id={display_id}, timestamp={timestamp}") + else: + logger.error(f"Failed to create database record with backend session_id: {backend_session_id}") + else: + logger.info(f"📡 Camera {camera_id}: Full pipeline completed, detection data will be sent to backend. Database operations will occur when sessionId is received.") + # Store detection info for later database operations when sessionId arrives + detection_result["awaiting_session_id"] = True + from datetime import datetime + detection_result["timestamp"] = datetime.now().strftime("%Y-%m-%dT%H-%M-%S") + + # Execute actions for root node only if it doesn't have branches + # Branch nodes with actions will execute them after branch processing + if not node.get("branches") or node.get("modelId") == "yolo11n": + execute_actions(node, frame, detection_result, regions_dict) + + # ─── Branch processing (no stability check here) ───────────────────────────── + if node["branches"]: + branch_results = {} + + # Extract camera_id for logging + camera_id = detection_result.get("camera_id", context.get("camera_id", "unknown") if context else "unknown") + + + # Filter branches that should be triggered + active_branches = [] + for br in node["branches"]: + trigger_classes = br.get("triggerClasses", []) + min_conf = br.get("minConfidence", 0) + + logger.debug(f"Evaluating branch {br['modelId']}: trigger_classes={trigger_classes}, min_conf={min_conf}") + + # Check if any detected class matches branch trigger + branch_triggered = False + for det_class in regions_dict: + det_confidence = regions_dict[det_class]["confidence"] + logger.debug(f" Checking detected class '{det_class}' (confidence={det_confidence:.3f}) against triggers {trigger_classes}") + + if (det_class in trigger_classes and det_confidence >= min_conf): + active_branches.append(br) + branch_triggered = True + logger.info(f"Branch {br['modelId']} activated by class '{det_class}' (conf={det_confidence:.3f} >= {min_conf})") + break + + if not branch_triggered: + logger.debug(f"Branch {br['modelId']} not triggered - no matching classes or insufficient confidence") + + if active_branches: + if node.get("parallel", False) or any(br.get("parallel", False) for br in active_branches): + # Run branches in parallel + with concurrent.futures.ThreadPoolExecutor(max_workers=len(active_branches)) as executor: + futures = {} + + for br in active_branches: + sub_frame = frame + crop_class = br.get("cropClass") + + logger.info(f"Starting parallel branch: {br['modelId']}, cropClass: {crop_class}") + + if br.get("crop", False) and crop_class: + if crop_class in regions_dict: + cropped = crop_region_by_class(frame, regions_dict, crop_class) + if cropped is not None: + sub_frame = cropped # Use cropped image without manual resizing + logger.debug(f"Successfully cropped {crop_class} region for {br['modelId']} - model will handle resizing") + else: + logger.warning(f"Failed to crop {crop_class} region for {br['modelId']}, skipping branch") + continue + else: + logger.warning(f"Crop class {crop_class} not found in detected regions for {br['modelId']}, skipping branch") + continue + + # Add regions_dict and session_id to context for child branches + branch_context = dict(context) if context else {} + branch_context["regions_dict"] = regions_dict + + # Pass session_id from detection_result to branch context for Redis actions + if "session_id" in detection_result: + branch_context["session_id"] = detection_result["session_id"] + logger.debug(f"Added session_id to branch context: {detection_result['session_id']}") + elif backend_session_id: + branch_context["session_id"] = backend_session_id + logger.debug(f"Added backend_session_id to branch context: {backend_session_id}") + + future = executor.submit(run_pipeline, sub_frame, br, True, branch_context) + futures[future] = br + + # Collect results + for future in concurrent.futures.as_completed(futures): + br = futures[future] + try: + result, _ = future.result() + if result: + branch_results[br["modelId"]] = result + logger.info(f"Branch {br['modelId']} completed: {result}") + + # Collect nested branch results if they exist + if "branch_results" in result: + for nested_id, nested_result in result["branch_results"].items(): + branch_results[nested_id] = nested_result + logger.info(f"Collected nested branch result: {nested_id} = {nested_result}") + except Exception as e: + logger.error(f"Branch {br['modelId']} failed: {e}") + else: + # Run branches sequentially + for br in active_branches: + sub_frame = frame + crop_class = br.get("cropClass") + + logger.info(f"Starting sequential branch: {br['modelId']}, cropClass: {crop_class}") + + if br.get("crop", False) and crop_class: + if crop_class in regions_dict: + cropped = crop_region_by_class(frame, regions_dict, crop_class) + if cropped is not None: + sub_frame = cropped # Use cropped image without manual resizing + logger.debug(f"Successfully cropped {crop_class} region for {br['modelId']} - model will handle resizing") + else: + logger.warning(f"Failed to crop {crop_class} region for {br['modelId']}, skipping branch") + continue + else: + logger.warning(f"Crop class {crop_class} not found in detected regions for {br['modelId']}, skipping branch") + continue + + try: + # Add regions_dict and session_id to context for child branches + branch_context = dict(context) if context else {} + branch_context["regions_dict"] = regions_dict + + # Pass session_id from detection_result to branch context for Redis actions + if "session_id" in detection_result: + branch_context["session_id"] = detection_result["session_id"] + logger.debug(f"Added session_id to sequential branch context: {detection_result['session_id']}") + elif backend_session_id: + branch_context["session_id"] = backend_session_id + logger.debug(f"Added backend_session_id to sequential branch context: {backend_session_id}") + + result, _ = run_pipeline(sub_frame, br, True, branch_context) + if result: + branch_results[br["modelId"]] = result + logger.info(f"Branch {br['modelId']} completed: {result}") + + # Collect nested branch results if they exist + if "branch_results" in result: + for nested_id, nested_result in result["branch_results"].items(): + branch_results[nested_id] = nested_result + logger.info(f"Collected nested branch result: {nested_id} = {nested_result}") + else: + logger.warning(f"Branch {br['modelId']} returned no result") + except Exception as e: + logger.error(f"Error in sequential branch {br['modelId']}: {e}") + import traceback + logger.debug(f"Branch error traceback: {traceback.format_exc()}") + + # Store branch results in detection_result for parallel actions + detection_result["branch_results"] = branch_results + + # ─── Execute Parallel Actions ─────────────────────────────── + if node.get("parallelActions") and "branch_results" in detection_result: + execute_parallel_actions(node, frame, detection_result, regions_dict) + + # ─── Auto-enable occupancy mode after successful pipeline completion ───────────────── + camera_id = context.get("camera_id", "unknown") if context else "unknown" + model_id = node.get("modelId", "unknown") + + # Enable occupancy detector automatically after first successful pipeline + # Auto-enabling occupancy logging removed - not used in enhanced lightweight mode + occupancy_detector(camera_id, model_id, enable=True) + + logger.info(f"✅ Camera {camera_id}: Pipeline completed, detection data will be sent to backend") + logger.info(f"🛑 Camera {camera_id}: Model will stop inference for future frames") + logger.info(f"📡 Backend sessionId will be handled when received via WebSocket") + + # ─── Execute actions after successful detection AND branch processing ────────── + # This ensures detection nodes (like frontal_detection_v1) execute their actions + # after completing both detection and branch processing + if node.get("actions") and regions_dict and node.get("modelId") != "yolo11n": + # Execute actions for branch detection nodes, skip root to avoid duplication + logger.debug(f"Executing post-detection actions for branch node {node.get('modelId')}") + execute_actions(node, frame, detection_result, regions_dict) + + # ─── Return detection result ──────────────────────────────── + primary_detection = max(all_detections, key=lambda x: x["confidence"]) + primary_bbox = primary_detection["bbox"] + + # Add branch results and session_id to primary detection for compatibility + if "branch_results" in detection_result: + primary_detection["branch_results"] = detection_result["branch_results"] + if "session_id" in detection_result: + primary_detection["session_id"] = detection_result["session_id"] + + return (primary_detection, primary_bbox) if return_bbox else primary_detection + + except Exception as e: + logger.error(f"Error in node {node.get('modelId')}: {e}") + import traceback + traceback.print_exc() + return (None, None) if return_bbox else None diff --git a/test/sample.png b/test/sample.png new file mode 100644 index 0000000..568e38f Binary files /dev/null and b/test/sample.png differ diff --git a/test/sample2.png b/test/sample2.png new file mode 100644 index 0000000..c1e8485 Binary files /dev/null and b/test/sample2.png differ diff --git a/test/test.py b/test/test.py new file mode 100644 index 0000000..ff073c4 --- /dev/null +++ b/test/test.py @@ -0,0 +1,60 @@ +from ultralytics import YOLO +import cv2 +import os + +# Load the model +# model = YOLO('../models/webcam-local-01/4/bangchak_poc/yolo11n.pt') +model = YOLO('yolo11m.pt') + +def test_image(image_path): + """Test a single image with YOLO model""" + if not os.path.exists(image_path): + print(f"Image not found: {image_path}") + return + + # Run inference - filter for car class only (class 2 in COCO) + results = model(image_path, classes=[2, 5, 7]) # 2, 5, 7 = car, bus, truck in COCO dataset + + # Display results + for r in results: + im_array = r.plot() # plot a BGR numpy array of predictions + + # Resize image for display (max width/height 800px) + height, width = im_array.shape[:2] + max_dimension = 800 + if width > max_dimension or height > max_dimension: + if width > height: + new_width = max_dimension + new_height = int(height * (max_dimension / width)) + else: + new_height = max_dimension + new_width = int(width * (max_dimension / height)) + im_array = cv2.resize(im_array, (new_width, new_height)) + + # Show image with predictions + cv2.imshow('YOLO Test - Car Detection Only', im_array) + cv2.waitKey(0) + cv2.destroyAllWindows() + + # Print detection info + print(f"\nDetections for {image_path}:") + if r.boxes is not None and len(r.boxes) > 0: + for i, box in enumerate(r.boxes): + cls = int(box.cls[0]) + conf = float(box.conf[0]) + original_class = model.names[cls] # Original class name (car/bus/truck) + # Get bounding box coordinates + x1, y1, x2, y2 = box.xyxy[0].tolist() + # Rename all vehicle types to "car" + print(f"Detection {i+1}: car (was: {original_class}) - Confidence: {conf:.3f} - BBox: ({x1:.0f}, {y1:.0f}, {x2:.0f}, {y2:.0f})") + print(f"Total cars detected: {len(r.boxes)}") + else: + print("No cars detected in the image") + +if __name__ == "__main__": + # Test with an image file + image_path = input("Enter image path (or press Enter for default test): ") + if not image_path: + image_path = "sample.png" # Default test image + + test_image(image_path) \ No newline at end of file diff --git a/test/test_botsort_zone_track.py b/test/test_botsort_zone_track.py new file mode 100644 index 0000000..bbbd188 --- /dev/null +++ b/test/test_botsort_zone_track.py @@ -0,0 +1,352 @@ +import cv2 +import torch +import numpy as np +import time +from collections import defaultdict +from ultralytics import YOLO + +def point_in_polygon(point, polygon): + """Check if a point is inside a polygon using ray casting algorithm""" + x, y = point + n = len(polygon) + inside = False + + p1x, p1y = polygon[0] + for i in range(1, n + 1): + p2x, p2y = polygon[i % n] + if y > min(p1y, p2y): + if y <= max(p1y, p2y): + if x <= max(p1x, p2x): + if p1y != p2y: + xinters = (y - p1y) * (p2x - p1x) / (p2y - p1y) + p1x + if p1x == p2x or x <= xinters: + inside = not inside + p1x, p1y = p2x, p2y + + return inside + +def draw_zone(frame, zone_polygon, color=(255, 0, 0), thickness=3): + """Draw tracking zone on frame""" + pts = np.array(zone_polygon, np.int32) + pts = pts.reshape((-1, 1, 2)) + cv2.polylines(frame, [pts], True, color, thickness) + + # Add semi-transparent fill + overlay = frame.copy() + cv2.fillPoly(overlay, [pts], color) + cv2.addWeighted(overlay, 0.2, frame, 0.8, 0, frame) + +def setup_video_writer(output_path, fps, width, height): + """Setup video writer for output""" + fourcc = cv2.VideoWriter_fourcc(*'mp4v') + return cv2.VideoWriter(output_path, fourcc, fps, (width, height)) + +def write_frame_to_video(video_writer, frame, repeat_count): + """Write frame to video with specified repeat count""" + for _ in range(repeat_count): + video_writer.write(frame) + +def finalize_video(video_writer): + """Release video writer""" + video_writer.release() + +def main(): + video_path = "sample2.mp4" + yolo_model = "bangchakv2/yolov8n.pt" + + device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + print(f"Using device: {device}") + + print("Loading YOLO model...") + model = YOLO(yolo_model) + + print("Opening video...") + cap = cv2.VideoCapture(video_path) + fps = int(cap.get(cv2.CAP_PROP_FPS)) + width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) + + print(f"Video info: {width}x{height}, {fps} FPS, {total_frames} frames") + + # Define tracking zone - Gas station floor area (trapezoidal shape) + # Based on the perspective of the gas station floor from your image + # width 2560, height 1440 + + tracking_zone = [ + (423, 974), # Point 1 + (1540, 1407), # Point 2 + (1976, 806), # Point 3 + (1364, 749) # Point 4 + ] + + print(f"🎯 Tracking zone defined: {tracking_zone}") + + # CONTINUOUS TRACKING: Process every 118 frames (~2.0s intervals) + frame_skip = 118 + + print(f"🎯 CONTINUOUS MODE: Processing every {frame_skip} frames ({frame_skip/fps:.2f}s intervals)") + print(f"🎬 Output video will have same duration as input (each processed frame shown for 2 seconds)") + print("🔥 ZONE-FIRST TRACKING: Only cars entering the zone will be tracked!") + print("Requires 5 consecutive detections IN ZONE for verification") + print("🕐 24/7 MODE: Memory reset every hour to prevent overflow") + print("Press 'q' to quit") + + # Setup video writer for output (same fps as input for normal playback speed) + output_path = "tracking_output_botsort_zone_track.mp4" + output_fps = fps # Use same fps as input video + out = setup_video_writer(output_path, output_fps, width, height) + + # Track car IDs and their consecutive detections + car_id_counts = defaultdict(int) + successful_cars = set() + last_positions = {} + processed_count = 0 + + # ID remapping for clean sequential zone IDs + tracker_to_zone_id = {} # Maps tracker IDs to clean zone IDs + next_zone_id = 1 # Next clean zone ID to assign + + # Store previous frame detections to filter tracking inputs + previous_zone_cars = set() + + # 24/7 operation: Reset every hour (1800 snapshots at 2-sec intervals = 1 hour) + RESET_INTERVAL = 1800 # Reset every 1800 processed frames (1 hour) + + frame_idx = 0 + + while True: + # Skip frames to maintain interval + for _ in range(frame_skip): + ret, frame = cap.read() + if not ret: + print("\nNo more frames to read") + cap.release() + cv2.destroyAllWindows() + return + frame_idx += 1 + + processed_count += 1 + current_time = frame_idx / fps + + print(f"\n🎬 Frame {frame_idx} at {current_time:.2f}s (processed #{processed_count})") + + # 24/7 Memory Management: Reset every hour + if processed_count % RESET_INTERVAL == 0: + print(f"🕐 HOURLY RESET: Clearing all tracking data (processed {processed_count} frames)") + print(f" 📊 Before reset: {len(tracker_to_zone_id)} tracked cars, next Zone ID was {next_zone_id}") + + # Clear all tracking data + tracker_to_zone_id.clear() + car_id_counts.clear() + successful_cars.clear() + last_positions.clear() + next_zone_id = 1 # Reset to 1 + + # Reset BoT-SORT tracker state + try: + model.reset() + print(f" ✅ BoT-SORT tracker reset successfully") + except: + print(f" ⚠️ BoT-SORT reset not available (continuing without reset)") + + print(f" 🆕 Zone IDs will start from 1 again") + + # Draw tracking zone on frame + draw_zone(frame, tracking_zone, color=(0, 255, 255), thickness=3) # Yellow zone + + # First run YOLO detection (without tracking) to find cars in zone + detection_results = model(frame, verbose=False, conf=0.7, classes=[2]) + + # Find cars currently in the tracking zone + current_zone_cars = [] + total_detections = 0 + + if detection_results[0].boxes is not None: + boxes = detection_results[0].boxes.xyxy.cpu() + scores = detection_results[0].boxes.conf.cpu() + + total_detections = len(boxes) + print(f" 🔍 Total car detections: {total_detections}") + + for i in range(len(boxes)): + x1, y1, x2, y2 = boxes[i] + conf = float(scores[i]) + + # Check if detection is in zone (using bottom center) + box_bottom = ((x1 + x2) / 2, y2) + if point_in_polygon(box_bottom, tracking_zone): + current_zone_cars.append({ + 'bbox': [float(x1), float(y1), float(x2), float(y2)], + 'conf': conf, + 'center': ((x1 + x2) / 2, (y1 + y2) / 2), + 'bottom': box_bottom + }) + + print(f" 🎯 Cars in zone: {len(current_zone_cars)}") + + # Only run tracking if there are cars in the zone + detected_car_ids = set() + + if current_zone_cars: + # Run tracking on the full frame (let tracker handle associations) + # But we'll filter results to only zone cars afterward + results = model.track( + frame, + persist=True, + verbose=False, + conf=0.7, + classes=[2], + tracker="botsort_reid.yaml" + ) + + if results[0].boxes is not None and results[0].boxes.id is not None: + boxes = results[0].boxes.xyxy.cpu() + scores = results[0].boxes.conf.cpu() + track_ids = results[0].boxes.id.cpu().int() + + print(f" 📊 Total tracked objects: {len(track_ids)}") + + # Filter tracked objects to only those in zone + zone_tracks = [] + for i, track_id in enumerate(track_ids): + x1, y1, x2, y2 = boxes[i] + conf = float(scores[i]) + + # Check if this tracked object is in our zone + box_bottom = ((x1 + x2) / 2, y2) + if point_in_polygon(box_bottom, tracking_zone): + zone_tracks.append({ + 'id': int(track_id), + 'bbox': [int(x1), int(y1), int(x2), int(y2)], + 'conf': conf, + 'center': ((x1 + x2) / 2, (y1 + y2) / 2), + 'bottom': box_bottom + }) + + print(f" ✅ Zone tracks: {len(zone_tracks)}") + + # Process each zone track + for track in zone_tracks: + tracker_id = track['id'] # Original tracker ID + x1, y1, x2, y2 = track['bbox'] + conf = track['conf'] + box_center = track['center'] + + # Map tracker ID to clean zone ID + if tracker_id not in tracker_to_zone_id: + tracker_to_zone_id[tracker_id] = next_zone_id + print(f" 🆕 New car: Tracker ID {tracker_id} → Zone ID {next_zone_id}") + next_zone_id += 1 + + zone_id = tracker_to_zone_id[tracker_id] # Clean sequential ID + + # Validate track continuity (use tracker_id for internal logic) + is_valid = True + + # Check for suspicious jumps + if tracker_id in last_positions: + last_center = last_positions[tracker_id] + distance = np.sqrt((box_center[0] - last_center[0])**2 + + (box_center[1] - last_center[1])**2) + + if distance > 400: # pixels in ~2.0s + is_valid = False + print(f" ⚠️ Zone ID {zone_id} (Tracker {tracker_id}): suspicious jump {distance:.0f}px") + + # Skip already successful cars (use zone_id for user logic) + if zone_id in successful_cars: + is_valid = False + print(f" ✅ Zone ID {zone_id}: already successful, skipping") + + # Only process valid, high-confidence zone tracks + if is_valid and conf > 0.7: + detected_car_ids.add(zone_id) # Use zone_id for display + car_id_counts[zone_id] += 1 + last_positions[tracker_id] = box_center # Track by tracker_id internally + + # Draw tracking results with clean zone ID + zone_color = (0, 255, 0) # Green for zone cars + cv2.rectangle(frame, (x1, y1), (x2, y2), zone_color, 2) + cv2.putText(frame, f'ZONE ID:{zone_id}', + (x1, y1-30), cv2.FONT_HERSHEY_SIMPLEX, 0.6, zone_color, 2) + cv2.putText(frame, f'#{car_id_counts[zone_id]} {conf:.2f}', + (x1, y1-10), cv2.FONT_HERSHEY_SIMPLEX, 0.6, zone_color, 2) + + # Draw center point + cv2.circle(frame, (int(track['bottom'][0]), int(track['bottom'][1])), 5, zone_color, -1) + + print(f" ✅ Zone ID {zone_id} (Tracker {tracker_id}): ZONE detection #{car_id_counts[zone_id]} (conf: {conf:.2f})") + + # Check for success (5 consecutive detections IN ZONE) + if car_id_counts[zone_id] == 5: + print(f"🏆 SUCCESS: Zone ID {zone_id} achieved 5 continuous ZONE detections - TRIGGER NEXT MODEL!") + successful_cars.add(zone_id) + + # Add success indicator to frame + cv2.putText(frame, f"SUCCESS: Zone Car {zone_id}!", + (50, height-50), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 3) + else: + print(" 📋 No cars in zone - no tracking performed") + + # Draw any cars outside the zone in red (for reference) + if detection_results[0].boxes is not None: + boxes = detection_results[0].boxes.xyxy.cpu() + scores = detection_results[0].boxes.conf.cpu() + + for i in range(len(boxes)): + x1, y1, x2, y2 = boxes[i] + conf = float(scores[i]) + + box_bottom = ((x1 + x2) / 2, y2) + if not point_in_polygon(box_bottom, tracking_zone): + # Draw cars outside zone in red (not tracked) + x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2) + cv2.rectangle(frame, (x1, y1), (x2, y2), (0, 0, 255), 1) + cv2.putText(frame, f'OUT {conf:.2f}', + (x1, y1-10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1) + + # Display results + if detected_car_ids: + print(f" 📋 Active Zone IDs: {sorted(detected_car_ids)} (Clean sequential IDs)") + + # Show ID mapping for debugging + if tracker_to_zone_id: + mapping_str = ", ".join([f"Tracker{k}→Zone{v}" for k, v in tracker_to_zone_id.items()]) + print(f" 🔄 ID Mapping: {mapping_str}") + + # Add annotations to frame + cv2.putText(frame, f"BoT-SORT Zone-First Tracking | Frame: {frame_idx} | {current_time:.2f}s", + (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255, 255, 255), 2) + cv2.putText(frame, f"Zone Cars: {len(current_zone_cars)} | Active Tracks: {len(detected_car_ids)}", + (10, 65), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2) + cv2.putText(frame, f"Successful Cars: {len(successful_cars)}", + (10, 100), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0), 2) + cv2.putText(frame, "TRACKING ZONE", + (tracking_zone[0][0], tracking_zone[0][1]-10), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 255, 255), 2) + + # Write annotated frame to output video (repeat for 2 seconds duration) + write_frame_to_video(out, frame, frame_skip) + + # Show video with zone tracking info + display_frame = cv2.resize(frame, (960, 540)) + cv2.imshow('BoT-SORT Zone-First Tracking', display_frame) + + # Quick check for quit + key = cv2.waitKey(1) & 0xFF + if key == ord('q'): + break + + # Small delay to see results + time.sleep(0.1) + + cap.release() + finalize_video(out) + cv2.destroyAllWindows() + print(f"\n🎯 BoT-SORT zone-first tracking completed!") + print(f"📊 Processed {processed_count} frames with {frame_skip/fps:.2f}s intervals") + print(f"🏆 Successfully tracked {len(successful_cars)} unique cars IN ZONE") + print(f"💾 Annotated video saved to: {output_path}") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/test_protocol.py b/test_protocol.py index 6b32fd8..74af7d8 100644 --- a/test_protocol.py +++ b/test_protocol.py @@ -9,7 +9,7 @@ import time async def test_protocol(): """Test the worker protocol implementation""" - uri = "ws://localhost:8001" + uri = "ws://localhost:8000" try: async with websockets.connect(uri) as websocket: @@ -119,7 +119,7 @@ async def test_protocol(): except Exception as e: print(f"✗ Connection failed: {e}") - print("Make sure the worker is running on localhost:8001") + print("Make sure the worker is running on localhost:8000") if __name__ == "__main__": asyncio.run(test_protocol()) \ No newline at end of file diff --git a/view_redis_images.py b/view_redis_images.py new file mode 100644 index 0000000..b1b3c63 --- /dev/null +++ b/view_redis_images.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +""" +Script to view frontal images saved in Redis +""" +import redis +import cv2 +import numpy as np +import sys +from datetime import datetime + +# Redis connection config (from pipeline.json) +REDIS_CONFIG = { + "host": "10.100.1.3", + "port": 6379, + "password": "FBQgi0i5RevAAMO5Hh66", + "db": 0 +} + +def connect_redis(): + """Connect to Redis server.""" + try: + client = redis.Redis( + host=REDIS_CONFIG["host"], + port=REDIS_CONFIG["port"], + password=REDIS_CONFIG["password"], + db=REDIS_CONFIG["db"], + decode_responses=False # Keep bytes for images + ) + client.ping() + print(f"✅ Connected to Redis at {REDIS_CONFIG['host']}:{REDIS_CONFIG['port']}") + return client + except redis.exceptions.ConnectionError as e: + print(f"❌ Failed to connect to Redis: {e}") + return None + +def list_image_keys(client): + """List all image keys in Redis.""" + try: + # Look for keys matching the inference pattern + keys = client.keys("inference:*") + print(f"\n📋 Found {len(keys)} image keys:") + for i, key in enumerate(keys): + key_str = key.decode() if isinstance(key, bytes) else key + print(f"{i+1}. {key_str}") + return keys + except Exception as e: + print(f"❌ Error listing keys: {e}") + return [] + +def view_image(client, key): + """View a specific image from Redis.""" + try: + # Get image data from Redis + image_data = client.get(key) + if image_data is None: + print(f"❌ No data found for key: {key}") + return + + print(f"📸 Image size: {len(image_data)} bytes") + + # Convert bytes to numpy array + nparr = np.frombuffer(image_data, np.uint8) + + # Decode image + img = cv2.imdecode(nparr, cv2.IMREAD_COLOR) + if img is None: + print("❌ Failed to decode image data") + return + + print(f"🖼️ Image dimensions: {img.shape[1]}x{img.shape[0]} pixels") + + # Display image + key_str = key.decode() if isinstance(key, bytes) else key + cv2.imshow(f'Redis Image: {key_str}', img) + print("👁️ Image displayed. Press any key to close...") + cv2.waitKey(0) + cv2.destroyAllWindows() + + # Ask if user wants to save the image + save = input("💾 Save image to file? (y/n): ").lower().strip() + if save == 'y': + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = f"redis_image_{timestamp}.jpg" + cv2.imwrite(filename, img) + print(f"💾 Image saved as: {filename}") + + except Exception as e: + print(f"❌ Error viewing image: {e}") + +def monitor_new_images(client): + """Monitor for new images being added to Redis.""" + print("👀 Monitoring for new images... (Press Ctrl+C to stop)") + try: + # Subscribe to Redis pub/sub for car detections + pubsub = client.pubsub() + pubsub.subscribe('car_detections') + + for message in pubsub.listen(): + if message['type'] == 'message': + data = message['data'].decode() + print(f"🚨 New detection: {data}") + + # Try to extract image key from message + import json + try: + detection_data = json.loads(data) + image_key = detection_data.get('image_key') + if image_key: + print(f"🖼️ New image available: {image_key}") + view_choice = input("View this image now? (y/n): ").lower().strip() + if view_choice == 'y': + view_image(client, image_key) + except json.JSONDecodeError: + pass + + except KeyboardInterrupt: + print("\n👋 Stopping monitor...") + except Exception as e: + print(f"❌ Monitor error: {e}") + +def main(): + """Main function.""" + print("🔍 Redis Image Viewer") + print("=" * 50) + + # Connect to Redis + client = connect_redis() + if not client: + return + + while True: + print("\n📋 Options:") + print("1. List all image keys") + print("2. View specific image") + print("3. Monitor for new images") + print("4. Exit") + + choice = input("\nEnter choice (1-4): ").strip() + + if choice == '1': + keys = list_image_keys(client) + elif choice == '2': + keys = list_image_keys(client) + if keys: + try: + idx = int(input(f"\nEnter image number (1-{len(keys)}): ")) - 1 + if 0 <= idx < len(keys): + view_image(client, keys[idx]) + else: + print("❌ Invalid selection") + except ValueError: + print("❌ Please enter a valid number") + elif choice == '3': + monitor_new_images(client) + elif choice == '4': + print("👋 Goodbye!") + break + else: + print("❌ Invalid choice") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/webcam_rtsp_server.py b/webcam_rtsp_server.py new file mode 100644 index 0000000..65698ac --- /dev/null +++ b/webcam_rtsp_server.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python3 +""" +Enhanced webcam server that provides both RTSP streaming and HTTP snapshot endpoints +Compatible with CMS UI requirements for camera configuration +""" + +import cv2 +import threading +import time +import logging +import socket +from http.server import BaseHTTPRequestHandler, HTTPServer +import subprocess +import sys +import os + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s" +) +logger = logging.getLogger("webcam_rtsp_server") + +# Global webcam capture object +webcam_cap = None +rtsp_process = None + +class WebcamHTTPHandler(BaseHTTPRequestHandler): + """HTTP handler for snapshot requests""" + + def do_GET(self): + if self.path == '/snapshot' or self.path == '/snapshot.jpg': + try: + # Capture fresh frame from webcam for each request + ret, frame = webcam_cap.read() + if ret and frame is not None: + # Encode as JPEG + success, buffer = cv2.imencode('.jpg', frame, [cv2.IMWRITE_JPEG_QUALITY, 85]) + if success: + self.send_response(200) + self.send_header('Content-Type', 'image/jpeg') + self.send_header('Content-Length', str(len(buffer))) + self.send_header('Cache-Control', 'no-cache, no-store, must-revalidate') + self.send_header('Pragma', 'no-cache') + self.send_header('Expires', '0') + self.end_headers() + self.wfile.write(buffer.tobytes()) + logger.debug(f"Served webcam snapshot, size: {len(buffer)} bytes") + return + else: + logger.error("Failed to encode frame as JPEG") + else: + logger.error("Failed to capture frame from webcam") + + # Send error response + self.send_response(500) + self.send_header('Content-Type', 'text/plain') + self.end_headers() + self.wfile.write(b'Failed to capture webcam frame') + + except Exception as e: + logger.error(f"Error serving snapshot: {e}") + self.send_response(500) + self.send_header('Content-Type', 'text/plain') + self.end_headers() + self.wfile.write(f'Error: {str(e)}'.encode()) + + elif self.path == '/status': + # Status endpoint for health checking + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + + width = int(webcam_cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + height = int(webcam_cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + fps = webcam_cap.get(cv2.CAP_PROP_FPS) + + status = f'{{"status": "online", "width": {width}, "height": {height}, "fps": {fps}}}' + self.wfile.write(status.encode()) + + else: + # 404 for other paths + self.send_response(404) + self.send_header('Content-Type', 'text/plain') + self.end_headers() + self.wfile.write(b'Not Found - Available endpoints: /snapshot, /snapshot.jpg, /status') + + def log_message(self, format, *args): + # Suppress default HTTP server logging to avoid spam + pass + +def check_ffmpeg(): + """Check if FFmpeg is available for RTSP streaming""" + try: + result = subprocess.run(['ffmpeg', '-version'], + capture_output=True, text=True, timeout=5) + if result.returncode == 0: + logger.info("FFmpeg found and working") + return True + except (subprocess.TimeoutExpired, FileNotFoundError, subprocess.SubprocessError): + pass + + logger.warning("FFmpeg not found. RTSP streaming will not be available.") + logger.info("To enable RTSP streaming, install FFmpeg:") + logger.info(" Windows: Download from https://ffmpeg.org/download.html") + logger.info(" Linux: sudo apt install ffmpeg") + logger.info(" macOS: brew install ffmpeg") + return False + +def get_windows_camera_name(): + """Get the actual camera device name on Windows""" + try: + # List video devices using FFmpeg with proper encoding handling + result = subprocess.run(['ffmpeg', '-f', 'dshow', '-list_devices', 'true', '-i', 'dummy'], + capture_output=True, text=True, timeout=10, encoding='utf-8', errors='ignore') + output = result.stderr # FFmpeg outputs device list to stderr + + # Look for video devices in the output + lines = output.split('\n') + video_devices = [] + + # Parse the output - look for lines with (video) that contain device names in quotes + for line in lines: + if '[dshow @' in line and '(video)' in line and '"' in line: + # Extract device name between first pair of quotes + start = line.find('"') + 1 + end = line.find('"', start) + if start > 0 and end > start: + device_name = line[start:end] + video_devices.append(device_name) + + logger.info(f"Found Windows video devices: {video_devices}") + if video_devices: + # Force use the first device (index 0) which is the Logitech HD webcam + return video_devices[0] # This will be "罗技高清网络摄像机 C930c" + else: + logger.info("No devices found via FFmpeg detection, using fallback") + # Fall through to fallback names + + except Exception as e: + logger.debug(f"Failed to get Windows camera name: {e}") + + # Try common camera device names as fallback + # Prioritize Integrated Camera since that's what's working now + common_names = [ + "Integrated Camera", # This is working for the current setup + "USB Video Device", # Common name for USB cameras + "USB2.0 Camera", + "C930c", # Direct model name + "HD Pro Webcam C930c", # Full Logitech name + "Logitech", # Brand name + "USB Camera", + "Webcam" + ] + logger.info(f"Using fallback camera names: {common_names}") + return common_names[0] # Return "Integrated Camera" first + +def start_rtsp_stream(webcam_index=0, rtsp_port=8554): + """Start RTSP streaming using FFmpeg""" + global rtsp_process + + if not check_ffmpeg(): + return None + + try: + # Get the actual camera device name for Windows + if sys.platform.startswith('win'): + camera_name = get_windows_camera_name() + logger.info(f"Using Windows camera device: {camera_name}") + + # FFmpeg command to stream webcam via RTSP + if sys.platform.startswith('win'): + cmd = [ + 'ffmpeg', + '-f', 'dshow', + '-i', f'video={camera_name}', # Use detected camera name + '-c:v', 'libx264', + '-preset', 'veryfast', + '-tune', 'zerolatency', + '-r', '30', + '-s', '1280x720', + '-f', 'rtsp', + f'rtsp://localhost:{rtsp_port}/stream' + ] + elif sys.platform.startswith('linux'): + cmd = [ + 'ffmpeg', + '-f', 'v4l2', + '-i', f'/dev/video{webcam_index}', + '-c:v', 'libx264', + '-preset', 'veryfast', + '-tune', 'zerolatency', + '-r', '30', + '-s', '1280x720', + '-f', 'rtsp', + f'rtsp://localhost:{rtsp_port}/stream' + ] + else: # macOS + cmd = [ + 'ffmpeg', + '-f', 'avfoundation', + '-i', f'{webcam_index}:', + '-c:v', 'libx264', + '-preset', 'veryfast', + '-tune', 'zerolatency', + '-r', '30', + '-s', '1280x720', + '-f', 'rtsp', + f'rtsp://localhost:{rtsp_port}/stream' + ] + + logger.info(f"Starting RTSP stream on rtsp://localhost:{rtsp_port}/stream") + logger.info(f"FFmpeg command: {' '.join(cmd)}") + + rtsp_process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + + # Give FFmpeg a moment to start + time.sleep(2) + + # Check if process is still running + if rtsp_process.poll() is None: + logger.info("RTSP streaming started successfully") + return rtsp_process + else: + # Get error output if process failed + stdout, stderr = rtsp_process.communicate(timeout=2) + logger.error("RTSP streaming failed to start") + logger.error(f"FFmpeg stdout: {stdout}") + logger.error(f"FFmpeg stderr: {stderr}") + return None + + except Exception as e: + logger.error(f"Failed to start RTSP stream: {e}") + return None + +def get_local_ip(): + """Get the Wireguard IP address for external access""" + # Use Wireguard IP for external access + return "10.101.1.4" + +def main(): + global webcam_cap, rtsp_process + + # Configuration - Force use index 0 for Logitech HD webcam + webcam_index = 0 # Logitech HD webcam C930c (1920x1080@30fps) + http_port = 8080 + rtsp_port = 8554 + + logger.info("=== Webcam RTSP & HTTP Server ===") + + # Initialize webcam + logger.info("Initializing webcam...") + webcam_cap = cv2.VideoCapture(webcam_index) + + if not webcam_cap.isOpened(): + logger.error(f"Failed to open webcam at index {webcam_index}") + logger.info("Try different webcam indices (0, 1, 2, etc.)") + return + + # Set webcam properties - Use high resolution for Logitech HD webcam + webcam_cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1920) + webcam_cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 1080) + webcam_cap.set(cv2.CAP_PROP_FPS, 30) + + width = int(webcam_cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + height = int(webcam_cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + fps = webcam_cap.get(cv2.CAP_PROP_FPS) + + logger.info(f"Webcam initialized: {width}x{height} @ {fps}fps") + + # Get local IP for CMS configuration + local_ip = get_local_ip() + + # Start RTSP streaming (optional, requires FFmpeg) + rtsp_process = start_rtsp_stream(webcam_index, rtsp_port) + + # Start HTTP server for snapshots + server_address = ('0.0.0.0', http_port) # Bind to all interfaces + http_server = HTTPServer(server_address, WebcamHTTPHandler) + + logger.info("\n=== Server URLs for CMS Configuration ===") + logger.info(f"HTTP Snapshot URL: http://{local_ip}:{http_port}/snapshot") + + if rtsp_process: + logger.info(f"RTSP Stream URL: rtsp://{local_ip}:{rtsp_port}/stream") + else: + logger.info("RTSP Stream: Not available (FFmpeg not found)") + logger.info("HTTP-only mode: Use Snapshot URL for camera input") + + logger.info(f"Status URL: http://{local_ip}:{http_port}/status") + logger.info("\n=== CMS Configuration Suggestions ===") + logger.info(f"Camera Identifier: webcam-local-01") + logger.info(f"RTSP Stream URL: rtsp://{local_ip}:{rtsp_port}/stream") + logger.info(f"Snapshot URL: http://{local_ip}:{http_port}/snapshot") + logger.info(f"Snapshot Interval: 2000 (ms)") + logger.info("\nPress Ctrl+C to stop all servers") + + try: + # Start HTTP server + http_server.serve_forever() + except KeyboardInterrupt: + logger.info("Shutting down servers...") + finally: + # Clean up + if webcam_cap: + webcam_cap.release() + + if rtsp_process: + logger.info("Stopping RTSP stream...") + rtsp_process.terminate() + try: + rtsp_process.wait(timeout=5) + except subprocess.TimeoutExpired: + rtsp_process.kill() + + http_server.server_close() + logger.info("All servers stopped") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/worker.md b/worker.md index 72c5e69..c485db5 100644 --- a/worker.md +++ b/worker.md @@ -15,86 +15,9 @@ Communication is bidirectional and asynchronous. All messages are JSON objects w - **Worker -> Backend:** You will send messages to the backend to report status, forward detection events, or request changes to session data. - **Backend -> Worker:** The backend will send commands to you to manage camera subscriptions. -### 2.1. Multi-Process Cluster Architecture +## 3. Dynamic Configuration via MPTA File -The backend uses a sophisticated multi-process cluster architecture with Redis-based coordination to manage worker connections at scale: - -**Redis Communication Channels:** - -- `worker:commands` - Commands sent TO workers (subscribe, unsubscribe, setSessionId, setProgressionStage) -- `worker:responses` - Detection responses and state reports FROM workers -- `worker:events` - Worker lifecycle events (connection, disconnection, health status) - -**Distributed State Management:** - -- `worker:states` - Redis hash map storing real-time worker performance metrics and connection status -- `worker:assignments` - Redis hash map tracking camera-to-worker assignments across the cluster -- `worker:owners` - Redis key-based worker ownership leases with 30-second TTL for automatic failover - -**Load Balancing & Failover:** - -- **Assignment Algorithm**: Workers are assigned based on subscription count and CPU usage -- **Distributed Locking**: Assignment operations use Redis locks to prevent race conditions -- **Automatic Failover**: Orphaned workers are detected via lease expiration and automatically reclaimed -- **Horizontal Scaling**: New backend processes automatically join the cluster and participate in load balancing - -**Inter-Process Coordination:** - -- Each backend process maintains local WebSocket connections with workers -- Commands are routed via Redis pub/sub to the process that owns the target worker connection -- Master election ensures coordinated cluster management and prevents split-brain scenarios -- Process identification uses UUIDs for clean process tracking and ownership management - -## 3. Message Types and Command Structure - -All worker communication follows a standardized message structure with the following command types: - -**Commands from Backend to Worker:** - -- `setSubscriptionList` - Set complete list of camera subscriptions for declarative state management -- `setSessionId` - Associate a session ID with a display for detection linking -- `setProgressionStage` - Update the progression stage for context-aware processing -- `requestState` - Request immediate state report from worker -- `patchSessionResult` - Response to worker's patch session request - -**Messages from Worker to Backend:** - -- `stateReport` - Periodic heartbeat with performance metrics and subscription status -- `imageDetection` - Real-time detection results with timestamp and data -- `patchSession` - Request to modify display persistent session data - -**Command Structure:** - -```typescript -interface WorkerCommand { - type: string; - subscriptions?: SubscriptionObject[]; // For setSubscriptionList - payload?: { - displayIdentifier?: string; - sessionId?: number | null; - progressionStage?: string | null; - // Additional payload fields based on command type - }; -} - -interface SubscriptionObject { - subscriptionIdentifier: string; // Format: "displayId;cameraId" - rtspUrl: string; - snapshotUrl?: string; - snapshotInterval?: number; // milliseconds - modelUrl: string; // Fresh pre-signed URL (1-hour TTL) - modelId: number; - modelName: string; - cropX1?: number; - cropY1?: number; - cropX2?: number; - cropY2?: number; -} -``` - -## 4. Dynamic Configuration via MPTA File - -To enable modularity and dynamic configuration, the backend will send you a URL to a `.mpta` file in each subscription within the `setSubscriptionList` command. This file is a renamed `.zip` archive that contains everything your worker needs to perform its task. +To enable modularity and dynamic configuration, the backend will send you a URL to a `.mpta` file when it issues a `subscribe` command. This file is a renamed `.zip` archive that contains everything your worker needs to perform its task. **Your worker is responsible for:** @@ -111,66 +34,11 @@ To enable modularity and dynamic configuration, the backend will send you a URL Essentially, the `.mpta` file is a self-contained package that tells your worker _how_ to process the video stream for a given subscription. -## 5. Worker State Recovery and Reconnection - -The system provides comprehensive state recovery mechanisms to ensure seamless operation across worker disconnections and backend restarts. - -### 5.1. Automatic Resubscription - -**Connection Recovery Flow:** - -1. **Connection Detection**: Backend detects worker reconnection via WebSocket events -2. **State Restoration**: All subscription states are restored from backend memory and Redis -3. **Fresh Model URLs**: New model URLs are generated to handle S3 URL expiration -4. **Session Recovery**: Session IDs and progression stages are automatically restored -5. **Heartbeat Resumption**: Worker immediately begins sending state reports - -### 5.2. State Persistence Architecture - -**Backend State Storage:** - -- **Local State**: Each backend process maintains `DetectorWorkerState` with active subscriptions -- **Redis Coordination**: Assignment mappings stored in `worker:assignments` Redis hash -- **Session Tracking**: Display session IDs tracked in display persistent data -- **Progression Stages**: Current stages maintained in display controllers - -**Recovery Guarantees:** - -- **Zero Configuration Loss**: All subscription parameters are preserved across disconnections -- **Session Continuity**: Active sessions remain linked after worker reconnection -- **Stage Synchronization**: Progression stages are immediately synchronized on reconnection -- **Model Availability**: Fresh model URLs ensure continuous access to detection models - -### 5.3. Heartbeat and Health Monitoring - -**Health Check Protocol:** - -- **Heartbeat Interval**: Workers send `stateReport` every 2 seconds -- **Timeout Detection**: Backend marks workers offline after 10-second timeout -- **Automatic Recovery**: Offline workers are automatically rescheduled when they reconnect -- **Performance Tracking**: CPU, memory, and GPU usage monitored for load balancing - -**Failure Scenarios:** - -- **Worker Crash**: Subscriptions are reassigned to other available workers -- **Network Interruption**: Automatic reconnection with full state restoration -- **Backend Restart**: Worker assignments are restored from Redis state -- **Redis Failure**: Local state provides temporary operation until Redis recovers - -### 5.4. Multi-Process Coordination - -**Ownership and Leasing:** - -- **Worker Ownership**: Each worker is owned by a single backend process via Redis lease -- **Lease Renewal**: 30-second TTL leases automatically renewed by owning process -- **Orphan Detection**: Expired leases allow worker reassignment to active processes -- **Graceful Handover**: Clean ownership transfer during process shutdown - -## 6. Messages from Worker to Backend +## 4. Messages from Worker to Backend These are the messages your worker is expected to send to the backend. -### 6.1. State Report (Heartbeat) +### 4.1. State Report (Heartbeat) This message is crucial for the backend to monitor your worker's health and status, including GPU usage. @@ -205,7 +73,7 @@ This message is crucial for the backend to monitor your worker's health and stat > > - `cropX1`, `cropY1`, `cropX2`, `cropY2` (optional, integer) should be included in each camera connection to indicate the crop coordinates for that subscription. -### 6.2. Image Detection +### 4.2. Image Detection Sent when the worker detects a relevant object. The `detection` object should be flat and contain key-value pairs corresponding to the detected attributes. @@ -233,7 +101,7 @@ Sent when the worker detects a relevant object. The `detection` object should be } ``` -### 6.3. Patch Session +### 4.3. Patch Session > **Note:** Patch messages are only used when the worker can't keep up and needs to retroactively send detections. Normally, detections should be sent in real-time using `imageDetection` messages. Use `patchSession` only to update session data after the fact. @@ -302,91 +170,68 @@ interface DisplayPersistentData { - **`null`** values will set the corresponding field to `null`. - Nested objects are merged recursively. -## 7. Commands from Backend to Worker +## 5. Commands from Backend to Worker -These are the commands your worker will receive from the backend. The subscription system uses a **fully declarative approach** with `setSubscriptionList` - the backend sends the complete desired subscription list, and workers handle reconciliation internally. +These are the commands your worker will receive from the backend. -### 7.1. Set Subscription List (Declarative Subscriptions) +### 5.1. Subscribe to Camera -**The primary subscription command that replaces individual subscribe/unsubscribe operations.** +Instructs the worker to process a camera's RTSP stream using the configuration from the specified `.mpta` file. -Instructs the worker to process the complete list of camera streams. The worker must reconcile this list with its current subscriptions, adding new ones, removing obsolete ones, and updating existing ones as needed. - -- **Type:** `setSubscriptionList` +- **Type:** `subscribe` **Payload:** ```json { - "type": "setSubscriptionList", - "subscriptions": [ - { - "subscriptionIdentifier": "display-001;cam-001", - "rtspUrl": "rtsp://user:pass@host:port/stream1", - "snapshotUrl": "http://go2rtc/snapshot/1", - "snapshotInterval": 5000, - "modelUrl": "http://storage/models/us-lpr.mpta?token=fresh-token", - "modelName": "US-LPR-and-Vehicle-ID", - "modelId": 102, - "cropX1": 100, - "cropY1": 200, - "cropX2": 300, - "cropY2": 400 - }, - { - "subscriptionIdentifier": "display-002;cam-001", - "rtspUrl": "rtsp://user:pass@host:port/stream1", - "snapshotUrl": "http://go2rtc/snapshot/1", - "snapshotInterval": 5000, - "modelUrl": "http://storage/models/vehicle-detect.mpta?token=fresh-token", - "modelName": "Vehicle Detection", - "modelId": 201, - "cropX1": 0, - "cropY1": 0, - "cropX2": 1920, - "cropY2": 1080 - } - ] + "type": "subscribe", + "payload": { + "subscriptionIdentifier": "display-001;cam-002", + "rtspUrl": "rtsp://user:pass@host:port/stream", + "snapshotUrl": "http://go2rtc/snapshot/1", + "snapshotInterval": 5000, + "modelUrl": "http://storage/models/us-lpr.mpta", + "modelName": "US-LPR-and-Vehicle-ID", + "modelId": 102, + "cropX1": 100, + "cropY1": 200, + "cropX2": 300, + "cropY2": 400 + } } ``` -**Declarative Subscription Behavior:** - -- **Complete State Definition**: The backend sends the complete desired subscription list for this worker -- **Worker-Side Reconciliation**: Workers compare the new list with current subscriptions and handle differences -- **Fresh Model URLs**: Each command includes fresh pre-signed S3 URLs (1-hour TTL) for ML models -- **Load Balancing**: The backend intelligently distributes subscriptions across available workers -- **State Recovery**: Complete subscription list is sent on worker reconnection - -**Worker Reconciliation Responsibility:** - -When receiving a `setSubscriptionList` command, your worker must: - -1. **Compare with Current State**: Identify new subscriptions, removed subscriptions, and updated subscriptions -2. **Add New Subscriptions**: Start processing new camera streams with the provided configuration -3. **Remove Obsolete Subscriptions**: Stop processing camera streams not in the new list -4. **Update Existing Subscriptions**: Handle configuration changes (model updates, crop coordinates, etc.) -5. **Maintain Single Streams**: Ensure only one RTSP stream per camera, even with multiple display bindings -6. **Report Final State**: Send updated `stateReport` confirming the actual subscription state - > **Note:** > -> - `cropX1`, `cropY1`, `cropX2`, `cropY2` (optional, integer) specify the crop coordinates for the camera stream -> - `snapshotUrl` and `snapshotInterval` (optional) enable periodic snapshot capture -> - Multiple subscriptions may share the same `rtspUrl` but have different `subscriptionIdentifier` values +> - `cropX1`, `cropY1`, `cropX2`, `cropY2` (optional, integer) specify the crop coordinates for the camera stream. These values are configured per display and passed in the subscription payload. If not provided, the worker should process the full frame. > -> **Camera Stream Optimization:** -> When multiple subscriptions share the same camera (same `rtspUrl`), your worker must: +> **Important:** +> If multiple displays are bound to the same camera, your worker must ensure that only **one stream** is opened per camera. When you receive multiple subscriptions for the same camera (with different `subscriptionIdentifier` values), you should: > -> - Open the RTSP stream **once** for that camera -> - Capture each frame/snapshot **once** per cycle -> - Process the shared stream for each subscription's requirements (crop coordinates, model) -> - Route detection results separately for each `subscriptionIdentifier` -> - Apply display-specific crop coordinates during processing -> -> This optimization reduces bandwidth usage and ensures consistent detection timing across displays. +> - Open the RTSP stream **once** for that camera if using RTSP. +> - Capture each snapshot only once per cycle, and reuse it for all display subscriptions sharing that camera. +> - Capture each frame/image only once per cycle. +> - Reuse the same captured image and snapshot for all display subscriptions that share the camera, processing and routing detection results separately for each display as needed. +> This avoids unnecessary load and bandwidth usage, and ensures consistent detection results and snapshots across all displays sharing the same camera. -### 7.2. Request State +### 5.2. Unsubscribe from Camera + +Instructs the worker to stop processing a camera's stream. + +- **Type:** `unsubscribe` + +**Payload:** + +```json +{ + "type": "unsubscribe", + "payload": { + "subscriptionIdentifier": "display-001;cam-002" + } +} +``` + +### 5.3. Request State Direct request for the worker's current state. Respond with a `stateReport` message. @@ -400,7 +245,7 @@ Direct request for the worker's current state. Respond with a `stateReport` mess } ``` -### 7.3. Patch Session Result +### 5.4. Patch Session Result Backend's response to a `patchSession` message. @@ -419,11 +264,9 @@ Backend's response to a `patchSession` message. } ``` -### 7.4. Set Session ID +### 5.5. Set Session ID -**Real-time session association for linking detection events to user sessions.** - -Allows the backend to instruct the worker to associate a session ID with a display. This enables linking detection events to specific user sessions. The system automatically propagates session changes across all worker processes via Redis pub/sub. +Allows the backend to instruct the worker to associate a session ID with a subscription. This is useful for linking detection events to a specific session. The session ID can be `null` to indicate no active session. - **Type:** `setSessionId` @@ -451,94 +294,11 @@ Or to clear the session: } ``` -**Session Management Flow:** +> **Note:** +> +> - The worker should store the session ID for the given subscription and use it in subsequent detection or patch messages as appropriate. If `sessionId` is `null`, the worker should treat the subscription as having no active session. -1. **Session Creation**: When a new session is created (user interaction), the backend immediately sends `setSessionId` to all relevant workers -2. **Cross-Process Distribution**: The command is distributed across multiple backend processes via Redis `worker:commands` channel -3. **Worker State Synchronization**: Workers maintain session IDs for each display and apply them to all matching subscriptions -4. **Automatic Recovery**: Session IDs are restored when workers reconnect, ensuring no session context is lost -5. **Multi-Subscription Support**: A single session ID applies to all camera subscriptions for the given display - -**Worker Responsibility:** - -- Store the session ID for the given `displayIdentifier` -- Apply the session ID to **all active subscriptions** that start with `displayIdentifier;` (e.g., `display-001;cam-001`, `display-001;cam-002`) -- Include the session ID in subsequent `imageDetection` and `patchSession` messages -- Handle session clearing when `sessionId` is `null` -- Restore session IDs from backend state after reconnection - -**Multi-Process Coordination:** - -The session ID command uses the distributed worker communication system: - -- Commands are routed via Redis pub/sub to the process managing the target worker -- Automatic failover ensures session updates reach workers even during process changes -- Lease-based worker ownership prevents duplicate session notifications - -### 7.5. Set Progression Stage - -**Real-time progression stage synchronization for dynamic content adaptation.** - -Notifies workers about the current progression stage of a display, enabling context-aware content selection and detection behavior. The system automatically tracks stage changes and avoids redundant updates. - -- **Type:** `setProgressionStage` - -**Payload:** - -```json -{ - "type": "setProgressionStage", - "payload": { - "displayIdentifier": "display-001", - "progressionStage": "car_fueling" - } -} -``` - -Or to clear the progression stage: - -```json -{ - "type": "setProgressionStage", - "payload": { - "displayIdentifier": "display-001", - "progressionStage": null - } -} -``` - -**Available Progression Stages:** - -- `"welcome"` - Initial state, awaiting user interaction -- `"car_fueling"` - Vehicle is actively fueling -- `"car_waitpayment"` - Fueling complete, awaiting payment -- `"car_postpayment"` - Payment completed, transaction finishing -- `null` - No active progression stage - -**Progression Stage Flow:** - -1. **Automatic Detection**: Display controllers automatically detect progression stage changes based on display persistent data -2. **Change Filtering**: The system compares current stage with last sent stage to avoid redundant updates -3. **Instant Propagation**: Stage changes are immediately sent to all workers associated with the display -4. **Cross-Process Distribution**: Commands are distributed via Redis `worker:commands` channel to all backend processes -5. **State Recovery**: Progression stages are restored when workers reconnect - -**Worker Responsibility:** - -- Store the progression stage for the given `displayIdentifier` -- Apply the stage to **all active subscriptions** for that display -- Use progression stage for context-aware detection and content adaptation -- Handle stage clearing when `progressionStage` is `null` -- Restore progression stages from backend state after reconnection - -**Use Cases:** - -- **Fuel Station Displays**: Adapt content based on fueling progress (welcome ads vs. payment prompts) -- **Dynamic Detection**: Adjust detection sensitivity based on interaction stage -- **Content Personalization**: Select appropriate advertisements for current user journey stage -- **Analytics**: Track user progression through interaction stages - -## 8. Subscription Identifier Format +## Subscription Identifier Format The `subscriptionIdentifier` used in all messages is constructed as: @@ -557,11 +317,11 @@ When the backend sends a `setSessionId` command, it will only provide the `displ - The worker must match the `displayIdentifier` to all active subscriptions for that display (i.e., all `subscriptionIdentifier` values that start with `displayIdentifier;`). - The worker should set or clear the session ID for all matching subscriptions. -## 9. Example Communication Log +## 6. Example Communication Log -This section shows a typical sequence of messages between the backend and the worker, including the new declarative subscription model, session ID management, and progression stage synchronization. +This section shows a typical sequence of messages between the backend and the worker. Patch messages are not included, as they are only used when the worker cannot keep up. -> **Note:** Unsubscribe is triggered during load rebalancing or when displays/cameras are removed from the system. The system automatically handles worker reconnection with full state recovery. +> **Note:** Unsubscribe is triggered when a user removes a camera or when the node is too heavily loaded and needs rebalancing. 1. **Connection Established** & **Heartbeat** - **Worker -> Backend** @@ -575,24 +335,21 @@ This section shows a typical sequence of messages between the backend and the wo "cameraConnections": [] } ``` -2. **Backend Sets Subscription List** +2. **Backend Subscribes Camera** - **Backend -> Worker** ```json { - "type": "setSubscriptionList", - "subscriptions": [ - { - "subscriptionIdentifier": "display-001;entry-cam-01", - "rtspUrl": "rtsp://192.168.1.100/stream1", - "modelUrl": "http://storage/models/vehicle-id.mpta?token=fresh-token", - "modelName": "Vehicle Identification", - "modelId": 201, - "snapshotInterval": 5000 - } - ] + "type": "subscribe", + "payload": { + "subscriptionIdentifier": "display-001;entry-cam-01", + "rtspUrl": "rtsp://192.168.1.100/stream1", + "modelUrl": "http://storage/models/vehicle-id.mpta", + "modelName": "Vehicle Identification", + "modelId": 201 + } } ``` -3. **Worker Acknowledges with Reconciled State** +3. **Worker Acknowledges in Heartbeat** - **Worker -> Backend** ```json { @@ -611,44 +368,13 @@ This section shows a typical sequence of messages between the backend and the wo ] } ``` -4. **Backend Sets Session ID** - - - **Backend -> Worker** - - ```json - { - "type": "setSessionId", - "payload": { - "displayIdentifier": "display-001", - "sessionId": 12345 - } - } - ``` - -5. **Backend Sets Progression Stage** - - - **Backend -> Worker** - - ```json - { - "type": "setProgressionStage", - "payload": { - "displayIdentifier": "display-001", - "progressionStage": "welcome" - } - } - ``` - -6. **Worker Detects a Car with Session Context** - +4. **Worker Detects a Car** - **Worker -> Backend** - ```json { "type": "imageDetection", "subscriptionIdentifier": "display-001;entry-cam-01", "timestamp": "2025-07-15T10:00:00.000Z", - "sessionId": 12345, "data": { "detection": { "carBrand": "Honda", @@ -662,89 +388,56 @@ This section shows a typical sequence of messages between the backend and the wo } } ``` - -7. **Progression Stage Change** - + - **Worker -> Backend** + ```json + { + "type": "imageDetection", + "subscriptionIdentifier": "display-001;entry-cam-01", + "timestamp": "2025-07-15T10:00:01.000Z", + "data": { + "detection": { + "carBrand": "Toyota", + "carModel": "Corolla", + "bodyType": "Sedan", + "licensePlateText": "CMS-1234", + "licensePlateConfidence": 0.97 + }, + "modelId": 201, + "modelName": "Vehicle Identification" + } + } + ``` + - **Worker -> Backend** + ```json + { + "type": "imageDetection", + "subscriptionIdentifier": "display-001;entry-cam-01", + "timestamp": "2025-07-15T10:00:02.000Z", + "data": { + "detection": { + "carBrand": "Ford", + "carModel": "Focus", + "bodyType": "Hatchback", + "licensePlateText": "CMS-5678", + "licensePlateConfidence": 0.96 + }, + "modelId": 201, + "modelName": "Vehicle Identification" + } + } + ``` +5. **Backend Unsubscribes Camera** - **Backend -> Worker** - ```json { - "type": "setProgressionStage", + "type": "unsubscribe", "payload": { - "displayIdentifier": "display-001", - "progressionStage": "car_fueling" + "subscriptionIdentifier": "display-001;entry-cam-01" } } ``` - -8. **Worker Reconnection with State Recovery** - - - **Worker Disconnects and Reconnects** - - **Worker -> Backend** (Immediate heartbeat after reconnection) - - ```json - { - "type": "stateReport", - "cpuUsage": 70.0, - "memoryUsage": 38.0, - "gpuUsage": 55.0, - "gpuMemoryUsage": 20.0, - "cameraConnections": [] - } - ``` - - - **Backend -> Worker** (Automatic subscription list restoration with fresh model URLs) - - ```json - { - "type": "setSubscriptionList", - "subscriptions": [ - { - "subscriptionIdentifier": "display-001;entry-cam-01", - "rtspUrl": "rtsp://192.168.1.100/stream1", - "modelUrl": "http://storage/models/vehicle-id.mpta?token=fresh-reconnect-token", - "modelName": "Vehicle Identification", - "modelId": 201, - "snapshotInterval": 5000 - } - ] - } - ``` - - - **Backend -> Worker** (Session ID recovery) - - ```json - { - "type": "setSessionId", - "payload": { - "displayIdentifier": "display-001", - "sessionId": 12345 - } - } - ``` - - - **Backend -> Worker** (Progression stage recovery) - - ```json - { - "type": "setProgressionStage", - "payload": { - "displayIdentifier": "display-001", - "progressionStage": "car_fueling" - } - } - ``` - -9. **Backend Updates Subscription List** (Load balancing or system cleanup) - - **Backend -> Worker** (Empty list removes all subscriptions) - ```json - { - "type": "setSubscriptionList", - "subscriptions": [] - } - ``` -10. **Worker Acknowledges Subscription Removal** - - **Worker -> Backend** (Updated heartbeat showing no active connections after reconciliation) +6. **Worker Acknowledges Unsubscription** + - **Worker -> Backend** ```json { "type": "stateReport", @@ -756,17 +449,7 @@ This section shows a typical sequence of messages between the backend and the wo } ``` -**Key Improvements in Communication Flow:** - -1. **Fully Declarative Subscriptions**: Complete subscription list sent in single command, worker handles reconciliation -2. **Worker-Side Reconciliation**: Workers compare desired vs. current state and make necessary changes internally -3. **Session Context**: All detection events include session IDs for proper user linking -4. **Progression Stages**: Real-time stage updates enable context-aware content selection -5. **State Recovery**: Complete automatic recovery of subscription lists, session IDs, and progression stages -6. **Fresh Model URLs**: S3 URL expiration is handled transparently with 1-hour TTL tokens -7. **Load Balancing**: Backend intelligently distributes complete subscription lists across available workers - -## 10. HTTP API: Image Retrieval +## 7. HTTP API: Image Retrieval In addition to the WebSocket protocol, the worker exposes an HTTP endpoint for retrieving the latest image frame from a camera.