python-detector-worker/siwatsystem/pympta.py

import os
import json
import logging
import torch
import cv2
import zipfile
import shutil
import traceback
import redis
import time
import uuid
import concurrent.futures
from ultralytics import YOLO
from urllib.parse import urlparse
from .database import DatabaseManager
from datetime import datetime

# Create a logger specifically for this module
logger = logging.getLogger("detector_worker.pympta")

# Global camera-aware stability tracking
# Structure: {camera_id: {model_id: {"track_stability_counters": {track_id: count}, "stable_tracks": set(), "session_state": {...}}}}
_camera_stability_tracking = {}

# Session timeout configuration (waiting for backend sessionId)
_session_timeout_seconds = 15

def validate_redis_config(redis_config: dict) -> bool:
    """Validate Redis configuration parameters."""
    required_fields = ["host", "port"]
    for field in required_fields:
        if field not in redis_config:
            logger.error(f"Missing required Redis config field: {field}")
            return False
    
    if not isinstance(redis_config["port"], int) or redis_config["port"] <= 0:
        logger.error(f"Invalid Redis port: {redis_config['port']}")
        return False
    
    return True

def validate_postgresql_config(pg_config: dict) -> bool:
    """Validate PostgreSQL configuration parameters."""
    required_fields = ["host", "port", "database", "username", "password"]
    for field in required_fields:
        if field not in pg_config:
            logger.error(f"Missing required PostgreSQL config field: {field}")
            return False
    
    if not isinstance(pg_config["port"], int) or pg_config["port"] <= 0:
        logger.error(f"Invalid PostgreSQL port: {pg_config['port']}")
        return False
    
    return True

def crop_region_by_class(frame, regions_dict, class_name):
    """Crop a specific region from frame based on detected class."""
    if class_name not in regions_dict:
        logger.warning(f"Class '{class_name}' not found in detected regions")
        return None
    
    bbox = regions_dict[class_name]['bbox']
    x1, y1, x2, y2 = bbox
    
    # Diagnostic logging for crop issues
    frame_h, frame_w = frame.shape[:2]
    logger.debug(f"CROP DEBUG: Frame dimensions: {frame_w}x{frame_h}")
    logger.debug(f"CROP DEBUG: Original bbox: {bbox}")
    logger.debug(f"CROP DEBUG: Bbox dimensions: {x2-x1}x{y2-y1}")
    
    # Check if bbox is within frame bounds
    if x1 < 0 or y1 < 0 or x2 > frame_w or y2 > frame_h:
        logger.warning(f"CROP DEBUG: Bbox extends beyond frame! Clipping...")
        x1, y1 = max(0, x1), max(0, y1)
        x2, y2 = min(frame_w, x2), min(frame_h, y2)
        logger.debug(f"CROP DEBUG: Clipped bbox: ({x1}, {y1}, {x2}, {y2})")
    
    cropped = frame[y1:y2, x1:x2]
    
    if cropped.size == 0:
        logger.warning(f"Empty crop for class '{class_name}' with bbox {bbox}")
        return None
    
    logger.debug(f"CROP DEBUG: Successful crop shape: {cropped.shape}")
    return cropped

def format_action_context(base_context, additional_context=None):
    """Format action context with dynamic values."""
    context = {**base_context}
    if additional_context:
        context.update(additional_context)
    return context

def load_pipeline_node(node_config: dict, mpta_dir: str, redis_client, db_manager=None) -> dict:
    # Recursively load a model node from configuration.
    model_path = os.path.join(mpta_dir, node_config["modelFile"])
    if not os.path.exists(model_path):
        logger.error(f"Model file {model_path} not found. Current directory: {os.getcwd()}")
        logger.error(f"Directory content: {os.listdir(os.path.dirname(model_path))}")
        raise FileNotFoundError(f"Model file {model_path} not found.")
    logger.info(f"Loading model for node {node_config['modelId']} from {model_path}")
    model = YOLO(model_path)
    if torch.cuda.is_available():
        logger.info(f"CUDA available. Moving model {node_config['modelId']} to GPU VRAM")
        model.to("cuda")
    else:
        logger.info(f"CUDA not available. Using CPU for model {node_config['modelId']}")

    # Prepare trigger class indices for optimization
    trigger_classes = node_config.get("triggerClasses", [])
    trigger_class_indices = None
    if trigger_classes and hasattr(model, "names"):
        # Convert class names to indices for the model
        trigger_class_indices = [i for i, name in model.names.items() 
                                if name in trigger_classes]
        logger.debug(f"Converted trigger classes to indices: {trigger_class_indices}")

    # Extract stability threshold from main pipeline config (not tracking config)
    tracking_config = node_config.get("tracking", {"enabled": True, "reidConfigPath": "botsort.yaml"})
    stability_threshold = node_config.get("stabilityThreshold", 4)  # Read from main config, default to 4

    node = {
        "modelId": node_config["modelId"],
        "modelFile": node_config["modelFile"],
        "triggerClasses": trigger_classes,
        "triggerClassIndices": trigger_class_indices,
        "classMapping": node_config.get("classMapping", {}),
        "crop": node_config.get("crop", False),
        "cropClass": node_config.get("cropClass"),
        "minConfidence": node_config.get("minConfidence", None),
        "frontalMinConfidence": node_config.get("frontalMinConfidence", None),
        "minBboxAreaRatio": node_config.get("minBboxAreaRatio", 0.0),
        "multiClass": node_config.get("multiClass", False),
        "expectedClasses": node_config.get("expectedClasses", []),
        "parallel": node_config.get("parallel", False),
        "actions": node_config.get("actions", []),
        "parallelActions": node_config.get("parallelActions", []),
        "tracking": tracking_config,
        "stabilityThreshold": stability_threshold,
        "model": model,
        "branches": [],
        "redis_client": redis_client,
        "db_manager": db_manager
    }
    logger.debug(f"Configured node {node_config['modelId']} with trigger classes: {node['triggerClasses']}")
    for child in node_config.get("branches", []):
        logger.debug(f"Loading branch for parent node {node_config['modelId']}")
        node["branches"].append(load_pipeline_node(child, mpta_dir, redis_client, db_manager))
    return node

def load_pipeline_from_zip(zip_source: str, target_dir: str) -> dict:
    logger.info(f"Attempting to load pipeline from {zip_source} to {target_dir}")
    os.makedirs(target_dir, exist_ok=True)
    zip_path = os.path.join(target_dir, "pipeline.mpta")
    
    # Parse the source; only local files are supported here.
    parsed = urlparse(zip_source)
    if parsed.scheme in ("", "file"):
        local_path = parsed.path if parsed.scheme == "file" else zip_source
        logger.debug(f"Checking if local file exists: {local_path}")
        if os.path.exists(local_path):
            try:
                shutil.copy(local_path, zip_path)
                logger.info(f"Copied local .mpta file from {local_path} to {zip_path}")
            except Exception as e:
                logger.error(f"Failed to copy local .mpta file from {local_path}: {str(e)}", exc_info=True)
                return None
        else:
            logger.error(f"Local file {local_path} does not exist. Current directory: {os.getcwd()}")
            # List all subdirectories of models directory to help debugging
            if os.path.exists("models"):
                logger.error(f"Content of models directory: {os.listdir('models')}")
                for root, dirs, files in os.walk("models"):
                    logger.error(f"Directory {root} contains subdirs: {dirs} and files: {files}")
            else:
                logger.error("The models directory doesn't exist")
            return None
    else:
        logger.error(f"HTTP download functionality has been moved. Use a local file path here. Received: {zip_source}")
        return None

    try:
        if not os.path.exists(zip_path):
            logger.error(f"Zip file not found at expected location: {zip_path}")
            return None
            
        logger.debug(f"Extracting .mpta file from {zip_path} to {target_dir}")
        # Extract contents and track the directories created
        extracted_dirs = []
        with zipfile.ZipFile(zip_path, "r") as zip_ref:
            file_list = zip_ref.namelist()
            logger.debug(f"Files in .mpta archive: {file_list}")
            
            # Extract and track the top-level directories
            for file_path in file_list:
                parts = file_path.split('/')
                if len(parts) > 1:
                    top_dir = parts[0]
                    if top_dir and top_dir not in extracted_dirs:
                        extracted_dirs.append(top_dir)
            
            # Now extract the files
            zip_ref.extractall(target_dir)
            
        logger.info(f"Successfully extracted .mpta file to {target_dir}")
        logger.debug(f"Extracted directories: {extracted_dirs}")
        
        # Check what was actually created after extraction
        actual_dirs = [d for d in os.listdir(target_dir) if os.path.isdir(os.path.join(target_dir, d))]
        logger.debug(f"Actual directories created: {actual_dirs}")
    except zipfile.BadZipFile as e:
        logger.error(f"Bad zip file {zip_path}: {str(e)}", exc_info=True)
        return None
    except Exception as e:
        logger.error(f"Failed to extract .mpta file {zip_path}: {str(e)}", exc_info=True)
        return None
    finally:
        if os.path.exists(zip_path):
            os.remove(zip_path)
            logger.debug(f"Removed temporary zip file: {zip_path}")

    # Use the first extracted directory if it exists, otherwise use the expected name
    pipeline_name = os.path.basename(zip_source)
    pipeline_name = os.path.splitext(pipeline_name)[0]
    
    # Find the directory with pipeline.json
    mpta_dir = None
    # First try the expected directory name
    expected_dir = os.path.join(target_dir, pipeline_name)
    if os.path.exists(expected_dir) and os.path.exists(os.path.join(expected_dir, "pipeline.json")):
        mpta_dir = expected_dir
        logger.debug(f"Found pipeline.json in the expected directory: {mpta_dir}")
    else:
        # Look through all subdirectories for pipeline.json
        for subdir in actual_dirs:
            potential_dir = os.path.join(target_dir, subdir)
            if os.path.exists(os.path.join(potential_dir, "pipeline.json")):
                mpta_dir = potential_dir
                logger.info(f"Found pipeline.json in directory: {mpta_dir} (different from expected: {expected_dir})")
                break
    
    if not mpta_dir:
        logger.error(f"Could not find pipeline.json in any extracted directory. Directory content: {os.listdir(target_dir)}")
        return None
        
    pipeline_json_path = os.path.join(mpta_dir, "pipeline.json")
    if not os.path.exists(pipeline_json_path):
        logger.error(f"pipeline.json not found in the .mpta file. Files in directory: {os.listdir(mpta_dir)}")
        return None

    try:
        with open(pipeline_json_path, "r") as f:
            pipeline_config = json.load(f)
        logger.info(f"Successfully loaded pipeline configuration from {pipeline_json_path}")
        logger.debug(f"Pipeline config: {json.dumps(pipeline_config, indent=2)}")
        
        # Establish Redis connection if configured
        redis_client = None
        if "redis" in pipeline_config:
            redis_config = pipeline_config["redis"]
            if not validate_redis_config(redis_config):
                logger.error("Invalid Redis configuration, skipping Redis connection")
            else:
                try:
                    redis_client = redis.Redis(
                        host=redis_config["host"],
                        port=redis_config["port"],
                        password=redis_config.get("password"),
                        db=redis_config.get("db", 0),
                        decode_responses=True
                    )
                    redis_client.ping()
                    logger.info(f"Successfully connected to Redis at {redis_config['host']}:{redis_config['port']}")
                except redis.exceptions.ConnectionError as e:
                    logger.error(f"Failed to connect to Redis: {e}")
                    redis_client = None
        
        # Establish PostgreSQL connection if configured
        db_manager = None
        if "postgresql" in pipeline_config:
            pg_config = pipeline_config["postgresql"]
            if not validate_postgresql_config(pg_config):
                logger.error("Invalid PostgreSQL configuration, skipping database connection")
            else:
                try:
                    db_manager = DatabaseManager(pg_config)
                    if db_manager.connect():
                        logger.info(f"Successfully connected to PostgreSQL at {pg_config['host']}:{pg_config['port']}")
                    else:
                        logger.error("Failed to connect to PostgreSQL")
                        db_manager = None
                except Exception as e:
                    logger.error(f"Error initializing PostgreSQL connection: {e}")
                    db_manager = None
        
        return load_pipeline_node(pipeline_config["pipeline"], mpta_dir, redis_client, db_manager)
    except json.JSONDecodeError as e:
        logger.error(f"Error parsing pipeline.json: {str(e)}", exc_info=True)
        return None
    except KeyError as e:
        logger.error(f"Missing key in pipeline.json: {str(e)}", exc_info=True)
        return None
    except Exception as e:
        logger.error(f"Error loading pipeline.json: {str(e)}", exc_info=True)
        return None

def execute_actions(node, frame, detection_result, regions_dict=None):
    if not node["redis_client"] or not node["actions"]:
        return

    # Create a dynamic context for this detection event
    from datetime import datetime
    action_context = {
        **detection_result,
        "timestamp_ms": int(time.time() * 1000),
        "uuid": str(uuid.uuid4()),
        "timestamp": datetime.now().strftime("%Y-%m-%dT%H-%M-%S"),
        "filename": f"{uuid.uuid4()}.jpg"
    }

    for action in node["actions"]:
        try:
            if action["type"] == "redis_save_image":
                key = action["key"].format(**action_context)
                
                # Check if we need to crop a specific region
                region_name = action.get("region")
                image_to_save = frame
                
                if region_name and regions_dict:
                    cropped_image = crop_region_by_class(frame, regions_dict, region_name)
                    if cropped_image is not None:
                        image_to_save = cropped_image
                        logger.debug(f"Cropped region '{region_name}' for redis_save_image")
                    else:
                        logger.warning(f"Could not crop region '{region_name}', saving full frame instead")
                
                # Encode image with specified format and quality (default to JPEG)
                img_format = action.get("format", "jpeg").lower()
                quality = action.get("quality", 90)
                
                if img_format == "jpeg":
                    encode_params = [cv2.IMWRITE_JPEG_QUALITY, quality]
                    success, buffer = cv2.imencode('.jpg', image_to_save, encode_params)
                elif img_format == "png":
                    success, buffer = cv2.imencode('.png', image_to_save)
                else:
                    success, buffer = cv2.imencode('.jpg', image_to_save, [cv2.IMWRITE_JPEG_QUALITY, quality])
                
                if not success:
                    logger.error(f"Failed to encode image for redis_save_image")
                    continue
                
                expire_seconds = action.get("expire_seconds")
                if expire_seconds:
                    node["redis_client"].setex(key, expire_seconds, buffer.tobytes())
                    logger.info(f"Saved image to Redis with key: {key} (expires in {expire_seconds}s)")
                else:
                    node["redis_client"].set(key, buffer.tobytes())
                    logger.info(f"Saved image to Redis with key: {key}")
                action_context["image_key"] = key
            elif action["type"] == "redis_publish":
                channel = action["channel"]
                try:
                    # Handle JSON message format by creating it programmatically
                    message_template = action["message"]
                    
                    # Check if the message is JSON-like (starts and ends with braces)
                    if message_template.strip().startswith('{') and message_template.strip().endswith('}'):
                        # Create JSON data programmatically to avoid formatting issues
                        json_data = {}
                        
                        # Add common fields
                        json_data["event"] = "frontal_detected"
                        json_data["display_id"] = action_context.get("display_id", "unknown")
                        json_data["session_id"] = action_context.get("session_id")
                        json_data["timestamp"] = action_context.get("timestamp", "")
                        json_data["image_key"] = action_context.get("image_key", "")
                        
                        # Convert to JSON string
                        message = json.dumps(json_data)
                    else:
                        # Use regular string formatting for non-JSON messages
                        message = message_template.format(**action_context)
                    
                    # Publish to Redis
                    if not node["redis_client"]:
                        logger.error("Redis client is None, cannot publish message")
                        continue
                        
                    # Test Redis connection
                    try:
                        node["redis_client"].ping()
                        logger.debug("Redis connection is active")
                    except Exception as ping_error:
                        logger.error(f"Redis connection test failed: {ping_error}")
                        continue
                    
                    result = node["redis_client"].publish(channel, message)
                    logger.info(f"Published message to Redis channel '{channel}': {message}")
                    logger.info(f"Redis publish result (subscribers count): {result}")
                    
                    # Additional debug info
                    if result == 0:
                        logger.warning(f"No subscribers listening to channel '{channel}'")
                    else:
                        logger.info(f"Message delivered to {result} subscriber(s)")
                    
                except KeyError as e:
                    logger.error(f"Missing key in redis_publish message template: {e}")
                    logger.debug(f"Available context keys: {list(action_context.keys())}")
                except Exception as e:
                    logger.error(f"Error in redis_publish action: {e}")
                    logger.debug(f"Message template: {action['message']}")
                    logger.debug(f"Available context keys: {list(action_context.keys())}")
                    import traceback
                    logger.debug(f"Full traceback: {traceback.format_exc()}")
        except Exception as e:
            logger.error(f"Error executing action {action['type']}: {e}")

def execute_parallel_actions(node, frame, detection_result, regions_dict):
    """Execute parallel actions after all required branches have completed."""
    if not node.get("parallelActions"):
        return
    
    logger.debug("Executing parallel actions...")
    branch_results = detection_result.get("branch_results", {})
    
    for action in node["parallelActions"]:
        try:
            action_type = action.get("type")
            logger.debug(f"Processing parallel action: {action_type}")
            
            if action_type == "postgresql_update_combined":
                # Check if all required branches have completed
                wait_for_branches = action.get("waitForBranches", [])
                missing_branches = [branch for branch in wait_for_branches if branch not in branch_results]
                
                if missing_branches:
                    logger.warning(f"Cannot execute postgresql_update_combined: missing branch results for {missing_branches}")
                    continue
                
                logger.info(f"All required branches completed: {wait_for_branches}")
                
                # Execute the database update
                execute_postgresql_update_combined(node, action, detection_result, branch_results)
            else:
                logger.warning(f"Unknown parallel action type: {action_type}")
                
        except Exception as e:
            logger.error(f"Error executing parallel action {action.get('type', 'unknown')}: {e}")
            import traceback
            logger.debug(f"Full traceback: {traceback.format_exc()}")

def execute_postgresql_update_combined(node, action, detection_result, branch_results):
    """Execute a PostgreSQL update with combined branch results."""
    if not node.get("db_manager"):
        logger.error("No database manager available for postgresql_update_combined action")
        return
        
    try:
        table = action["table"]
        key_field = action["key_field"]
        key_value_template = action["key_value"]
        fields = action["fields"]
        
        # Create context for key value formatting
        action_context = {**detection_result}
        key_value = key_value_template.format(**action_context)
        
        logger.info(f"Executing database update: table={table}, {key_field}={key_value}")
        logger.debug(f"Available branch results: {list(branch_results.keys())}")
        
        # Process field mappings
        mapped_fields = {}
        for db_field, value_template in fields.items():
            try:
                mapped_value = resolve_field_mapping(value_template, branch_results, action_context)
                if mapped_value is not None:
                    mapped_fields[db_field] = mapped_value
                    logger.info(f"Mapped field: {db_field} = {mapped_value}")
                else:
                    logger.warning(f"Could not resolve field mapping for {db_field}: {value_template}")
                    logger.debug(f"Available branch results: {branch_results}")
            except Exception as e:
                logger.error(f"Error mapping field {db_field} with template '{value_template}': {e}")
                import traceback
                logger.debug(f"Field mapping error traceback: {traceback.format_exc()}")
        
        if not mapped_fields:
            logger.warning("No fields mapped successfully, skipping database update")
            logger.debug(f"Branch results available: {branch_results}")
            logger.debug(f"Field templates: {fields}")
            return
            
        # Add updated_at field automatically
        mapped_fields["updated_at"] = "NOW()"
        
        # Execute the database update
        logger.info(f"Attempting database update with fields: {mapped_fields}")
        success = node["db_manager"].execute_update(table, key_field, key_value, mapped_fields)
        
        if success:
            logger.info(f"✅ Successfully updated database: {table} with {len(mapped_fields)} fields")
            logger.info(f"Updated fields: {mapped_fields}")
        else:
            logger.error(f"❌ Failed to update database: {table}")
            logger.error(f"Attempted update with: {key_field}={key_value}, fields={mapped_fields}")
            
    except KeyError as e:
        logger.error(f"Missing required field in postgresql_update_combined action: {e}")
        logger.debug(f"Action config: {action}")
    except Exception as e:
        logger.error(f"Error in postgresql_update_combined action: {e}")
        import traceback
        logger.debug(f"Full traceback: {traceback.format_exc()}")

def resolve_field_mapping(value_template, branch_results, action_context):
    """Resolve field mapping templates like {car_brand_cls_v1.brand}."""
    try:
        logger.debug(f"Resolving field mapping: '{value_template}'")
        logger.debug(f"Available branch results: {list(branch_results.keys())}")
        
        # Handle simple context variables first (non-branch references)
        if not '.' in value_template:
            result = value_template.format(**action_context)
            logger.debug(f"Simple template resolved: '{value_template}' -> '{result}'")
            return result
        
        # Handle branch result references like {model_id.field}
        import re
        branch_refs = re.findall(r'\{([^}]+\.[^}]+)\}', value_template)
        logger.debug(f"Found branch references: {branch_refs}")
        
        resolved_template = value_template
        for ref in branch_refs:
            try:
                model_id, field_name = ref.split('.', 1)
                logger.debug(f"Processing branch reference: model_id='{model_id}', field_name='{field_name}'")
                
                if model_id in branch_results:
                    branch_data = branch_results[model_id]
                    logger.debug(f"Branch '{model_id}' data: {branch_data}")
                    
                    if field_name in branch_data:
                        field_value = branch_data[field_name]
                        resolved_template = resolved_template.replace(f'{{{ref}}}', str(field_value))
                        logger.info(f"✅ Resolved {ref} to '{field_value}'")
                    else:
                        logger.warning(f"Field '{field_name}' not found in branch '{model_id}' results.")
                        logger.debug(f"Available fields in '{model_id}': {list(branch_data.keys())}")
                        
                        # Try alternative field names based on the class result and model type
                        if isinstance(branch_data, dict):
                            fallback_value = None
                            
                            # First, try the exact field name
                            if field_name in branch_data:
                                fallback_value = branch_data[field_name]
                            # Then try 'class' field as fallback
                            elif 'class' in branch_data:
                                fallback_value = branch_data['class']
                                logger.info(f"Using 'class' field as fallback for '{field_name}': '{fallback_value}'")
                            # For brand models, also check if the class name exists as a key
                            elif field_name == 'brand' and branch_data.get('class') in branch_data:
                                fallback_value = branch_data[branch_data['class']]
                                logger.info(f"Found brand value using class name as key: '{fallback_value}'")
                            # For body_type models, also check if the class name exists as a key
                            elif field_name == 'body_type' and branch_data.get('class') in branch_data:
                                fallback_value = branch_data[branch_data['class']]
                                logger.info(f"Found body_type value using class name as key: '{fallback_value}'")
                            
                            if fallback_value is not None:
                                resolved_template = resolved_template.replace(f'{{{ref}}}', str(fallback_value))
                                logger.info(f"✅ Resolved {ref} to '{fallback_value}' (using fallback)")
                            else:
                                logger.error(f"No suitable field found for '{field_name}' in branch '{model_id}'")
                                logger.debug(f"Branch data structure: {branch_data}")
                                return None
                        else:
                            logger.error(f"Branch data for '{model_id}' is not a dictionary: {type(branch_data)}")
                            return None
                else:
                    logger.warning(f"Branch '{model_id}' not found in results. Available branches: {list(branch_results.keys())}")
                    return None
            except ValueError as e:
                logger.error(f"Invalid branch reference format: {ref}")
                return None
        
        # Format any remaining simple variables
        try:
            final_value = resolved_template.format(**action_context)
            logger.debug(f"Final resolved value: '{final_value}'")
            return final_value
        except KeyError as e:
            logger.warning(f"Could not resolve context variable in template: {e}")
            return resolved_template
            
    except Exception as e:
        logger.error(f"Error resolving field mapping '{value_template}': {e}")
        import traceback
        logger.debug(f"Field mapping error traceback: {traceback.format_exc()}")
        return None

def run_detection_with_tracking(frame, node, context=None):
    """
    Structured function for running YOLO detection with BoT-SORT tracking.
    Now includes track ID-based validation requiring N consecutive frames of the same track ID.
    
    Args:
        frame: Input frame/image
        node: Pipeline node configuration with model and settings
        context: Optional context information (camera info, session data, etc.)
        
    Returns:
        tuple: (all_detections, regions_dict, track_validation_result) where:
            - all_detections: List of all detection objects
            - regions_dict: Dict mapping class names to highest confidence detections
            - track_validation_result: Dict with validation status and stable tracks
            
    Configuration options in node:
        - model: YOLO model instance
        - triggerClassIndices: List of class indices to detect (None for all classes)
        - minConfidence: Minimum confidence threshold
        - multiClass: Whether to enable multi-class detection mode
        - expectedClasses: List of expected class names for multi-class validation
        - tracking: Dict with tracking configuration
            - enabled: Boolean to enable/disable tracking
            - method: Tracking method ("botsort")
            - reidConfig: Path to ReID config file
            - stabilityThreshold: Number of consecutive frames required for validation
    """
    try:
        # Extract tracking configuration
        tracking_config = node.get("tracking", {})
        tracking_enabled = tracking_config.get("enabled", True)
        reid_config_path = tracking_config.get("reidConfig", tracking_config.get("reidConfigPath", "botsort.yaml"))
        stability_threshold = tracking_config.get("stabilityThreshold", node.get("stabilityThreshold", 4))
        
        # Check if we need to reset tracker after cooldown
        camera_id = context.get("camera_id", "unknown") if context else "unknown"
        model_id = node.get("modelId", "unknown")
        stability_data = get_camera_stability_data(camera_id, model_id)
        session_state = stability_data["session_state"]
        
        if session_state.get("reset_tracker_on_resume", False):
            # Reset YOLO tracker to get fresh track IDs
            if hasattr(node["model"], 'trackers') and node["model"].trackers:
                node["model"].trackers.clear()  # Clear tracker state
                logger.info(f"Camera {camera_id}: 🔄 Reset YOLO tracker - new cars will get fresh track IDs")
            session_state["reset_tracker_on_resume"] = False  # Clear the flag
        
        # Tracking zones removed - process all detections
        
        # Prepare class filtering
        trigger_class_indices = node.get("triggerClassIndices")
        class_filter = {"classes": trigger_class_indices} if trigger_class_indices else {}
        
        logger.debug(f"Running detection for {node['modelId']} - tracking: {tracking_enabled}, stability_threshold: {stability_threshold}, classes: {node.get('triggerClasses', 'all')}")
        
        # Use predict for detection-only models (frontal detection), track for main detection models
        model_id = node.get("modelId", "")
        use_tracking = tracking_enabled and not ("frontal" in model_id.lower() or "detection" in model_id.lower())
        
        if use_tracking:
            # Use tracking for main detection models (yolo11m, etc.)
            logger.debug(f"Using tracking for {model_id}")
            res = node["model"].track(
                frame,
                stream=False,
                persist=True,
                **class_filter
            )[0]
        else:
            # Use detection only for frontal detection and other detection-only models
            logger.debug(f"Using prediction only for {model_id}")
            res = node["model"].predict(
                frame,
                stream=False,
                **class_filter
            )[0]
        
        # Process detection results
        candidate_detections = []
        # Use frontalMinConfidence for frontal detection models, otherwise use minConfidence
        model_id = node.get("modelId", "")
        if "frontal" in model_id.lower() and "frontalMinConfidence" in node:
            min_confidence = node.get("frontalMinConfidence", 0.0)
            logger.debug(f"Using frontalMinConfidence={min_confidence} for {model_id}")
        else:
            min_confidence = node.get("minConfidence", 0.0)
        
        if res.boxes is None or len(res.boxes) == 0:
            logger.debug(f"🚫 Camera {camera_id}: YOLO returned no detections")
            
            # Update stability tracking even when no detection (to reset counters)
            camera_id = context.get("camera_id", "unknown") if context else "unknown"
            model_id = node.get("modelId", "unknown")
            track_validation_result = update_single_track_stability(node, None, camera_id, frame.shape, stability_threshold, context)
            
            # Store validation state in context for pipeline decisions
            if context is not None:
                context["track_validation_result"] = track_validation_result
            
            return [], {}, track_validation_result
        
        logger.debug(f"🔍 Camera {camera_id}: YOLO detected {len(res.boxes)} raw objects - processing with tracking...")
        
        # First pass: collect all valid detections
        logger.debug(f"🔍 Camera {camera_id}: === DETECTION ANALYSIS ===")
        for i, box in enumerate(res.boxes):
            # Extract detection data
            conf = float(box.cpu().conf[0])
            cls_id = int(box.cpu().cls[0])
            class_name = node["model"].names[cls_id]
            
            # Extract bounding box
            xy = box.cpu().xyxy[0]
            x1, y1, x2, y2 = map(int, xy)
            bbox = (x1, y1, x2, y2)
            
            # Extract tracking ID if available
            track_id = None
            if hasattr(box, "id") and box.id is not None:
                track_id = int(box.id.item())
            
            logger.debug(f"🔍 Camera {camera_id}: Detection {i+1}: class='{class_name}' conf={conf:.3f} track_id={track_id} bbox={bbox}")
            
            # Apply confidence filtering
            if conf < min_confidence:
                logger.debug(f"❌ Camera {camera_id}: Detection {i+1} REJECTED - confidence {conf:.3f} < {min_confidence}")
                continue
            
            # Tracking zone validation removed - process all detections
            
            # Create detection object
            detection = {
                "class": class_name,
                "confidence": conf,
                "id": track_id,
                "bbox": bbox,
                "class_id": cls_id
            }
            
            candidate_detections.append(detection)
            logger.debug(f"✅ Camera {camera_id}: Detection {i+1} ACCEPTED as candidate: {class_name} (conf={conf:.3f}, track_id={track_id})")
        
        # Second pass: select only the highest confidence detection overall
        if not candidate_detections:
            logger.debug(f"🚫 Camera {camera_id}: No valid candidates after filtering - no car will be tracked")
            
            # Update stability tracking even when no detection (to reset counters)
            camera_id = context.get("camera_id", "unknown") if context else "unknown"
            model_id = node.get("modelId", "unknown")
            track_validation_result = update_single_track_stability(node, None, camera_id, frame.shape, stability_threshold, context)
            
            # Store validation state in context for pipeline decisions
            if context is not None:
                context["track_validation_result"] = track_validation_result
            
            return [], {}, track_validation_result
        
        logger.debug(f"🏆 Camera {camera_id}: === SELECTING HIGHEST CONFIDENCE CAR ===")
        for i, detection in enumerate(candidate_detections):
            logger.debug(f"🏆 Camera {camera_id}: Candidate {i+1}: {detection['class']} conf={detection['confidence']:.3f} track_id={detection['id']}")
        
        # Show all candidate detections before selection
        logger.debug(f"Found {len(candidate_detections)} candidate detections:")
        for i, det in enumerate(candidate_detections):
            logger.debug(f"Candidate {i+1}: {det['class']} conf={det['confidence']:.3f} bbox={det['bbox']}")
        
        # Find the single highest confidence detection across all detected classes
        best_detection = max(candidate_detections, key=lambda x: x["confidence"])
        original_class = best_detection["class"]
        track_id = best_detection["id"]
        
        logger.info(f"🎯 Camera {camera_id}: SELECTED WINNER: {original_class} (conf={best_detection['confidence']:.3f}, track_id={track_id}, bbox={best_detection['bbox']})")
        
        # Show which cars were NOT selected
        for detection in candidate_detections:
            if detection != best_detection:
                logger.debug(f"🚫 Camera {camera_id}: NOT SELECTED: {detection['class']} (conf={detection['confidence']:.3f}, track_id={detection['id']}) - lower confidence")
        
        # Apply class mapping if configured
        mapped_class = original_class
        class_mapping = node.get("classMapping", {})
        if original_class in class_mapping:
            mapped_class = class_mapping[original_class]
            logger.info(f"Class mapping applied: {original_class} → {mapped_class}")
            # Update the detection object with mapped class
            best_detection["class"] = mapped_class
            best_detection["original_class"] = original_class  # Keep original for reference
        
        # Keep only the single best detection with mapped class
        all_detections = [best_detection]
        regions_dict = {
            mapped_class: {
                "bbox": best_detection["bbox"],
                "confidence": best_detection["confidence"],
                "detection": best_detection,
                "track_id": track_id
            }
        }
        
        # Multi-class validation
        if node.get("multiClass", False) and node.get("expectedClasses"):
            expected_classes = node["expectedClasses"]
            detected_classes = list(regions_dict.keys())
            
            logger.debug(f"Multi-class validation: expected={expected_classes}, detected={detected_classes}")
            
            # Check for required classes (flexible - at least one must match)
            matching_classes = [cls for cls in expected_classes if cls in detected_classes]
            if not matching_classes:
                logger.warning(f"Multi-class validation failed: no expected classes detected")
                return [], {}
            
            logger.info(f"Multi-class validation passed: {matching_classes} detected")
        
        logger.info(f"✅ Camera {camera_id}: DETECTION COMPLETE - tracking single car: track_id={track_id}, conf={best_detection['confidence']:.3f}")
        logger.debug(f"📊 Camera {camera_id}: Detection summary: {len(res.boxes)} raw → {len(candidate_detections)} candidates → 1 selected")
        
        # Debug: Save vehicle crop for debugging (disabled for production)
        # if node.get("modelId") in ["yolo11n", "yolo11m"] and regions_dict:
        #     try:
        #         import datetime
        #         os.makedirs("temp_debug", exist_ok=True)
        #         timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S_%f")[:-3]
        #         
        #         for class_name, region_data in regions_dict.items():
        #             bbox = region_data['bbox']
        #             x1, y1, x2, y2 = bbox
        #             cropped = frame[y1:y2, x1:x2]
        #             if cropped.size > 0:
        #                 model_name = node.get("modelId", "yolo")
        #                 debug_path = f"temp_debug/{model_name}_{class_name}_crop_{timestamp}.jpg"
        #                 cv2.imwrite(debug_path, cropped)
        #                 logger.debug(f"Saved {model_name} {class_name} crop to {debug_path}")
        #     except Exception as e:
        #         logger.error(f"Failed to save {node.get('modelId', 'yolo')} crop: {e}")
        
        # Update track-based stability tracking for the single selected car
        camera_id = context.get("camera_id", "unknown") if context else "unknown"
        model_id = node.get("modelId", "unknown")
        
        # Update stability tracking for the single best detection
        track_validation_result = update_single_track_stability(node, best_detection, camera_id, frame.shape, stability_threshold, context)
        
        # Store validation state in context for pipeline decisions
        if context is not None:
            context["track_validation_result"] = track_validation_result
        
        return all_detections, regions_dict, track_validation_result
        
    except Exception as e:
        logger.error(f"Error in detection_with_tracking for {node.get('modelId', 'unknown')}: {e}")
        logger.debug(f"Detection error traceback: {traceback.format_exc()}")
        return [], {}, {"validation_complete": False, "stable_tracks": [], "current_tracks": []}


def get_camera_stability_data(camera_id, model_id):
    """Get or create stability tracking data for a specific camera and model."""
    global _camera_stability_tracking
    
    if camera_id not in _camera_stability_tracking:
        _camera_stability_tracking[camera_id] = {}
    
    if model_id not in _camera_stability_tracking[camera_id]:
        logger.warning(f"🔄 Camera {camera_id}: Creating NEW stability data for {model_id} - this will reset any cooldown!")
        _camera_stability_tracking[camera_id][model_id] = {
            "track_stability_counters": {},  # Track ID -> consecutive frame count
            "stable_tracks": set(),           # Set of track IDs that have reached stability threshold
            "session_state": {
                "active": True,
                "waiting_for_backend_session": False,
                "wait_start_time": 0.0,
                "reset_tracker_on_resume": False
            }
            # Removed obsolete occupancy_state - app.py handles all mode transitions now
        }
    
    return _camera_stability_tracking[camera_id][model_id]

def reset_camera_stability_tracking(camera_id, model_id):
    """Reset all stability tracking data for a specific camera and model."""
    if camera_id in _camera_stability_tracking and model_id in _camera_stability_tracking[camera_id]:
        stability_data = _camera_stability_tracking[camera_id][model_id]
        
        # Clear all tracking data
        track_counters = stability_data["track_stability_counters"]
        stable_tracks = stability_data["stable_tracks"]
        session_state = stability_data["session_state"]
        
        old_counters = dict(track_counters)
        old_stable = list(stable_tracks)
        
        track_counters.clear()
        stable_tracks.clear()
        
        # IMPORTANT: Set flag to reset YOLO tracker on next detection run
        # This will ensure track IDs start fresh (1, 2, 3...) instead of continuing from old IDs
        session_state["reset_tracker_on_resume"] = True
        
        logger.info(f"🧹 Camera {camera_id}: CLEARED stability tracking - old_counters={old_counters}, old_stable={old_stable}")
        logger.info(f"🔄 Camera {camera_id}: YOLO tracker will be reset on next detection - fresh track IDs will start from 1")
    else:
        logger.debug(f"🧹 Camera {camera_id}: No stability tracking data to clear for model {model_id}")

def update_single_track_stability(node, detection, camera_id, frame_shape=None, stability_threshold=4, context=None):
    """Update track stability validation for a single highest confidence car."""
    model_id = node.get("modelId", "unknown")
    
    # Branch nodes should not do validation - only main pipeline should
    is_branch_node = node.get("cropClass") is not None or node.get("parallel") is True
    if is_branch_node:
        logger.debug(f"⏭️ Camera {camera_id}: Skipping validation for branch node {model_id} - validation only done at main pipeline level")
        return {"validation_complete": False, "branch_node": True, "stable_tracks": [], "current_tracks": []}
    
    # Check current mode - VALIDATION COUNTERS should increment in both validation_detecting and full_pipeline modes
    current_mode = context.get("current_mode", "unknown") if context else "unknown"
    is_validation_mode = (current_mode in ["validation_detecting", "full_pipeline"])
    
    # Get camera-specific stability data
    stability_data = get_camera_stability_data(camera_id, model_id)
    track_counters = stability_data["track_stability_counters"]
    stable_tracks = stability_data["stable_tracks"]
    
    current_track_id = detection.get("id") if detection else None
    
    # ═══ MODE-AWARE TRACK VALIDATION ═══
    logger.debug(f"📋 Camera {camera_id}: === TRACK VALIDATION ANALYSIS ===")
    logger.debug(f"📋 Camera {camera_id}: Current mode: {current_mode} (validation_mode={is_validation_mode})")
    logger.debug(f"📋 Camera {camera_id}: Current track_id: {current_track_id} (assigned by YOLO tracking - not sequential)")
    logger.debug(f"📋 Camera {camera_id}: Existing counters: {dict(track_counters)}")
    logger.debug(f"📋 Camera {camera_id}: Stable tracks: {list(stable_tracks)}")
    
    # IMPORTANT: Only modify validation counters during validation_detecting mode
    if not is_validation_mode:
        logger.debug(f"🚫 Camera {camera_id}: NOT in validation mode - skipping counter modifications")
        return {
            "validation_complete": False, 
            "stable_tracks": list(stable_tracks),
            "current_tracks": [current_track_id] if current_track_id is not None else []
        }
    
    if current_track_id is not None:
        # Check if this is a different track than we were tracking
        previous_track_ids = list(track_counters.keys())
        
        # VALIDATION MODE: Reset counter if different track OR if track was previously stable
        should_reset = (
            len(previous_track_ids) == 0 or  # No previous tracking
            current_track_id not in previous_track_ids or  # Different track ID
            current_track_id in stable_tracks  # Track was stable - start fresh validation
        )
        
        logger.debug(f"📋 Camera {camera_id}: Previous track_ids: {previous_track_ids}")
        logger.debug(f"📋 Camera {camera_id}: Track {current_track_id} was stable: {current_track_id in stable_tracks}")
        logger.debug(f"📋 Camera {camera_id}: Should reset counters: {should_reset}")
        
        if should_reset:
            # Clear all previous tracking - fresh validation needed
            if previous_track_ids:
                for old_track_id in previous_track_ids:
                    old_count = track_counters.pop(old_track_id, 0)
                    stable_tracks.discard(old_track_id)
                    logger.info(f"🔄 Camera {camera_id}: VALIDATION RESET - track {old_track_id} counter from {old_count} to 0 (reason: {'stable_track_restart' if current_track_id == old_track_id else 'different_track'})")
            
            # Start fresh validation for this track
            old_count = track_counters.get(current_track_id, 0)  # Store old count for logging
            track_counters[current_track_id] = 1
            current_count = 1
            logger.info(f"🆕 Camera {camera_id}: FRESH VALIDATION - Track {current_track_id} starting at 1/{stability_threshold}")
        else:
            # Continue validation for same track
            old_count = track_counters.get(current_track_id, 0)
            track_counters[current_track_id] = old_count + 1
            current_count = track_counters[current_track_id]
        
        logger.debug(f"🔢 Camera {camera_id}: Track {current_track_id} counter: {old_count} → {current_count}")
        logger.info(f"🔍 Camera {camera_id}: Track ID {current_track_id} validation {current_count}/{stability_threshold}")
        
        # Check if track has reached stability threshold
        logger.debug(f"📊 Camera {camera_id}: Checking stability: {current_count} >= {stability_threshold}? {current_count >= stability_threshold}")
        logger.debug(f"📊 Camera {camera_id}: Already stable: {current_track_id in stable_tracks}")
        
        if current_count >= stability_threshold and current_track_id not in stable_tracks:
            stable_tracks.add(current_track_id)
            logger.info(f"✅ Camera {camera_id}: Track ID {current_track_id} STABLE after {current_count} consecutive frames")
            logger.info(f"🎯 Camera {camera_id}: TRACK VALIDATION COMPLETE")
            logger.debug(f"🎯 Camera {camera_id}: Stable tracks now: {list(stable_tracks)}")
            return {
                "validation_complete": True, 
                "send_none_detection": True,
                "stable_tracks": [current_track_id],
                "newly_stable_tracks": [current_track_id],
                "current_tracks": [current_track_id]
            }
        elif current_count >= stability_threshold:
            logger.debug(f"📊 Camera {camera_id}: Track {current_track_id} already stable - not re-adding")
    else:
        # No car detected - ALWAYS clear all tracking and reset counters
        logger.debug(f"🚫 Camera {camera_id}: NO CAR DETECTED - clearing all tracking")
        if track_counters or stable_tracks:
            logger.debug(f"🚫 Camera {camera_id}: Existing state before reset: counters={dict(track_counters)}, stable={list(stable_tracks)}")
            for track_id in list(track_counters.keys()):
                old_count = track_counters.pop(track_id, 0)
                logger.info(f"🔄 Camera {camera_id}: No car detected - RESET track {track_id} counter from {old_count} to 0")
            track_counters.clear()  # Ensure complete reset
            stable_tracks.clear()   # Clear all stable tracks
            logger.info(f"✅ Camera {camera_id}: RESET TO VALIDATION PHASE - All counters and stable tracks cleared")
        else:
            logger.debug(f"🚫 Camera {camera_id}: No existing counters to clear")
        logger.debug(f"Camera {camera_id}: VALIDATION - no car detected (all counters reset)")
    
    # Final return - validation not complete
    result = {
        "validation_complete": False, 
        "stable_tracks": list(stable_tracks),
        "current_tracks": [current_track_id] if current_track_id is not None else []
    }
    
    logger.debug(f"📋 Camera {camera_id}: Track stability result: {result}")
    logger.debug(f"📋 Camera {camera_id}: Final counters: {dict(track_counters)}")
    logger.debug(f"📋 Camera {camera_id}: Final stable tracks: {list(stable_tracks)}")
    
    return result

# Keep the old function for backward compatibility but mark as deprecated
def update_track_stability_validation(node, detections, camera_id, frame_shape=None, stability_threshold=4):
    """DEPRECATED: Use update_single_track_stability instead."""
    logger.warning(f"update_track_stability_validation called for camera {camera_id} - this function is deprecated, use update_single_track_stability instead")
    if detections:
        best_detection = max(detections, key=lambda x: x.get("confidence", 0))
        return update_single_track_stability(node, best_detection, camera_id, frame_shape, stability_threshold, None)
    else:
        return update_single_track_stability(node, None, camera_id, frame_shape, stability_threshold, None)

def update_detection_stability(node, detections, camera_id, frame_shape=None):
    """Legacy detection-based stability counter - DEPRECATED."""
    # This function is deprecated in favor of track-based validation only
    logger.warning(f"update_detection_stability called for camera {camera_id} - this function is deprecated, use track-based validation instead")
    return {"validation_complete": False, "valid_detections": 0, "deprecated": True}

def update_track_stability(node, detections, camera_id, frame_shape=None):
    """DEPRECATED: This function is obsolete and should not be used."""
    logger.warning(f"update_track_stability called for camera {camera_id} - this function is deprecated and obsolete")
    return {"phase": "validation", "absence_counter": 0, "deprecated": True}

def check_stable_tracks(camera_id, model_id, regions_dict):
    """Check if any stable tracks match the detected classes for a specific camera."""
    # Get camera-specific stability data
    stability_data = get_camera_stability_data(camera_id, model_id)
    stable_tracks = stability_data["stable_tracks"]
    
    if not stable_tracks:
        return False, []
    
    # Check for track-based stability
    stable_detections = []
    
    for class_name, region_data in regions_dict.items():
        detection = region_data.get("detection", {})
        track_id = detection.get("id")
        
        if track_id is not None and track_id in stable_tracks:
            stable_detections.append((class_name, track_id))
            logger.debug(f"Camera {camera_id}: Found stable detection: {class_name} with stable track ID {track_id}")
    
    has_stable_tracks = len(stable_detections) > 0
    return has_stable_tracks, stable_detections

def reset_tracking_state(camera_id, model_id, reason="session ended"):
    """Reset tracking state after session completion or timeout."""
    stability_data = get_camera_stability_data(camera_id, model_id)
    session_state = stability_data["session_state"]
    
    # Clear all tracking data for fresh start
    stability_data["track_stability_counters"].clear()
    stability_data["stable_tracks"].clear()
    session_state["active"] = True
    session_state["waiting_for_backend_session"] = False
    session_state["wait_start_time"] = 0.0
    session_state["reset_tracker_on_resume"] = True
    
    logger.info(f"Camera {camera_id}: 🔄 Reset tracking state - {reason}")
    logger.info(f"Camera {camera_id}: 🧹 Cleared stability counters and stable tracks for fresh session")

def is_camera_active(camera_id, model_id):
    """Check if camera should be processing detections."""
    stability_data = get_camera_stability_data(camera_id, model_id)
    session_state = stability_data["session_state"]
    
    # Check if waiting for backend sessionId has timed out
    if session_state.get("waiting_for_backend_session", False):
        current_time = time.time()
        wait_start_time = session_state.get("wait_start_time", 0)
        elapsed_time = current_time - wait_start_time
        
        if elapsed_time >= _session_timeout_seconds:
            logger.warning(f"Camera {camera_id}: Backend sessionId timeout ({_session_timeout_seconds}s) - resetting tracking")
            reset_tracking_state(camera_id, model_id, "backend sessionId timeout")
            return True
        else:
            remaining_time = _session_timeout_seconds - elapsed_time
            logger.debug(f"Camera {camera_id}: Still waiting for backend sessionId - {remaining_time:.1f}s remaining")
            return False
    
    return session_state.get("active", True)

def cleanup_camera_stability(camera_id):
    """Clean up stability tracking data when a camera is disconnected."""
    global _camera_stability_tracking
    if camera_id in _camera_stability_tracking:
        del _camera_stability_tracking[camera_id]
        logger.info(f"Cleaned up stability tracking data for camera {camera_id}")

def occupancy_detector(camera_id, model_id, enable=True):
    """
    Temporary function to stop model inference after pipeline completion.
    
    Args:
        camera_id (str): Camera identifier
        model_id (str): Model identifier  
        enable (bool): True to enable occupancy mode (stop model after pipeline), False to disable
    
    When enabled:
    - Model stops inference after completing full pipeline
    - Backend sessionId handling continues in background
    
    Note: This is a temporary function that will be changed in the future.
    """
    stability_data = get_camera_stability_data(camera_id, model_id)
    session_state = stability_data["session_state"]
    
    if enable:
        session_state["occupancy_mode"] = True
        session_state["occupancy_enabled_at"] = time.time()
        # Occupancy mode logging removed - not used in enhanced lightweight mode
    else:
        session_state["occupancy_mode"] = False
        session_state.pop("occupancy_enabled_at", None)
        # Occupancy mode logging removed - not used in enhanced lightweight mode
    
    return session_state.get("occupancy_mode", False)

def validate_pipeline_execution(node, regions_dict):
    """
    Pre-validate that all required branches will execute successfully before 
    committing to Redis actions and database records.
    
    Returns:
        - (True, []) if pipeline can execute completely
        - (False, missing_branches) if some required branches won't execute
    """
    # Get all branches that parallel actions are waiting for
    required_branches = set()
    
    for action in node.get("parallelActions", []):
        if action.get("type") == "postgresql_update_combined":
            wait_for_branches = action.get("waitForBranches", [])
            required_branches.update(wait_for_branches)
    
    if not required_branches:
        # No parallel actions requiring specific branches
        logger.debug("No parallel actions with waitForBranches - validation passes")
        return True, []
    
    logger.debug(f"Pre-validation: checking if required branches {list(required_branches)} will execute")
    
    # Check each required branch
    missing_branches = []
    
    for branch in node.get("branches", []):
        branch_id = branch["modelId"]
        
        if branch_id not in required_branches:
            continue  # This branch is not required by parallel actions
            
        # Check if this branch would be triggered
        trigger_classes = branch.get("triggerClasses", [])
        min_conf = branch.get("minConfidence", 0)
        
        branch_triggered = False
        for det_class in regions_dict:
            det_confidence = regions_dict[det_class]["confidence"]
            
            if (det_class in trigger_classes and det_confidence >= min_conf):
                branch_triggered = True
                logger.debug(f"Pre-validation: branch {branch_id} WILL be triggered by {det_class} (conf={det_confidence:.3f} >= {min_conf})")
                break
        
        if not branch_triggered:
            missing_branches.append(branch_id)
            logger.warning(f"Pre-validation: branch {branch_id} will NOT be triggered - no matching classes or insufficient confidence")
            logger.debug(f"  Required: {trigger_classes} with min_conf={min_conf}")
            logger.debug(f"  Available: {[(cls, regions_dict[cls]['confidence']) for cls in regions_dict]}")
    
    if missing_branches:
        logger.error(f"Pipeline pre-validation FAILED: required branches {missing_branches} will not execute")
        return False, missing_branches
    else:
        logger.info(f"Pipeline pre-validation PASSED: all required branches {list(required_branches)} will execute")
        return True, []

def run_lightweight_detection_with_validation(frame, node: dict, min_confidence=0.7, min_bbox_area_ratio=0.3):
    """
    Run lightweight detection with validation rules for session ID triggering.
    Returns detection info only if it passes validation thresholds.
    """
    logger.debug(f"Running lightweight detection with validation: {node['modelId']} (conf>={min_confidence}, bbox_area>={min_bbox_area_ratio})")
    
    try:
        # Run basic detection only - no branches, no actions
        model = node["model"]
        trigger_classes = node.get("triggerClasses", [])
        trigger_class_indices = node.get("triggerClassIndices")
        
        # Run YOLO inference
        res = model(frame, verbose=False)
        
        best_detection = None
        frame_height, frame_width = frame.shape[:2]
        frame_area = frame_height * frame_width
        
        for r in res:
            boxes = r.boxes
            if boxes is None or len(boxes) == 0:
                continue
                
            for box in boxes:
                # Extract detection info
                xyxy = box.xyxy[0].cpu().numpy()
                conf = box.conf[0].cpu().numpy()
                cls_id = int(box.cls[0].cpu().numpy())
                class_name = model.names[cls_id]
                
                # Apply confidence threshold
                if conf < min_confidence:
                    continue
                
                # Apply trigger class filtering if specified
                if trigger_class_indices and cls_id not in trigger_class_indices:
                    continue
                if trigger_classes and class_name not in trigger_classes:
                    continue
                
                # Calculate bbox area ratio
                x1, y1, x2, y2 = xyxy
                bbox_area = (x2 - x1) * (y2 - y1)
                bbox_area_ratio = bbox_area / frame_area if frame_area > 0 else 0
                
                # Apply bbox area threshold
                if bbox_area_ratio < min_bbox_area_ratio:
                    logger.debug(f"Detection filtered out: bbox_area_ratio={bbox_area_ratio:.3f} < {min_bbox_area_ratio}")
                    continue
                
                # Validation passed
                if not best_detection or conf > best_detection["confidence"]:
                    best_detection = {
                        "class": class_name,
                        "confidence": float(conf),
                        "bbox": [int(x) for x in xyxy],
                        "bbox_area_ratio": float(bbox_area_ratio),
                        "validation_passed": True
                    }
        
        if best_detection:
            logger.debug(f"Validation PASSED: {best_detection['class']} (conf: {best_detection['confidence']:.3f}, area: {best_detection['bbox_area_ratio']:.3f})")
            return best_detection
        else:
            logger.debug(f"Validation FAILED: No detection meets criteria (conf>={min_confidence}, area>={min_bbox_area_ratio})")
            return {"validation_passed": False}
        
    except Exception as e:
        logger.error(f"Error in lightweight detection with validation: {str(e)}", exc_info=True)
        return {"validation_passed": False}

def run_lightweight_detection(frame, node: dict):
    """
    Run lightweight detection for car presence validation only.
    Returns basic detection info without running branches or external actions.
    """
    logger.debug(f"Running lightweight detection: {node['modelId']}")
    
    try:
        # Run basic detection only - no branches, no actions
        model = node["model"]
        min_confidence = node.get("minConfidence", 0.5)
        trigger_classes = node.get("triggerClasses", [])
        trigger_class_indices = node.get("triggerClassIndices")
        
        # Run YOLO inference
        res = model(frame, verbose=False)
        
        car_detected = False
        best_detection = None
        
        for r in res:
            boxes = r.boxes
            if boxes is None or len(boxes) == 0:
                continue
                
            for box in boxes:
                # Extract detection info
                xyxy = box.xyxy[0].cpu().numpy()
                conf = box.conf[0].cpu().numpy()
                cls_id = int(box.cls[0].cpu().numpy())
                class_name = model.names[cls_id]
                
                # Apply confidence threshold
                if conf < min_confidence:
                    continue
                
                # Apply trigger class filtering if specified
                if trigger_class_indices and cls_id not in trigger_class_indices:
                    continue
                if trigger_classes and class_name not in trigger_classes:
                    continue
                
                # Car detected
                car_detected = True
                if not best_detection or conf > best_detection["confidence"]:
                    best_detection = {
                        "class": class_name,
                        "confidence": float(conf),
                        "bbox": [int(x) for x in xyxy]
                    }
        
        logger.debug(f"Lightweight detection result: car_detected={car_detected}")
        if best_detection:
            logger.debug(f"Best detection: {best_detection['class']} (conf: {best_detection['confidence']:.3f})")
        
        return {
            "car_detected": car_detected,
            "best_detection": best_detection
        }
        
    except Exception as e:
        logger.error(f"Error in lightweight detection: {str(e)}", exc_info=True)
        return {"car_detected": False, "best_detection": None}

def run_pipeline(frame, node: dict, return_bbox: bool=False, context=None, validated_detection=None):
    """
    Enhanced pipeline that supports:
    - Multi-class detection (detecting multiple classes simultaneously)
    - Parallel branch processing
    - Region-based actions and cropping
    - Context passing for session/camera information
    """
    try:
        # Extract backend sessionId from context at the start of function
        backend_session_id = context.get("backend_session_id") if context else None
        camera_id = context.get("camera_id", "unknown") if context else "unknown"
        model_id = node.get("modelId", "unknown")
        
        if backend_session_id:
            logger.info(f"🔑 PIPELINE USING BACKEND SESSION_ID: {backend_session_id} for camera {camera_id}")
        
        task = getattr(node["model"], "task", None)

        # ─── Classification stage ───────────────────────────────────
        if task == "classify":
            results = node["model"].predict(frame, stream=False)
            if not results:
                return (None, None) if return_bbox else None

            r = results[0]
            probs = r.probs
            if probs is None:
                return (None, None) if return_bbox else None

            top1_idx = int(probs.top1)
            top1_conf = float(probs.top1conf)
            class_name = node["model"].names[top1_idx]

            det = {
                "class": class_name,
                "confidence": top1_conf,
                "id": None,
                class_name: class_name  # Add class name as key for backward compatibility
            }
            
            # Add specific field mappings for database operations based on model type
            model_id = node.get("modelId", "").lower()
            if "brand" in model_id or "brand_cls" in model_id:
                det["brand"] = class_name
            elif "bodytype" in model_id or "body" in model_id:
                det["body_type"] = class_name
            elif "color" in model_id:
                det["color"] = class_name
            
            execute_actions(node, frame, det, context.get("regions_dict") if context else None)
            return (det, None) if return_bbox else det

        # ─── Occupancy mode check (stop future frames after pipeline completion) ───────────────────────────────────────
        # Old occupancy mode logic removed - now using two-phase detection system

        # ─── Session management check ───────────────────────────────────────
        if not is_camera_active(camera_id, model_id):
            logger.debug(f"⏰ Camera {camera_id}: Waiting for backend sessionId, sending 'none' detection")
            none_detection = {
                "class": "none",
                "confidence": 1.0,
                "bbox": [0, 0, 0, 0],
                "branch_results": {}
            }
            return (none_detection, [0, 0, 0, 0]) if return_bbox else none_detection

        # ─── Detection stage - Use validated detection if provided (full_pipeline mode) ───
        if validated_detection:
            track_id = validated_detection.get('track_id')
            logger.info(f"🔄 PIPELINE: Using validated detection from validation phase - track_id={track_id}")
            # Convert validated detection back to all_detections format for branch processing
            all_detections = [validated_detection]
            # Create regions_dict based on validated detection class with proper structure
            class_name = validated_detection.get("class", "car")
            regions_dict = {
                class_name: {
                    "confidence": validated_detection.get("confidence"),
                    "bbox": validated_detection.get("bbox", [0, 0, 0, 0]),
                    "detection": validated_detection
                }
            }
            # Bypass track validation completely - force pipeline execution
            track_validation_result = {
                "validation_complete": True, 
                "stable_tracks": ["cached"],  # Use dummy stable track to force pipeline execution
                "current_tracks": ["cached"],
                "bypass_validation": True
            }
        else:
            # Normal detection stage - Using structured detection function
            all_detections, regions_dict, track_validation_result = run_detection_with_tracking(frame, node, context)
        
        # Debug: Save crops for debugging (disabled for production)
        # if regions_dict:
        #     try:
        #         import datetime
        #         os.makedirs("temp_debug", exist_ok=True)
        #         timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
        #         model_id = node.get("modelId", "unknown")
        #         
        #         # Save vehicle crop from yolo model (any vehicle: car, truck, bus, motorcycle)
        #         if model_id in ["yolo11n", "yolo11m"]:
        #             # Look for any vehicle class in regions_dict
        #             vehicle_classes = ["car", "truck", "bus", "motorcycle"]
        #             found_vehicle = None
        #             for vehicle_class in vehicle_classes:
        #                 if vehicle_class in regions_dict:
        #                     found_vehicle = vehicle_class
        #                     break
        #             
        #             if found_vehicle:
        #                 bbox = regions_dict[found_vehicle]['bbox']
        #                 x1, y1, x2, y2 = bbox
        #                 cropped = frame[y1:y2, x1:x2]
        #                 if cropped.size > 0:
        #                     debug_path = f"temp_debug/{found_vehicle}_crop_{timestamp}.jpg"
        #                     cv2.imwrite(debug_path, cropped)
        #                     logger.debug(f"Saved {found_vehicle} crop to {debug_path}")
        #         
        #         # Save frontal crop from frontal_detection_v1
        #         elif model_id == "frontal_detection_v1" and "frontal" in regions_dict:
        #             bbox = regions_dict["frontal"]['bbox']
        #             x1, y1, x2, y2 = bbox
        #             cropped = frame[y1:y2, x1:x2]
        #             if cropped.size > 0:
        #                 debug_path = f"temp_debug/frontal_crop_{timestamp}.jpg"
        #                 cv2.imwrite(debug_path, cropped)
        #                 logger.debug(f"Saved frontal crop to {debug_path}")
        #                 
        #     except Exception as e:
        #         logger.error(f"Failed to save crops: {e}")
        
        if not all_detections:
            logger.debug("No detections from structured detection function - sending 'none' detection")
            none_detection = {
                "class": "none",
                "confidence": 1.0,
                "bbox": [0, 0, 0, 0],
                "branch_results": {}
            }
            return (none_detection, [0, 0, 0, 0]) if return_bbox else none_detection

        # Extract bounding boxes for compatibility
        all_boxes = [det["bbox"] for det in all_detections]

        # ─── Track-Based Validation System: Using Track ID Stability ────────────────────────
        tracking_config = node.get("tracking", {})
        stability_threshold = tracking_config.get("stabilityThreshold", node.get("stabilityThreshold", 1))
        
        camera_id = context.get("camera_id", "unknown") if context else "unknown"
        
        if stability_threshold > 1 and tracking_config.get("enabled", True):
            # Note: Old occupancy state system removed - app.py handles all mode transitions now
            # Track validation is handled by update_single_track_stability function
            model_id = node.get("modelId", "unknown")
            
            # Simplified: just check if we have stable tracks from track validation
            current_phase = "validation"  # Always validation phase in simplified system
            absence_counter = 0
            max_absence_frames = 3
            
            if current_phase == "validation":
                # ═══ TRACK VALIDATION PHASE ═══
                # Check if this is a branch node - branches should execute regardless of main validation state
                is_branch_node = node.get("cropClass") is not None or node.get("parallel") is True
                
                if is_branch_node:
                    # This is a branch node - allow normal execution regardless of main pipeline validation
                    logger.debug(f"🔍 Camera {camera_id}: Branch node {model_id} executing during track validation phase")
                else:
                    # Main pipeline node during track validation - check for stable tracks
                    stable_tracks = track_validation_result.get("stable_tracks", [])
                    
                    if not stable_tracks:
                        # No stable tracks yet - return detection without branches until track validation completes
                        if all_detections:
                            # Return the best detection but skip branches during validation
                            primary_detection = max(all_detections, key=lambda x: x["confidence"])
                            logger.debug(f"🔍 Camera {camera_id}: TRACK VALIDATION PHASE - returning detection without branches (stable_tracks: {len(stable_tracks)}, sessionId: {backend_session_id or 'none'})")
                        else:
                            # No detection - return none
                            primary_detection = {"class": "none", "confidence": 0.0, "bbox": [0, 0, 0, 0]}
                            logger.debug(f"🔍 Camera {camera_id}: TRACK VALIDATION PHASE - no detection found (sessionId: {backend_session_id or 'none'})")
                        
                        primary_bbox = primary_detection.get("bbox", [0, 0, 0, 0])
                        return (primary_detection, primary_bbox) if return_bbox else primary_detection
                    else:
                        # We have stable tracks - validation is complete, proceed with pipeline
                        logger.info(f"🎯 Camera {camera_id}: STABLE TRACKS DETECTED - proceeding with full pipeline (tracks: {stable_tracks})")
                    
            # Note: Old waiting_for_session and occupancy phases removed
            # app.py lightweight mode handles all state transitions now

        # ─── Pre-validate pipeline execution (only proceed if we have stable tracks for main pipeline) ────────────────────────
        is_branch_node = node.get("cropClass") is not None or node.get("parallel") is True
        
        if not is_branch_node and stability_threshold > 1 and tracking_config.get("enabled", True):
            # Main pipeline node with tracking - check for stable tracks before proceeding
            stable_tracks = track_validation_result.get("stable_tracks", [])
            if not stable_tracks:
                logger.debug(f"🔒 Camera {camera_id}: Main pipeline requires stable tracks - none found, skipping pipeline execution")
                none_detection = {"class": "none", "confidence": 1.0, "bbox": [0, 0, 0, 0], "awaiting_stable_tracks": True}
                return (none_detection, [0, 0, 0, 0]) if return_bbox else none_detection
        
        pipeline_valid, missing_branches = validate_pipeline_execution(node, regions_dict)
        
        if not pipeline_valid:
            logger.error(f"Pipeline execution validation FAILED - required branches {missing_branches} cannot execute")
            logger.error("Aborting pipeline: no Redis actions or database records will be created")
            return (None, None) if return_bbox else None

        # ─── Execute actions with region information ────────────────
        detection_result = {
            "detections": all_detections,
            "regions": regions_dict,
            **(context or {})
        }
        
        # ─── Database operations will be handled when backend sessionId is received ────
        
        if node.get("db_manager") and regions_dict:
            detected_classes = list(regions_dict.keys())
            logger.debug(f"Valid detections found: {detected_classes}")
            
            if backend_session_id:
                # Backend sessionId is available, proceed with database operations
                from datetime import datetime
                display_id = detection_result.get("display_id", "unknown")
                timestamp = datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
                
                inserted_session_id = node["db_manager"].insert_initial_detection(
                    display_id=display_id,
                    captured_timestamp=timestamp,
                    session_id=backend_session_id
                )
                
                if inserted_session_id:
                    detection_result["session_id"] = inserted_session_id
                    detection_result["timestamp"] = timestamp
                    logger.info(f"💾 DATABASE RECORD CREATED with backend session_id: {inserted_session_id}")
                    logger.debug(f"Database record: display_id={display_id}, timestamp={timestamp}")
                else:
                    logger.error(f"Failed to create database record with backend session_id: {backend_session_id}")
            else:
                logger.info(f"📡 Camera {camera_id}: Full pipeline completed, detection data will be sent to backend. Database operations will occur when sessionId is received.")
                # Store detection info for later database operations when sessionId arrives
                detection_result["awaiting_session_id"] = True
                from datetime import datetime
                detection_result["timestamp"] = datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
        
        # Execute actions for root node only if it doesn't have branches
        # Branch nodes with actions will execute them after branch processing
        if not node.get("branches") or node.get("modelId") == "yolo11n":
            execute_actions(node, frame, detection_result, regions_dict)

        # ─── Branch processing (no stability check here) ─────────────────────────────
        if node["branches"]:
            branch_results = {}
            
            # Extract camera_id for logging
            camera_id = detection_result.get("camera_id", context.get("camera_id", "unknown") if context else "unknown")
            
            
            # Filter branches that should be triggered
            active_branches = []
            for br in node["branches"]:
                trigger_classes = br.get("triggerClasses", [])
                min_conf = br.get("minConfidence", 0)
                
                logger.debug(f"Evaluating branch {br['modelId']}: trigger_classes={trigger_classes}, min_conf={min_conf}")
                
                # Check if any detected class matches branch trigger
                branch_triggered = False
                for det_class in regions_dict:
                    det_confidence = regions_dict[det_class]["confidence"]
                    logger.debug(f"  Checking detected class '{det_class}' (confidence={det_confidence:.3f}) against triggers {trigger_classes}")
                    
                    if (det_class in trigger_classes and det_confidence >= min_conf):
                        active_branches.append(br)
                        branch_triggered = True
                        logger.info(f"Branch {br['modelId']} activated by class '{det_class}' (conf={det_confidence:.3f} >= {min_conf})")
                        break
                
                if not branch_triggered:
                    logger.debug(f"Branch {br['modelId']} not triggered - no matching classes or insufficient confidence")
            
            if active_branches:
                if node.get("parallel", False) or any(br.get("parallel", False) for br in active_branches):
                    # Run branches in parallel
                    with concurrent.futures.ThreadPoolExecutor(max_workers=len(active_branches)) as executor:
                        futures = {}
                        
                        for br in active_branches:
                            sub_frame = frame
                            crop_class = br.get("cropClass")
                            
                            logger.info(f"Starting parallel branch: {br['modelId']}, cropClass: {crop_class}")
                            
                            if br.get("crop", False) and crop_class:
                                if crop_class in regions_dict:
                                    cropped = crop_region_by_class(frame, regions_dict, crop_class)
                                    if cropped is not None:
                                        sub_frame = cropped  # Use cropped image without manual resizing
                                        logger.debug(f"Successfully cropped {crop_class} region for {br['modelId']} - model will handle resizing")
                                    else:
                                        logger.warning(f"Failed to crop {crop_class} region for {br['modelId']}, skipping branch")
                                        continue
                                else:
                                    logger.warning(f"Crop class {crop_class} not found in detected regions for {br['modelId']}, skipping branch")
                                    continue
                            
                            # Add regions_dict and session_id to context for child branches
                            branch_context = dict(context) if context else {}
                            branch_context["regions_dict"] = regions_dict
                            
                            # Pass session_id from detection_result to branch context for Redis actions
                            if "session_id" in detection_result:
                                branch_context["session_id"] = detection_result["session_id"]
                                logger.debug(f"Added session_id to branch context: {detection_result['session_id']}")
                            elif backend_session_id:
                                branch_context["session_id"] = backend_session_id
                                logger.debug(f"Added backend_session_id to branch context: {backend_session_id}")
                            
                            future = executor.submit(run_pipeline, sub_frame, br, True, branch_context)
                            futures[future] = br
                        
                        # Collect results
                        for future in concurrent.futures.as_completed(futures):
                            br = futures[future]
                            try:
                                result, _ = future.result()
                                if result:
                                    branch_results[br["modelId"]] = result
                                    logger.info(f"Branch {br['modelId']} completed: {result}")
                                    
                                    # Collect nested branch results if they exist
                                    if "branch_results" in result:
                                        for nested_id, nested_result in result["branch_results"].items():
                                            branch_results[nested_id] = nested_result
                                            logger.info(f"Collected nested branch result: {nested_id} = {nested_result}")
                            except Exception as e:
                                logger.error(f"Branch {br['modelId']} failed: {e}")
                else:
                    # Run branches sequentially  
                    for br in active_branches:
                        sub_frame = frame
                        crop_class = br.get("cropClass")
                        
                        logger.info(f"Starting sequential branch: {br['modelId']}, cropClass: {crop_class}")
                        
                        if br.get("crop", False) and crop_class:
                            if crop_class in regions_dict:
                                cropped = crop_region_by_class(frame, regions_dict, crop_class)
                                if cropped is not None:
                                    sub_frame = cropped  # Use cropped image without manual resizing
                                    logger.debug(f"Successfully cropped {crop_class} region for {br['modelId']} - model will handle resizing")
                                else:
                                    logger.warning(f"Failed to crop {crop_class} region for {br['modelId']}, skipping branch")
                                    continue
                            else:
                                logger.warning(f"Crop class {crop_class} not found in detected regions for {br['modelId']}, skipping branch")
                                continue
                        
                        try:
                            # Add regions_dict and session_id to context for child branches
                            branch_context = dict(context) if context else {}
                            branch_context["regions_dict"] = regions_dict
                            
                            # Pass session_id from detection_result to branch context for Redis actions
                            if "session_id" in detection_result:
                                branch_context["session_id"] = detection_result["session_id"]
                                logger.debug(f"Added session_id to sequential branch context: {detection_result['session_id']}")
                            elif backend_session_id:
                                branch_context["session_id"] = backend_session_id
                                logger.debug(f"Added backend_session_id to sequential branch context: {backend_session_id}")
                            
                            result, _ = run_pipeline(sub_frame, br, True, branch_context)
                            if result:
                                branch_results[br["modelId"]] = result
                                logger.info(f"Branch {br['modelId']} completed: {result}")
                                
                                # Collect nested branch results if they exist
                                if "branch_results" in result:
                                    for nested_id, nested_result in result["branch_results"].items():
                                        branch_results[nested_id] = nested_result
                                        logger.info(f"Collected nested branch result: {nested_id} = {nested_result}")
                            else:
                                logger.warning(f"Branch {br['modelId']} returned no result")
                        except Exception as e:
                            logger.error(f"Error in sequential branch {br['modelId']}: {e}")
                            import traceback
                            logger.debug(f"Branch error traceback: {traceback.format_exc()}")

            # Store branch results in detection_result for parallel actions
            detection_result["branch_results"] = branch_results

        # ─── Execute Parallel Actions ───────────────────────────────
        if node.get("parallelActions") and "branch_results" in detection_result:
            execute_parallel_actions(node, frame, detection_result, regions_dict)
            
            # ─── Auto-enable occupancy mode after successful pipeline completion ─────────────────
            camera_id = context.get("camera_id", "unknown") if context else "unknown"
            model_id = node.get("modelId", "unknown")
            
            # Enable occupancy detector automatically after first successful pipeline
            # Auto-enabling occupancy logging removed - not used in enhanced lightweight mode
            occupancy_detector(camera_id, model_id, enable=True)
            
            logger.info(f"✅ Camera {camera_id}: Pipeline completed, detection data will be sent to backend")
            logger.info(f"🛑 Camera {camera_id}: Model will stop inference for future frames")
            logger.info(f"📡 Backend sessionId will be handled when received via WebSocket")

        # ─── Execute actions after successful detection AND branch processing ──────────
        # This ensures detection nodes (like frontal_detection_v1) execute their actions
        # after completing both detection and branch processing
        if node.get("actions") and regions_dict and node.get("modelId") != "yolo11n":
            # Execute actions for branch detection nodes, skip root to avoid duplication
            logger.debug(f"Executing post-detection actions for branch node {node.get('modelId')}")
            execute_actions(node, frame, detection_result, regions_dict)

        # ─── Return detection result ────────────────────────────────
        primary_detection = max(all_detections, key=lambda x: x["confidence"])
        primary_bbox = primary_detection["bbox"]
        
        # Add branch results and session_id to primary detection for compatibility
        if "branch_results" in detection_result:
            primary_detection["branch_results"] = detection_result["branch_results"]
        if "session_id" in detection_result:
            primary_detection["session_id"] = detection_result["session_id"]
        
        return (primary_detection, primary_bbox) if return_bbox else primary_detection

    except Exception as e:
        logger.error(f"Error in node {node.get('modelId')}: {e}")
        import traceback
        traceback.print_exc()
        return (None, None) if return_bbox else None