fix: cameras buffer
This commit is contained in:
ziesorx
parent
519e073f7f
commit
bd201acac1
1 changed files with 127 additions and 143 deletions
|
|
@ -12,8 +12,7 @@ import os
|
||||||
import subprocess
|
import subprocess
|
||||||
# import fcntl # No longer needed with atomic file operations
|
# import fcntl # No longer needed with atomic file operations
|
||||||
from typing import Optional, Callable
|
from typing import Optional, Callable
|
||||||
from watchdog.observers import Observer
|
# Removed watchdog imports - no longer using file watching
|
||||||
from watchdog.events import FileSystemEventHandler
|
|
||||||
|
|
||||||
# Suppress FFMPEG/H.264 error messages if needed
|
# Suppress FFMPEG/H.264 error messages if needed
|
||||||
# Set this environment variable to reduce noise from decoder errors
|
# Set this environment variable to reduce noise from decoder errors
|
||||||
|
|
@ -22,31 +21,14 @@ os.environ["OPENCV_FFMPEG_LOGLEVEL"] = "-8" # Suppress FFMPEG warnings
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# Suppress noisy watchdog debug logs
|
# Removed watchdog logging configuration - no longer using file watching
|
||||||
logging.getLogger('watchdog.observers.inotify_buffer').setLevel(logging.CRITICAL)
|
|
||||||
logging.getLogger('watchdog.observers.fsevents').setLevel(logging.CRITICAL)
|
|
||||||
logging.getLogger('fsevents').setLevel(logging.CRITICAL)
|
|
||||||
|
|
||||||
|
|
||||||
class FrameFileHandler(FileSystemEventHandler):
|
# Removed FrameFileHandler - no longer using file watching
|
||||||
"""File system event handler for frame file changes."""
|
|
||||||
|
|
||||||
def __init__(self, callback):
|
|
||||||
self.callback = callback
|
|
||||||
self.last_modified = 0
|
|
||||||
|
|
||||||
def on_modified(self, event):
|
|
||||||
if event.is_directory:
|
|
||||||
return
|
|
||||||
# Debounce rapid file changes
|
|
||||||
current_time = time.time()
|
|
||||||
if current_time - self.last_modified > 0.01: # 10ms debounce
|
|
||||||
self.last_modified = current_time
|
|
||||||
self.callback()
|
|
||||||
|
|
||||||
|
|
||||||
class FFmpegRTSPReader:
|
class FFmpegRTSPReader:
|
||||||
"""RTSP stream reader using subprocess FFmpeg with CUDA hardware acceleration and file watching."""
|
"""RTSP stream reader using subprocess FFmpeg piping frames directly to buffer."""
|
||||||
|
|
||||||
def __init__(self, camera_id: str, rtsp_url: str, max_retries: int = 3):
|
def __init__(self, camera_id: str, rtsp_url: str, max_retries: int = 3):
|
||||||
self.camera_id = camera_id
|
self.camera_id = camera_id
|
||||||
|
|
@ -56,10 +38,8 @@ class FFmpegRTSPReader:
|
||||||
self.stop_event = threading.Event()
|
self.stop_event = threading.Event()
|
||||||
self.thread = None
|
self.thread = None
|
||||||
self.frame_callback: Optional[Callable] = None
|
self.frame_callback: Optional[Callable] = None
|
||||||
self.observer = None
|
|
||||||
self.frame_ready_event = threading.Event()
|
|
||||||
|
|
||||||
# Stream specs
|
# Expected stream specs (for reference, actual dimensions read from PPM header)
|
||||||
self.width = 1280
|
self.width = 1280
|
||||||
self.height = 720
|
self.height = 720
|
||||||
|
|
||||||
|
|
@ -91,18 +71,58 @@ class FFmpegRTSPReader:
|
||||||
self.thread.join(timeout=5.0)
|
self.thread.join(timeout=5.0)
|
||||||
logger.info(f"Stopped FFmpeg reader for camera {self.camera_id}")
|
logger.info(f"Stopped FFmpeg reader for camera {self.camera_id}")
|
||||||
|
|
||||||
def _start_ffmpeg_process(self):
|
def _probe_stream_info(self):
|
||||||
"""Start FFmpeg subprocess writing timestamped frames for atomic reads."""
|
"""Probe stream to get resolution and other info."""
|
||||||
# Create temp file paths for this camera
|
try:
|
||||||
self.frame_dir = "/tmp/frame"
|
cmd = [
|
||||||
os.makedirs(self.frame_dir, exist_ok=True)
|
'ffprobe',
|
||||||
|
'-v', 'quiet',
|
||||||
|
'-print_format', 'json',
|
||||||
|
'-show_streams',
|
||||||
|
'-select_streams', 'v:0', # First video stream
|
||||||
|
'-rtsp_transport', 'tcp',
|
||||||
|
self.rtsp_url
|
||||||
|
]
|
||||||
|
|
||||||
# Use strftime pattern - FFmpeg writes each frame with unique timestamp
|
result = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
|
||||||
# This ensures each file is complete when written
|
if result.returncode != 0:
|
||||||
camera_id_safe = self.camera_id.replace(' ', '_')
|
logger.error(f"Camera {self.camera_id}: ffprobe failed (code {result.returncode})")
|
||||||
self.frame_prefix = f"camera_{camera_id_safe}"
|
if result.stderr:
|
||||||
# Using strftime pattern with seconds for unique filenames (avoid %f which may not work)
|
logger.error(f"Camera {self.camera_id}: ffprobe stderr: {result.stderr}")
|
||||||
self.frame_pattern = f"{self.frame_dir}/{self.frame_prefix}_%Y%m%d_%H%M%S.ppm"
|
if result.stdout:
|
||||||
|
logger.debug(f"Camera {self.camera_id}: ffprobe stdout: {result.stdout}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
import json
|
||||||
|
data = json.loads(result.stdout)
|
||||||
|
if not data.get('streams'):
|
||||||
|
logger.error(f"Camera {self.camera_id}: No video streams found")
|
||||||
|
return None
|
||||||
|
|
||||||
|
stream = data['streams'][0]
|
||||||
|
width = stream.get('width')
|
||||||
|
height = stream.get('height')
|
||||||
|
|
||||||
|
if not width or not height:
|
||||||
|
logger.error(f"Camera {self.camera_id}: Could not determine resolution")
|
||||||
|
return None
|
||||||
|
|
||||||
|
logger.info(f"Camera {self.camera_id}: Detected resolution {width}x{height}")
|
||||||
|
return width, height
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Camera {self.camera_id}: Error probing stream: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _start_ffmpeg_process(self):
|
||||||
|
"""Start FFmpeg subprocess outputting raw RGB frames to stdout pipe."""
|
||||||
|
# First probe the stream to get resolution
|
||||||
|
probe_result = self._probe_stream_info()
|
||||||
|
if not probe_result:
|
||||||
|
logger.error(f"Camera {self.camera_id}: Failed to probe stream info")
|
||||||
|
return False
|
||||||
|
|
||||||
|
self.actual_width, self.actual_height = probe_result
|
||||||
|
|
||||||
cmd = [
|
cmd = [
|
||||||
'ffmpeg',
|
'ffmpeg',
|
||||||
|
|
@ -111,50 +131,69 @@ class FFmpegRTSPReader:
|
||||||
# '-hwaccel_device', '0',
|
# '-hwaccel_device', '0',
|
||||||
'-rtsp_transport', 'tcp',
|
'-rtsp_transport', 'tcp',
|
||||||
'-i', self.rtsp_url,
|
'-i', self.rtsp_url,
|
||||||
'-f', 'image2',
|
'-f', 'rawvideo', # Raw video output instead of PPM
|
||||||
'-strftime', '1', # Enable strftime pattern expansion
|
'-pix_fmt', 'rgb24', # Raw RGB24 format
|
||||||
'-pix_fmt', 'rgb24', # PPM uses RGB not BGR
|
# Use native stream resolution and framerate
|
||||||
'-an', # No audio
|
'-an', # No audio
|
||||||
'-y', # Overwrite output file
|
'-' # Output to stdout
|
||||||
self.frame_pattern # Write timestamped frames
|
|
||||||
]
|
]
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Log the FFmpeg command for debugging
|
# Log the FFmpeg command for debugging
|
||||||
logger.info(f"Starting FFmpeg for camera {self.camera_id} with command: {' '.join(cmd)}")
|
logger.info(f"Starting FFmpeg for camera {self.camera_id} with command: {' '.join(cmd)}")
|
||||||
|
|
||||||
# Start FFmpeg detached - we don't need to communicate with it
|
# Start FFmpeg with stdout pipe to read frames directly
|
||||||
self.process = subprocess.Popen(
|
self.process = subprocess.Popen(
|
||||||
cmd,
|
cmd,
|
||||||
stdout=subprocess.DEVNULL,
|
stdout=subprocess.PIPE, # Capture stdout for frame data
|
||||||
stderr=subprocess.DEVNULL
|
stderr=subprocess.DEVNULL,
|
||||||
|
bufsize=0 # Unbuffered for real-time processing
|
||||||
)
|
)
|
||||||
logger.info(f"Started FFmpeg process PID {self.process.pid} for camera {self.camera_id} -> {self.frame_pattern}")
|
logger.info(f"Started FFmpeg process PID {self.process.pid} for camera {self.camera_id} -> stdout pipe (resolution: {self.actual_width}x{self.actual_height})")
|
||||||
return True
|
return True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Failed to start FFmpeg for camera {self.camera_id}: {e}")
|
logger.error(f"Failed to start FFmpeg for camera {self.camera_id}: {e}")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def _setup_file_watcher(self):
|
def _read_raw_frame(self, pipe):
|
||||||
"""Setup file system watcher for frame directory."""
|
"""Read raw RGB frame data from pipe with proper buffering."""
|
||||||
# Setup file watcher for the frame directory
|
try:
|
||||||
handler = FrameFileHandler(lambda: self._on_file_changed())
|
# Calculate frame size using actual detected dimensions
|
||||||
self.observer = Observer()
|
frame_size = self.actual_width * self.actual_height * 3
|
||||||
self.observer.schedule(handler, self.frame_dir, recursive=False)
|
|
||||||
self.observer.start()
|
|
||||||
logger.info(f"Started file watcher for {self.frame_dir} with pattern {self.frame_prefix}*.ppm")
|
|
||||||
|
|
||||||
def _on_file_changed(self):
|
# Read frame data in chunks until we have the complete frame
|
||||||
"""Called when a new frame file is created."""
|
frame_data = b''
|
||||||
# Signal that a new frame might be available
|
bytes_remaining = frame_size
|
||||||
self.frame_ready_event.set()
|
|
||||||
|
while bytes_remaining > 0:
|
||||||
|
chunk = pipe.read(bytes_remaining)
|
||||||
|
if not chunk: # EOF
|
||||||
|
if len(frame_data) == 0:
|
||||||
|
logger.debug(f"Camera {self.camera_id}: No more data (stream ended)")
|
||||||
|
else:
|
||||||
|
logger.warning(f"Camera {self.camera_id}: Stream ended mid-frame: {len(frame_data)}/{frame_size} bytes")
|
||||||
|
return None
|
||||||
|
|
||||||
|
frame_data += chunk
|
||||||
|
bytes_remaining -= len(chunk)
|
||||||
|
|
||||||
|
# Convert raw RGB data to numpy array using actual dimensions
|
||||||
|
frame_array = np.frombuffer(frame_data, dtype=np.uint8)
|
||||||
|
frame_rgb = frame_array.reshape((self.actual_height, self.actual_width, 3))
|
||||||
|
|
||||||
|
# Convert RGB to BGR for OpenCV compatibility
|
||||||
|
frame_bgr = cv2.cvtColor(frame_rgb, cv2.COLOR_RGB2BGR)
|
||||||
|
|
||||||
|
return frame_bgr
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Camera {self.camera_id}: Error reading raw frame: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
def _read_frames(self):
|
def _read_frames(self):
|
||||||
"""Reactively read frames when file changes."""
|
"""Read frames directly from FFmpeg stdout pipe."""
|
||||||
frame_count = 0
|
frame_count = 0
|
||||||
last_log_time = time.time()
|
last_log_time = time.time()
|
||||||
# Remove unused variable: bytes_per_frame = self.width * self.height * 3
|
|
||||||
restart_check_interval = 10 # Check FFmpeg status every 10 seconds
|
|
||||||
|
|
||||||
while not self.stop_event.is_set():
|
while not self.stop_event.is_set():
|
||||||
try:
|
try:
|
||||||
|
|
@ -167,100 +206,45 @@ class FFmpegRTSPReader:
|
||||||
time.sleep(5.0)
|
time.sleep(5.0)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Wait for FFmpeg to start writing frame files
|
logger.info(f"FFmpeg started for camera {self.camera_id}, reading frames from pipe...")
|
||||||
wait_count = 0
|
|
||||||
while wait_count < 30:
|
|
||||||
# Check if any frame files exist
|
|
||||||
import glob
|
|
||||||
frame_files = glob.glob(f"{self.frame_dir}/{self.frame_prefix}*.ppm")
|
|
||||||
if frame_files:
|
|
||||||
logger.info(f"Found {len(frame_files)} initial frame files for {self.camera_id}")
|
|
||||||
break
|
|
||||||
time.sleep(1.0)
|
|
||||||
wait_count += 1
|
|
||||||
|
|
||||||
if wait_count >= 30:
|
# Read frames directly from FFmpeg stdout
|
||||||
logger.error(f"No frame files created after 30s for {self.camera_id}")
|
try:
|
||||||
logger.error(f"Expected pattern: {self.frame_dir}/{self.frame_prefix}*.ppm")
|
if self.process and self.process.stdout:
|
||||||
continue
|
# Read raw frame data
|
||||||
|
frame = self._read_raw_frame(self.process.stdout)
|
||||||
|
if frame is None:
|
||||||
|
continue
|
||||||
|
|
||||||
# Setup file watcher
|
# Call frame callback
|
||||||
self._setup_file_watcher()
|
if self.frame_callback:
|
||||||
|
self.frame_callback(self.camera_id, frame)
|
||||||
|
logger.debug(f"Camera {self.camera_id}: Called frame callback with shape {frame.shape}")
|
||||||
|
|
||||||
# Wait for file change event (or timeout for health check)
|
frame_count += 1
|
||||||
if self.frame_ready_event.wait(timeout=restart_check_interval):
|
|
||||||
self.frame_ready_event.clear()
|
|
||||||
|
|
||||||
# Read latest complete frame file
|
# Log progress
|
||||||
try:
|
current_time = time.time()
|
||||||
import glob
|
if current_time - last_log_time >= 30:
|
||||||
# Find all frame files for this camera
|
logger.info(f"Camera {self.camera_id}: {frame_count} frames processed via pipe")
|
||||||
frame_files = glob.glob(f"{self.frame_dir}/{self.frame_prefix}*.ppm")
|
last_log_time = current_time
|
||||||
|
|
||||||
if frame_files:
|
except Exception as e:
|
||||||
# Sort by filename (which includes timestamp) and get the latest
|
logger.error(f"Camera {self.camera_id}: Error reading from pipe: {e}")
|
||||||
frame_files.sort()
|
# Process might have died, let it restart on next iteration
|
||||||
latest_frame = frame_files[-1]
|
if self.process:
|
||||||
logger.debug(f"Camera {self.camera_id}: Found {len(frame_files)} frames, processing latest: {latest_frame}")
|
self.process.terminate()
|
||||||
|
self.process = None
|
||||||
# Read the latest frame (it's complete since FFmpeg wrote it atomically)
|
time.sleep(1.0)
|
||||||
frame = cv2.imread(latest_frame)
|
|
||||||
|
|
||||||
if frame is not None:
|
|
||||||
logger.debug(f"Camera {self.camera_id}: Successfully read frame {frame.shape} from {latest_frame}")
|
|
||||||
# Accept any frame dimensions initially for debugging
|
|
||||||
if self.frame_callback:
|
|
||||||
self.frame_callback(self.camera_id, frame)
|
|
||||||
logger.debug(f"Camera {self.camera_id}: Called frame callback")
|
|
||||||
|
|
||||||
frame_count += 1
|
|
||||||
|
|
||||||
# Log progress
|
|
||||||
current_time = time.time()
|
|
||||||
if current_time - last_log_time >= 30:
|
|
||||||
logger.info(f"Camera {self.camera_id}: {frame_count} frames processed")
|
|
||||||
last_log_time = current_time
|
|
||||||
else:
|
|
||||||
logger.warning(f"Camera {self.camera_id}: Failed to read frame from {latest_frame}")
|
|
||||||
|
|
||||||
# Clean up old frame files to prevent disk filling
|
|
||||||
# Keep only the latest 5 frames
|
|
||||||
if len(frame_files) > 5:
|
|
||||||
for old_file in frame_files[:-5]:
|
|
||||||
try:
|
|
||||||
os.remove(old_file)
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
logger.warning(f"Camera {self.camera_id}: No frame files found in {self.frame_dir} with pattern {self.frame_prefix}*.ppm")
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.debug(f"Camera {self.camera_id}: Error reading frames: {e}")
|
|
||||||
pass
|
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Camera {self.camera_id}: Error in reactive frame reading: {e}")
|
logger.error(f"Camera {self.camera_id}: Error in pipe frame reading: {e}")
|
||||||
time.sleep(1.0)
|
time.sleep(1.0)
|
||||||
|
|
||||||
# Cleanup
|
# Cleanup
|
||||||
if self.observer:
|
|
||||||
self.observer.stop()
|
|
||||||
self.observer.join()
|
|
||||||
if self.process:
|
if self.process:
|
||||||
self.process.terminate()
|
self.process.terminate()
|
||||||
# Clean up all frame files for this camera
|
logger.info(f"FFmpeg pipe reader ended for camera {self.camera_id}")
|
||||||
try:
|
|
||||||
if hasattr(self, 'frame_prefix') and hasattr(self, 'frame_dir'):
|
|
||||||
import glob
|
|
||||||
frame_files = glob.glob(f"{self.frame_dir}/{self.frame_prefix}*.ppm")
|
|
||||||
for frame_file in frame_files:
|
|
||||||
try:
|
|
||||||
os.remove(frame_file)
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
logger.info(f"Reactive FFmpeg reader ended for camera {self.camera_id}")
|
|
||||||
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue