event driven system
This commit is contained in:
parent
0c5f56c8a6
commit
3a47920186
10 changed files with 782 additions and 253 deletions
91
scripts/decoder_test.py
Normal file
91
scripts/decoder_test.py
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
"""
|
||||
Test decoder frame rate in isolation without any processing.
|
||||
"""
|
||||
|
||||
import time
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
|
||||
import sys
|
||||
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
||||
|
||||
from services.stream_decoder import StreamDecoderFactory
|
||||
|
||||
load_dotenv()
|
||||
|
||||
def main():
|
||||
GPU_ID = 0
|
||||
STREAM_URL = os.getenv('CAMERA_URL_1', 'rtsp://localhost:8554/test')
|
||||
MAX_FRAMES = 100
|
||||
|
||||
print("=" * 80)
|
||||
print("Decoder Frame Rate Test (No Processing)")
|
||||
print("=" * 80)
|
||||
print(f"\nStream: {STREAM_URL}")
|
||||
print(f"Monitoring for {MAX_FRAMES} frames...\n")
|
||||
|
||||
# Create decoder
|
||||
factory = StreamDecoderFactory(gpu_id=GPU_ID)
|
||||
decoder = factory.create_decoder(STREAM_URL, buffer_size=30)
|
||||
|
||||
# Start decoder
|
||||
decoder.start()
|
||||
|
||||
# Wait for connection
|
||||
print("Waiting for connection...")
|
||||
max_wait = 10
|
||||
waited = 0
|
||||
while not decoder.is_connected() and waited < max_wait:
|
||||
time.sleep(0.5)
|
||||
waited += 0.5
|
||||
|
||||
if not decoder.is_connected():
|
||||
print(f"Failed to connect after {max_wait}s!")
|
||||
decoder.stop()
|
||||
return
|
||||
|
||||
print(f"✓ Connected\n")
|
||||
print("Monitoring frame arrivals...")
|
||||
print("-" * 60)
|
||||
|
||||
last_count = 0
|
||||
frame_times = []
|
||||
start_time = time.time()
|
||||
last_frame_time = start_time
|
||||
|
||||
while decoder.get_frame_count() < MAX_FRAMES:
|
||||
current_count = decoder.get_frame_count()
|
||||
|
||||
if current_count > last_count:
|
||||
current_time = time.time()
|
||||
interval = (current_time - last_frame_time) * 1000
|
||||
|
||||
frame_times.append(interval)
|
||||
print(f"Frame {current_count:3d}: interval={interval:6.1f}ms")
|
||||
|
||||
last_count = current_count
|
||||
last_frame_time = current_time
|
||||
|
||||
time.sleep(0.001) # 1ms poll
|
||||
|
||||
# Stop decoder
|
||||
decoder.stop()
|
||||
|
||||
# Analysis
|
||||
elapsed = time.time() - start_time
|
||||
actual_fps = MAX_FRAMES / elapsed
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print("DECODER PERFORMANCE")
|
||||
print("=" * 80)
|
||||
print(f"\nFrames received: {MAX_FRAMES}")
|
||||
print(f"Time: {elapsed:.1f}s")
|
||||
print(f"Actual FPS: {actual_fps:.2f}")
|
||||
print(f"\nFrame Intervals:")
|
||||
print(f" Min: {min(frame_times[1:]):.1f}ms") # Skip first
|
||||
print(f" Max: {max(frame_times[1:]):.1f}ms")
|
||||
print(f" Avg: {sum(frame_times[1:])/len(frame_times[1:]):.1f}ms")
|
||||
print(f" Expected (6 FPS): 166.7ms")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
165
scripts/detailed_profiling.py
Normal file
165
scripts/detailed_profiling.py
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
"""
|
||||
Detailed profiling with timing instrumentation to find the exact bottleneck.
|
||||
|
||||
This script adds detailed timing logs at each stage of the pipeline.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
import os
|
||||
import torch
|
||||
from dotenv import load_dotenv
|
||||
from collections import defaultdict
|
||||
|
||||
import sys
|
||||
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
||||
|
||||
from services import (
|
||||
StreamConnectionManager,
|
||||
YOLOv8Utils,
|
||||
)
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# Timing statistics
|
||||
timings = defaultdict(list)
|
||||
frame_timestamps = {}
|
||||
|
||||
def log_timing(event, frame_id=None, extra_data=None):
|
||||
"""Log timing event"""
|
||||
timestamp = time.time()
|
||||
timings[event].append(timestamp)
|
||||
if frame_id is not None:
|
||||
if frame_id not in frame_timestamps:
|
||||
frame_timestamps[frame_id] = {}
|
||||
frame_timestamps[frame_id][event] = timestamp
|
||||
if extra_data:
|
||||
frame_timestamps[frame_id].update(extra_data)
|
||||
|
||||
async def instrumented_main():
|
||||
"""Instrumented version of profiling script"""
|
||||
print("=" * 80)
|
||||
print("Detailed Profiling: Event-Driven GPU-Accelerated Object Tracking")
|
||||
print("=" * 80)
|
||||
|
||||
# Configuration
|
||||
GPU_ID = 0
|
||||
MODEL_PATH = "bangchak/models/frontal_detection_v5.pt"
|
||||
STREAM_URL = os.getenv('CAMERA_URL_1', 'rtsp://localhost:8554/test')
|
||||
BATCH_SIZE = 4
|
||||
FORCE_TIMEOUT = 0.05
|
||||
MAX_FRAMES = 50 # Fewer frames for detailed analysis
|
||||
|
||||
print(f"\nConfiguration:")
|
||||
print(f" GPU: {GPU_ID}")
|
||||
print(f" Model: {MODEL_PATH}")
|
||||
print(f" Stream: {STREAM_URL}")
|
||||
print(f" Batch size: {BATCH_SIZE}")
|
||||
print(f" Max frames: {MAX_FRAMES}\n")
|
||||
|
||||
# Create manager
|
||||
print("[1/3] Creating StreamConnectionManager...")
|
||||
manager = StreamConnectionManager(
|
||||
gpu_id=GPU_ID,
|
||||
batch_size=BATCH_SIZE,
|
||||
force_timeout=FORCE_TIMEOUT,
|
||||
enable_pt_conversion=True
|
||||
)
|
||||
print("✓ Manager created")
|
||||
|
||||
# Initialize
|
||||
print("\n[2/3] Initializing...")
|
||||
await manager.initialize(
|
||||
model_path=MODEL_PATH,
|
||||
model_id="detector",
|
||||
preprocess_fn=YOLOv8Utils.preprocess,
|
||||
postprocess_fn=YOLOv8Utils.postprocess,
|
||||
num_contexts=4,
|
||||
pt_input_shapes={"images": (1, 3, 640, 640)},
|
||||
pt_precision=torch.float16
|
||||
)
|
||||
print("✓ Initialized")
|
||||
|
||||
# Connect stream
|
||||
print("\n[3/3] Connecting to stream...")
|
||||
connection = await manager.connect_stream(
|
||||
rtsp_url=STREAM_URL,
|
||||
stream_id="camera_1",
|
||||
buffer_size=30
|
||||
)
|
||||
print("✓ Connected\n")
|
||||
|
||||
print(f"{'=' * 80}")
|
||||
print(f"Running instrumented profiling for {MAX_FRAMES} frames...")
|
||||
print(f"{'=' * 80}\n")
|
||||
|
||||
result_count = 0
|
||||
start_time = time.time()
|
||||
last_result_time = start_time
|
||||
|
||||
try:
|
||||
async for result in connection.tracking_results():
|
||||
current_time = time.time()
|
||||
result_interval = (current_time - last_result_time) * 1000
|
||||
|
||||
result_count += 1
|
||||
frame_id = result_count
|
||||
|
||||
log_timing('result_received', frame_id, {
|
||||
'interval_ms': result_interval,
|
||||
'num_objects': len(result.tracked_objects),
|
||||
'num_detections': len(result.detections)
|
||||
})
|
||||
|
||||
print(f"Frame {result_count:3d}: interval={result_interval:6.1f}ms, "
|
||||
f"objects={len(result.tracked_objects):2d}, "
|
||||
f"detections={len(result.detections):2d}")
|
||||
|
||||
last_result_time = current_time
|
||||
|
||||
if result_count >= MAX_FRAMES:
|
||||
print(f"\n✓ Reached max frames limit ({MAX_FRAMES})")
|
||||
break
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print(f"\n✓ Interrupted by user")
|
||||
|
||||
# Cleanup
|
||||
print(f"\n{'=' * 80}")
|
||||
print("Cleanup")
|
||||
print(f"{'=' * 80}")
|
||||
await connection.stop()
|
||||
await manager.shutdown()
|
||||
print("✓ Stopped")
|
||||
|
||||
# Analysis
|
||||
elapsed = time.time() - start_time
|
||||
avg_fps = result_count / elapsed if elapsed > 0 else 0
|
||||
|
||||
print(f"\n{'=' * 80}")
|
||||
print("TIMING ANALYSIS")
|
||||
print(f"{'=' * 80}")
|
||||
print(f"\nOverall:")
|
||||
print(f" Results: {result_count}")
|
||||
print(f" Time: {elapsed:.1f}s")
|
||||
print(f" FPS: {avg_fps:.2f}")
|
||||
|
||||
# Frame intervals
|
||||
if len(frame_timestamps) > 1:
|
||||
intervals = []
|
||||
for i in range(2, result_count + 1):
|
||||
if i in frame_timestamps and (i-1) in frame_timestamps:
|
||||
interval = (frame_timestamps[i]['result_received'] -
|
||||
frame_timestamps[i-1]['result_received']) * 1000
|
||||
intervals.append(interval)
|
||||
|
||||
if intervals:
|
||||
print(f"\nFrame Intervals:")
|
||||
print(f" Min: {min(intervals):.1f}ms")
|
||||
print(f" Max: {max(intervals):.1f}ms")
|
||||
print(f" Avg: {sum(intervals)/len(intervals):.1f}ms")
|
||||
print(f" Expected (6 FPS): 166.7ms")
|
||||
print(f" Deviation: {(sum(intervals)/len(intervals) - 166.7):.1f}ms")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(instrumented_main())
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
"""
|
||||
Profiling script for the real-time object tracking pipeline.
|
||||
|
||||
|
|
|
|||
149
scripts/timing_instrumentation.py
Normal file
149
scripts/timing_instrumentation.py
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
"""
|
||||
Add timing instrumentation to track where time is spent in the pipeline.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
import os
|
||||
import torch
|
||||
from dotenv import load_dotenv
|
||||
import logging
|
||||
|
||||
import sys
|
||||
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
||||
|
||||
# Monkey patch to add timing
|
||||
original_handle_result = None
|
||||
original_run_tracking = None
|
||||
original_infer = None
|
||||
|
||||
timings = []
|
||||
|
||||
def patch_timing():
|
||||
"""Add timing instrumentation to key functions"""
|
||||
from services import stream_connection_manager, model_repository
|
||||
|
||||
global original_handle_result, original_run_tracking, original_infer
|
||||
|
||||
# Patch _handle_inference_result
|
||||
original_handle_result = stream_connection_manager.StreamConnection._handle_inference_result
|
||||
async def timed_handle_result(self, result):
|
||||
t0 = time.perf_counter()
|
||||
await original_handle_result(self, result)
|
||||
t1 = time.perf_counter()
|
||||
timings.append(('handle_result', (t1 - t0) * 1000))
|
||||
stream_connection_manager.StreamConnection._handle_inference_result = timed_handle_result
|
||||
|
||||
# Patch _run_tracking_sync
|
||||
original_run_tracking = stream_connection_manager.StreamConnection._run_tracking_sync
|
||||
def timed_run_tracking(self, detections, min_confidence=0.7):
|
||||
t0 = time.perf_counter()
|
||||
result = original_run_tracking(self, detections, min_confidence)
|
||||
t1 = time.perf_counter()
|
||||
timings.append(('tracking', (t1 - t0) * 1000))
|
||||
return result
|
||||
stream_connection_manager.StreamConnection._run_tracking_sync = timed_run_tracking
|
||||
|
||||
# Patch infer
|
||||
original_infer = model_repository.TensorRTModelRepository.infer
|
||||
def timed_infer(self, model_id, inputs, synchronize=True):
|
||||
t0 = time.perf_counter()
|
||||
result = original_infer(self, model_id, inputs, synchronize)
|
||||
t1 = time.perf_counter()
|
||||
timings.append(('infer', (t1 - t0) * 1000))
|
||||
return result
|
||||
model_repository.TensorRTModelRepository.infer = timed_infer
|
||||
|
||||
async def instrumented_main():
|
||||
"""Instrumented profiling"""
|
||||
from services import StreamConnectionManager, YOLOv8Utils
|
||||
|
||||
load_dotenv()
|
||||
|
||||
print("=" * 80)
|
||||
print("Timing Instrumentation")
|
||||
print("=" * 80)
|
||||
|
||||
# Patch before creating manager
|
||||
patch_timing()
|
||||
|
||||
# Configuration
|
||||
GPU_ID = 0
|
||||
MODEL_PATH = "bangchak/models/frontal_detection_v5.pt"
|
||||
STREAM_URL = os.getenv('CAMERA_URL_1', 'rtsp://localhost:8554/test')
|
||||
BATCH_SIZE = 4
|
||||
FORCE_TIMEOUT = 0.05
|
||||
MAX_FRAMES = 30
|
||||
|
||||
print(f"\nConfiguration: GPU={GPU_ID}, BATCH={BATCH_SIZE}, MAX={MAX_FRAMES}\n")
|
||||
|
||||
# Create and initialize manager
|
||||
manager = StreamConnectionManager(
|
||||
gpu_id=GPU_ID,
|
||||
batch_size=BATCH_SIZE,
|
||||
force_timeout=FORCE_TIMEOUT,
|
||||
enable_pt_conversion=True
|
||||
)
|
||||
|
||||
await manager.initialize(
|
||||
model_path=MODEL_PATH,
|
||||
model_id="detector",
|
||||
preprocess_fn=YOLOv8Utils.preprocess,
|
||||
postprocess_fn=YOLOv8Utils.postprocess,
|
||||
num_contexts=4,
|
||||
pt_input_shapes={"images": (1, 3, 640, 640)},
|
||||
pt_precision=torch.float16
|
||||
)
|
||||
|
||||
connection = await manager.connect_stream(
|
||||
rtsp_url=STREAM_URL,
|
||||
stream_id="camera_1",
|
||||
buffer_size=30
|
||||
)
|
||||
print("✓ Connected\n")
|
||||
|
||||
print(f"{'=' * 80}")
|
||||
print(f"Processing {MAX_FRAMES} frames with timing...")
|
||||
print(f"{'=' * 80}\n")
|
||||
|
||||
result_count = 0
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
async for result in connection.tracking_results():
|
||||
result_count += 1
|
||||
if result_count >= MAX_FRAMES:
|
||||
break
|
||||
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
|
||||
# Cleanup
|
||||
await connection.stop()
|
||||
await manager.shutdown()
|
||||
|
||||
# Analysis
|
||||
elapsed = time.time() - start_time
|
||||
print(f"\nProcessed {result_count} frames in {elapsed:.1f}s ({result_count/elapsed:.2f} FPS)\n")
|
||||
|
||||
# Analyze timings
|
||||
from collections import defaultdict
|
||||
timing_stats = defaultdict(list)
|
||||
for operation, duration in timings:
|
||||
timing_stats[operation].append(duration)
|
||||
|
||||
print("=" * 80)
|
||||
print("TIMING BREAKDOWN")
|
||||
print("=" * 80)
|
||||
for operation in ['infer', 'tracking', 'handle_result']:
|
||||
if operation in timing_stats:
|
||||
times = timing_stats[operation]
|
||||
print(f"\n{operation}:")
|
||||
print(f" Calls: {len(times)}")
|
||||
print(f" Min: {min(times):.2f}ms")
|
||||
print(f" Max: {max(times):.2f}ms")
|
||||
print(f" Avg: {sum(times)/len(times):.2f}ms")
|
||||
print(f" Total: {sum(times):.2f}ms")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(instrumented_main())
|
||||
Loading…
Add table
Add a link
Reference in a new issue