MODEL/usda-vision
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Integrate auto-recording feature into USDA Vision Camera System

Browse Source 
- Added instructions for implementing auto-recording functionality in the React app.
- Updated TypeScript interfaces to include new fields for auto-recording status and configuration.
- Created new API endpoints for enabling/disabling auto-recording and retrieving system status.
- Enhanced UI components to display auto-recording status, controls, and error handling.
- Developed a comprehensive Auto-Recording Feature Implementation Guide.
- Implemented a test script for validating auto-recording functionality, including configuration checks and API connectivity.
- Introduced AutoRecordingManager to manage automatic recording based on machine state changes with retry logic.
- Established a retry mechanism for failed recording attempts and integrated status tracking for auto-recording.
This commit is contained in:
Alireza Vaezi
2025-07-29 09:43:14 -04:00
parent 0a26a8046e
commit 0c92b6c277
18 changed files with 1543 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
usda_vision_system/__pycache__/main.cpython-311.pyc
View File

Binary file not shown.

BIN
usda_vision_system/api/__pycache__/models.cpython-311.pyc
View File

Binary file not shown.

BIN
usda_vision_system/api/__pycache__/server.cpython-311.pyc
View File

Binary file not shown.

36
usda_vision_system/api/models.py
View File

@@ -57,6 +57,13 @@ class CameraStatusResponse(BaseModel):
current_recording_file: Optional[str] = None
recording_start_time: Optional[str] = None
# Auto-recording status
auto_recording_enabled: bool = False
auto_recording_active: bool = False
auto_recording_failure_count: int = 0
auto_recording_last_attempt: Optional[str] = None
auto_recording_last_error: Optional[str] = None
class RecordingInfoResponse(BaseModel):
"""Recording information response model"""
@@ -120,6 +127,11 @@ class CameraConfigResponse(BaseModel):
storage_path: str
enabled: bool
# Auto-recording settings
auto_start_recording_enabled: bool
auto_recording_max_retries: int
auto_recording_retry_delay_seconds: int
# Basic settings
exposure_ms: float
gain: float
@@ -173,6 +185,30 @@ class StopRecordingResponse(BaseModel):
duration_seconds: Optional[float] = None
class AutoRecordingConfigRequest(BaseModel):
"""Auto-recording configuration request model"""
enabled: bool
class AutoRecordingConfigResponse(BaseModel):
"""Auto-recording configuration response model"""
success: bool
message: str
camera_name: str
enabled: bool
class AutoRecordingStatusResponse(BaseModel):
"""Auto-recording manager status response model"""
running: bool
auto_recording_enabled: bool
retry_queue: Dict[str, Any]
enabled_cameras: List[str]
class StorageStatsResponse(BaseModel):
"""Storage statistics response model"""

87
usda_vision_system/api/server.py
View File

@@ -66,13 +66,14 @@ class WebSocketManager:
class APIServer:
"""FastAPI server for the USDA Vision Camera System"""
def __init__(self, config: Config, state_manager: StateManager, event_system: EventSystem, camera_manager, mqtt_client, storage_manager: StorageManager):
def __init__(self, config: Config, state_manager: StateManager, event_system: EventSystem, camera_manager, mqtt_client, storage_manager: StorageManager, auto_recording_manager=None):
self.config = config
self.state_manager = state_manager
self.event_system = event_system
self.camera_manager = camera_manager
self.mqtt_client = mqtt_client
self.storage_manager = storage_manager
self.auto_recording_manager = auto_recording_manager
self.logger = logging.getLogger(__name__)
# FastAPI app
@@ -162,7 +163,21 @@ class APIServer:
try:
cameras = self.state_manager.get_all_cameras()
return {
name: CameraStatusResponse(name=camera.name, status=camera.status.value, is_recording=camera.is_recording, last_checked=camera.last_checked.isoformat(), last_error=camera.last_error, device_info=camera.device_info, current_recording_file=camera.current_recording_file, recording_start_time=camera.recording_start_time.isoformat() if camera.recording_start_time else None)
name: CameraStatusResponse(
name=camera.name,
status=camera.status.value,
is_recording=camera.is_recording,
last_checked=camera.last_checked.isoformat(),
last_error=camera.last_error,
device_info=camera.device_info,
current_recording_file=camera.current_recording_file,
recording_start_time=camera.recording_start_time.isoformat() if camera.recording_start_time else None,
auto_recording_enabled=camera.auto_recording_enabled,
auto_recording_active=camera.auto_recording_active,
auto_recording_failure_count=camera.auto_recording_failure_count,
auto_recording_last_attempt=camera.auto_recording_last_attempt.isoformat() if camera.auto_recording_last_attempt else None,
auto_recording_last_error=camera.auto_recording_last_error,
)
for name, camera in cameras.items()
}
except Exception as e:
@@ -471,6 +486,74 @@ class APIServer:
self.logger.error(f"Error reinitializing camera: {e}")
raise HTTPException(status_code=500, detail=str(e))
@self.app.post("/cameras/{camera_name}/auto-recording/enable", response_model=AutoRecordingConfigResponse)
async def enable_auto_recording(camera_name: str):
"""Enable auto-recording for a camera"""
try:
if not self.auto_recording_manager:
raise HTTPException(status_code=503, detail="Auto-recording manager not available")
# Update camera configuration
camera_config = self.config.get_camera_by_name(camera_name)
if not camera_config:
raise HTTPException(status_code=404, detail=f"Camera {camera_name} not found")
camera_config.auto_start_recording_enabled = True
self.config.save_config()
# Update camera status in state manager
camera_info = self.state_manager.get_camera_info(camera_name)
if camera_info:
camera_info.auto_recording_enabled = True
return AutoRecordingConfigResponse(success=True, message=f"Auto-recording enabled for camera {camera_name}", camera_name=camera_name, enabled=True)
except HTTPException:
raise
except Exception as e:
self.logger.error(f"Error enabling auto-recording for camera {camera_name}: {e}")
raise HTTPException(status_code=500, detail=str(e))
@self.app.post("/cameras/{camera_name}/auto-recording/disable", response_model=AutoRecordingConfigResponse)
async def disable_auto_recording(camera_name: str):
"""Disable auto-recording for a camera"""
try:
if not self.auto_recording_manager:
raise HTTPException(status_code=503, detail="Auto-recording manager not available")
# Update camera configuration
camera_config = self.config.get_camera_by_name(camera_name)
if not camera_config:
raise HTTPException(status_code=404, detail=f"Camera {camera_name} not found")
camera_config.auto_start_recording_enabled = False
self.config.save_config()
# Update camera status in state manager
camera_info = self.state_manager.get_camera_info(camera_name)
if camera_info:
camera_info.auto_recording_enabled = False
camera_info.auto_recording_active = False
return AutoRecordingConfigResponse(success=True, message=f"Auto-recording disabled for camera {camera_name}", camera_name=camera_name, enabled=False)
except HTTPException:
raise
except Exception as e:
self.logger.error(f"Error disabling auto-recording for camera {camera_name}: {e}")
raise HTTPException(status_code=500, detail=str(e))
@self.app.get("/auto-recording/status", response_model=AutoRecordingStatusResponse)
async def get_auto_recording_status():
"""Get auto-recording manager status"""
try:
if not self.auto_recording_manager:
raise HTTPException(status_code=503, detail="Auto-recording manager not available")
status = self.auto_recording_manager.get_status()
return AutoRecordingStatusResponse(**status)
except Exception as e:
self.logger.error(f"Error getting auto-recording status: {e}")
raise HTTPException(status_code=500, detail=str(e))
@self.app.get("/recordings", response_model=Dict[str, RecordingInfoResponse])
async def get_recordings():
"""Get all recording sessions"""

BIN
usda_vision_system/core/__pycache__/config.cpython-311.pyc
View File

Binary file not shown.

BIN
usda_vision_system/core/__pycache__/state_manager.cpython-311.pyc
View File

Binary file not shown.

10
usda_vision_system/core/config.py
View File

@@ -40,6 +40,11 @@ class CameraConfig:
target_fps: float = 3.0
enabled: bool = True
# Auto-recording settings
auto_start_recording_enabled: bool = False # Enable automatic recording when machine turns on
auto_recording_max_retries: int = 3 # Maximum retry attempts for failed auto-recording starts
auto_recording_retry_delay_seconds: int = 5 # Delay between retry attempts
# Image Quality Settings
sharpness: int = 100 # 0-200, default 100 (no sharpening)
contrast: int = 100 # 0-200, default 100 (normal contrast)
@@ -86,7 +91,10 @@ class SystemConfig:
api_host: str = "0.0.0.0"
api_port: int = 8000
enable_api: bool = True
timezone: str = "America/New_York" # Atlanta, Georgia timezone
timezone: str = "America/New_York"
# Auto-recording system settings
auto_recording_enabled: bool = True # Global enable/disable for auto-recording feature # Atlanta, Georgia timezone
class Config:

7
usda_vision_system/core/state_manager.py
View File

@@ -77,6 +77,13 @@ class CameraInfo:
current_recording_file: Optional[str] = None
recording_start_time: Optional[datetime] = None
# Auto-recording status
auto_recording_enabled: bool = False
auto_recording_active: bool = False # Whether auto-recording is currently managing this camera
auto_recording_failure_count: int = 0
auto_recording_last_attempt: Optional[datetime] = None
auto_recording_last_error: Optional[str] = None
@dataclass
class RecordingInfo:

136
usda_vision_system/main.py
View File

@@ -19,58 +19,55 @@ from .core.timezone_utils import log_time_info, check_time_sync
from .mqtt.client import MQTTClient
from .camera.manager import CameraManager
from .storage.manager import StorageManager
from .recording.auto_manager import AutoRecordingManager
from .api.server import APIServer
class USDAVisionSystem:
"""Main application coordinator for the USDA Vision Camera System"""
def __init__(self, config_file: Optional[str] = None):
# Load configuration first (basic logging will be used initially)
self.config = Config(config_file)
# Setup comprehensive logging
self.logger_setup = setup_logging(
log_level=self.config.system.log_level,
log_file=self.config.system.log_file
)
self.logger_setup = setup_logging(log_level=self.config.system.log_level, log_file=self.config.system.log_file)
self.logger = logging.getLogger(__name__)
# Setup error tracking and performance monitoring
self.error_tracker = get_error_tracker("main_system")
self.performance_logger = get_performance_logger("main_system")
# Initialize core components
self.state_manager = StateManager()
self.event_system = EventSystem()
# Initialize system components
self.storage_manager = StorageManager(self.config, self.state_manager, self.event_system)
self.mqtt_client = MQTTClient(self.config, self.state_manager, self.event_system)
self.camera_manager = CameraManager(self.config, self.state_manager, self.event_system)
self.api_server = APIServer(
self.config, self.state_manager, self.event_system,
self.camera_manager, self.mqtt_client, self.storage_manager
)
self.auto_recording_manager = AutoRecordingManager(self.config, self.state_manager, self.event_system, self.camera_manager)
self.api_server = APIServer(self.config, self.state_manager, self.event_system, self.camera_manager, self.mqtt_client, self.storage_manager, self.auto_recording_manager)
# System state
self.running = False
self.start_time: Optional[datetime] = None
# Setup signal handlers for graceful shutdown
self._setup_signal_handlers()
self.logger.info("USDA Vision Camera System initialized")
def _setup_signal_handlers(self) -> None:
"""Setup signal handlers for graceful shutdown"""
def signal_handler(signum, frame):
self.logger.info(f"Received signal {signum}, initiating graceful shutdown...")
self.stop()
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
def start(self) -> bool:
"""Start the entire system"""
if self.running:
@@ -86,10 +83,7 @@ class USDAVisionSystem:
log_time_info(self.logger)
sync_info = check_time_sync()
if sync_info["sync_status"] == "out_of_sync":
self.error_tracker.log_warning(
f"System time may be out of sync (difference: {sync_info.get('time_diff_seconds', 'unknown')}s)",
"time_sync_check"
)
self.error_tracker.log_warning(f"System time may be out of sync (difference: {sync_info.get('time_diff_seconds', 'unknown')}s)", "time_sync_check")
elif sync_info["sync_status"] == "synchronized":
self.logger.info("✅ System time is synchronized")
@@ -131,6 +125,17 @@ class USDAVisionSystem:
self.mqtt_client.stop()
return False
# Start auto-recording manager
self.logger.info("Starting auto-recording manager...")
try:
if not self.auto_recording_manager.start():
self.error_tracker.log_warning("Failed to start auto-recording manager", "auto_recording_startup")
else:
self.logger.info("Auto-recording manager started successfully")
except Exception as e:
self.error_tracker.log_error(e, "auto_recording_startup")
self.logger.warning("Auto-recording manager failed to start (continuing without auto-recording)")
# Start API server
self.logger.info("Starting API server...")
try:
@@ -147,11 +152,7 @@ class USDAVisionSystem:
self.state_manager.set_system_started(True)
# Publish system started event
self.event_system.publish(
EventType.SYSTEM_SHUTDOWN, # We don't have SYSTEM_STARTED, using closest
"main_system",
{"action": "started", "timestamp": self.start_time.isoformat()}
)
self.event_system.publish(EventType.SYSTEM_SHUTDOWN, "main_system", {"action": "started", "timestamp": self.start_time.isoformat()}) # We don't have SYSTEM_STARTED, using closest
startup_time = self.performance_logger.end_timer("system_startup")
self.logger.info(f"USDA Vision Camera System started successfully in {startup_time:.2f}s")
@@ -161,89 +162,77 @@ class USDAVisionSystem:
self.error_tracker.log_error(e, "system_startup")
self.stop()
return False
def stop(self) -> None:
"""Stop the entire system gracefully"""
if not self.running:
return
self.logger.info("Stopping USDA Vision Camera System...")
self.running = False
try:
# Update system state
self.state_manager.set_system_started(False)
# Publish system shutdown event
self.event_system.publish(
EventType.SYSTEM_SHUTDOWN,
"main_system",
{"action": "stopping", "timestamp": datetime.now().isoformat()}
)
self.event_system.publish(EventType.SYSTEM_SHUTDOWN, "main_system", {"action": "stopping", "timestamp": datetime.now().isoformat()})
# Stop API server
self.api_server.stop()
# Stop auto-recording manager
self.auto_recording_manager.stop()
# Stop camera manager (this will stop all recordings)
self.camera_manager.stop()
# Stop MQTT client
self.mqtt_client.stop()
# Final cleanup
if self.start_time:
uptime = (datetime.now() - self.start_time).total_seconds()
self.logger.info(f"System uptime: {uptime:.1f} seconds")
self.logger.info("USDA Vision Camera System stopped")
except Exception as e:
self.logger.error(f"Error during system shutdown: {e}")
def run(self) -> None:
"""Run the system (blocking call)"""
if not self.start():
self.logger.error("Failed to start system")
return
try:
self.logger.info("System running... Press Ctrl+C to stop")
# Main loop - just keep the system alive
while self.running:
time.sleep(1)
# Periodic maintenance tasks could go here
# For example: cleanup old recordings, health checks, etc.
except KeyboardInterrupt:
self.logger.info("Keyboard interrupt received")
except Exception as e:
self.logger.error(f"Unexpected error in main loop: {e}")
finally:
self.stop()
def get_system_status(self) -> dict:
"""Get comprehensive system status"""
return {
"running": self.running,
"start_time": self.start_time.isoformat() if self.start_time else None,
"uptime_seconds": (datetime.now() - self.start_time).total_seconds() if self.start_time else 0,
"components": {
"mqtt_client": {
"running": self.mqtt_client.is_running(),
"connected": self.mqtt_client.is_connected()
},
"camera_manager": {
"running": self.camera_manager.is_running()
},
"api_server": {
"running": self.api_server.is_running()
}
},
"state_summary": self.state_manager.get_system_summary()
"components": {"mqtt_client": {"running": self.mqtt_client.is_running(), "connected": self.mqtt_client.is_connected()}, "camera_manager": {"running": self.camera_manager.is_running()}, "api_server": {"running": self.api_server.is_running()}},
"state_summary": self.state_manager.get_system_summary(),
}
def is_running(self) -> bool:
"""Check if system is running"""
return self.running
@@ -252,31 +241,20 @@ class USDAVisionSystem:
def main():
"""Main entry point for the application"""
import argparse
parser = argparse.ArgumentParser(description="USDA Vision Camera System")
parser.add_argument(
"--config",
type=str,
help="Path to configuration file",
default="config.json"
)
parser.add_argument(
"--log-level",
type=str,
choices=["DEBUG", "INFO", "WARNING", "ERROR"],
help="Override log level",
default=None
)
parser.add_argument("--config", type=str, help="Path to configuration file", default="config.json")
parser.add_argument("--log-level", type=str, choices=["DEBUG", "INFO", "WARNING", "ERROR"], help="Override log level", default=None)
args = parser.parse_args()
# Create and run system
system = USDAVisionSystem(args.config)
# Override log level if specified
if args.log_level:
logging.getLogger().setLevel(getattr(logging, args.log_level))
try:
system.run()
except Exception as e:

10
usda_vision_system/recording/__init__.py Normal file
View File

@@ -0,0 +1,10 @@
"""
Recording module for the USDA Vision Camera System.
This module contains components for managing automatic recording
based on machine state changes.
"""
from .auto_manager import AutoRecordingManager
__all__ = ["AutoRecordingManager"]

352
usda_vision_system/recording/auto_manager.py Normal file
View File

@@ -0,0 +1,352 @@
"""
Auto-Recording Manager for the USDA Vision Camera System.
This module manages automatic recording start/stop based on machine state changes
received via MQTT. It includes retry logic for failed recording attempts and
tracks auto-recording status for each camera.
"""
import threading
import time
import logging
from typing import Dict, Optional, Any
from datetime import datetime, timedelta
from ..core.config import Config, CameraConfig
from ..core.state_manager import StateManager, MachineState
from ..core.events import EventSystem, EventType, Event
from ..core.timezone_utils import format_filename_timestamp
class AutoRecordingManager:
"""Manages automatic recording based on machine state changes"""
def __init__(self, config: Config, state_manager: StateManager, event_system: EventSystem, camera_manager):
self.config = config
self.state_manager = state_manager
self.event_system = event_system
self.camera_manager = camera_manager
self.logger = logging.getLogger(__name__)
# Threading
self.running = False
self._retry_thread: Optional[threading.Thread] = None
self._stop_event = threading.Event()
# Track retry attempts for each camera
self._retry_queue: Dict[str, Dict[str, Any]] = {} # camera_name -> retry_info
self._retry_lock = threading.RLock()
# Subscribe to machine state change events
self.event_system.subscribe(EventType.MACHINE_STATE_CHANGED, self._on_machine_state_changed)
def start(self) -> bool:
"""Start the auto-recording manager"""
if self.running:
self.logger.warning("Auto-recording manager is already running")
return True
if not self.config.system.auto_recording_enabled:
self.logger.info("Auto-recording is disabled in system configuration")
return True
self.logger.info("Starting auto-recording manager...")
self.running = True
self._stop_event.clear()
# Initialize camera auto-recording status
self._initialize_camera_status()
# Start retry thread
self._retry_thread = threading.Thread(target=self._retry_loop, daemon=True)
self._retry_thread.start()
self.logger.info("Auto-recording manager started successfully")
return True
def stop(self) -> None:
"""Stop the auto-recording manager"""
if not self.running:
return
self.logger.info("Stopping auto-recording manager...")
self.running = False
self._stop_event.set()
# Wait for retry thread to finish
if self._retry_thread and self._retry_thread.is_alive():
self._retry_thread.join(timeout=5)
self.logger.info("Auto-recording manager stopped")
def _initialize_camera_status(self) -> None:
"""Initialize auto-recording status for all cameras"""
for camera_config in self.config.cameras:
if camera_config.enabled and camera_config.auto_start_recording_enabled:
# Update camera status in state manager
camera_info = self.state_manager.get_camera_info(camera_config.name)
if camera_info:
camera_info.auto_recording_enabled = True
self.logger.info(f"Auto-recording enabled for camera {camera_config.name}")
def _on_machine_state_changed(self, event: Event) -> None:
"""Handle machine state change events"""
try:
machine_name = event.data.get("machine_name")
new_state = event.data.get("state")
if not machine_name or not new_state:
return
self.logger.info(f"Machine state changed: {machine_name} -> {new_state}")
# Find cameras associated with this machine
associated_cameras = self._get_cameras_for_machine(machine_name)
for camera_config in associated_cameras:
if not camera_config.enabled or not camera_config.auto_start_recording_enabled:
continue
if new_state.lower() == "on":
self._handle_machine_on(camera_config)
elif new_state.lower() == "off":
self._handle_machine_off(camera_config)
except Exception as e:
self.logger.error(f"Error handling machine state change: {e}")
def _get_cameras_for_machine(self, machine_name: str) -> list[CameraConfig]:
"""Get all cameras associated with a machine topic"""
associated_cameras = []
# Map machine names to topics
machine_topic_map = {
"vibratory_conveyor": "vibratory_conveyor",
"blower_separator": "blower_separator"
}
machine_topic = machine_topic_map.get(machine_name)
if not machine_topic:
return associated_cameras
for camera_config in self.config.cameras:
if camera_config.machine_topic == machine_topic:
associated_cameras.append(camera_config)
return associated_cameras
def _handle_machine_on(self, camera_config: CameraConfig) -> None:
"""Handle machine turning on - start recording"""
camera_name = camera_config.name
# Check if camera is already recording
camera_info = self.state_manager.get_camera_info(camera_name)
if camera_info and camera_info.is_recording:
self.logger.info(f"Camera {camera_name} is already recording, skipping auto-start")
return
self.logger.info(f"Machine turned ON - attempting to start recording for camera {camera_name}")
# Update auto-recording status
if camera_info:
camera_info.auto_recording_active = True
camera_info.auto_recording_last_attempt = datetime.now()
# Attempt to start recording
success = self._start_recording_for_camera(camera_config)
if not success:
# Add to retry queue
self._add_to_retry_queue(camera_config, "start")
def _handle_machine_off(self, camera_config: CameraConfig) -> None:
"""Handle machine turning off - stop recording"""
camera_name = camera_config.name
self.logger.info(f"Machine turned OFF - attempting to stop recording for camera {camera_name}")
# Update auto-recording status
camera_info = self.state_manager.get_camera_info(camera_name)
if camera_info:
camera_info.auto_recording_active = False
# Remove from retry queue if present
with self._retry_lock:
if camera_name in self._retry_queue:
del self._retry_queue[camera_name]
# Attempt to stop recording
self._stop_recording_for_camera(camera_config)
def _start_recording_for_camera(self, camera_config: CameraConfig) -> bool:
"""Start recording for a specific camera"""
try:
camera_name = camera_config.name
# Generate filename with timestamp and machine info
timestamp = format_filename_timestamp()
machine_name = camera_config.machine_topic.replace("_", "-")
filename = f"{camera_name}_auto_{machine_name}_{timestamp}.avi"
# Use camera manager to start recording
success = self.camera_manager.manual_start_recording(camera_name, filename)
if success:
self.logger.info(f"Successfully started auto-recording for camera {camera_name}: {filename}")
# Update status
camera_info = self.state_manager.get_camera_info(camera_name)
if camera_info:
camera_info.auto_recording_failure_count = 0
camera_info.auto_recording_last_error = None
return True
else:
self.logger.error(f"Failed to start auto-recording for camera {camera_name}")
return False
except Exception as e:
self.logger.error(f"Error starting auto-recording for camera {camera_config.name}: {e}")
# Update error status
camera_info = self.state_manager.get_camera_info(camera_config.name)
if camera_info:
camera_info.auto_recording_last_error = str(e)
return False
def _stop_recording_for_camera(self, camera_config: CameraConfig) -> bool:
"""Stop recording for a specific camera"""
try:
camera_name = camera_config.name
# Use camera manager to stop recording
success = self.camera_manager.manual_stop_recording(camera_name)
if success:
self.logger.info(f"Successfully stopped auto-recording for camera {camera_name}")
return True
else:
self.logger.warning(f"Failed to stop auto-recording for camera {camera_name} (may not have been recording)")
return False
except Exception as e:
self.logger.error(f"Error stopping auto-recording for camera {camera_config.name}: {e}")
return False
def _add_to_retry_queue(self, camera_config: CameraConfig, action: str) -> None:
"""Add a camera to the retry queue"""
with self._retry_lock:
camera_name = camera_config.name
retry_info = {
"camera_config": camera_config,
"action": action,
"attempt_count": 0,
"next_retry_time": datetime.now() + timedelta(seconds=camera_config.auto_recording_retry_delay_seconds),
"max_retries": camera_config.auto_recording_max_retries
}
self._retry_queue[camera_name] = retry_info
self.logger.info(f"Added camera {camera_name} to retry queue for {action} (max retries: {retry_info['max_retries']})")
def _retry_loop(self) -> None:
"""Background thread to handle retry attempts"""
while self.running and not self._stop_event.is_set():
try:
current_time = datetime.now()
cameras_to_retry = []
# Find cameras ready for retry
with self._retry_lock:
for camera_name, retry_info in list(self._retry_queue.items()):
if current_time >= retry_info["next_retry_time"]:
cameras_to_retry.append((camera_name, retry_info))
# Process retries
for camera_name, retry_info in cameras_to_retry:
self._process_retry(camera_name, retry_info)
# Sleep for a short interval
self._stop_event.wait(1)
except Exception as e:
self.logger.error(f"Error in retry loop: {e}")
self._stop_event.wait(5)
def _process_retry(self, camera_name: str, retry_info: Dict[str, Any]) -> None:
"""Process a retry attempt for a camera"""
try:
retry_info["attempt_count"] += 1
camera_config = retry_info["camera_config"]
action = retry_info["action"]
self.logger.info(f"Retry attempt {retry_info['attempt_count']}/{retry_info['max_retries']} for camera {camera_name} ({action})")
# Update camera status
camera_info = self.state_manager.get_camera_info(camera_name)
if camera_info:
camera_info.auto_recording_last_attempt = datetime.now()
camera_info.auto_recording_failure_count = retry_info["attempt_count"]
# Attempt the action
success = False
if action == "start":
success = self._start_recording_for_camera(camera_config)
if success:
# Success - remove from retry queue
with self._retry_lock:
if camera_name in self._retry_queue:
del self._retry_queue[camera_name]
self.logger.info(f"Retry successful for camera {camera_name}")
else:
# Failed - check if we should retry again
if retry_info["attempt_count"] >= retry_info["max_retries"]:
# Max retries reached
with self._retry_lock:
if camera_name in self._retry_queue:
del self._retry_queue[camera_name]
error_msg = f"Max retry attempts ({retry_info['max_retries']}) reached for camera {camera_name}"
self.logger.error(error_msg)
# Update camera status
if camera_info:
camera_info.auto_recording_last_error = error_msg
camera_info.auto_recording_active = False
else:
# Schedule next retry
retry_info["next_retry_time"] = datetime.now() + timedelta(seconds=camera_config.auto_recording_retry_delay_seconds)
self.logger.info(f"Scheduling next retry for camera {camera_name} in {camera_config.auto_recording_retry_delay_seconds} seconds")
except Exception as e:
self.logger.error(f"Error processing retry for camera {camera_name}: {e}")
# Remove from retry queue on error
with self._retry_lock:
if camera_name in self._retry_queue:
del self._retry_queue[camera_name]
def get_status(self) -> Dict[str, Any]:
"""Get auto-recording manager status"""
with self._retry_lock:
retry_queue_status = {
camera_name: {
"action": info["action"],
"attempt_count": info["attempt_count"],
"max_retries": info["max_retries"],
"next_retry_time": info["next_retry_time"].isoformat()
}
for camera_name, info in self._retry_queue.items()
}
return {
"running": self.running,
"auto_recording_enabled": self.config.system.auto_recording_enabled,
"retry_queue": retry_queue_status,
"enabled_cameras": [
camera.name for camera in self.config.cameras
if camera.enabled and camera.auto_start_recording_enabled
]
}