feat: Add MQTT publisher and tester scripts for USDA Vision Camera System
- Implemented mqtt_publisher_test.py for manual MQTT message publishing - Created mqtt_test.py to test MQTT message reception and display statistics - Developed test_api_changes.py to verify API changes for camera settings and filename handling - Added test_camera_recovery_api.py for testing camera recovery API endpoints - Introduced test_max_fps.py to demonstrate maximum FPS capture functionality - Implemented test_mqtt_events_api.py to test MQTT events API endpoint - Created test_mqtt_logging.py for enhanced MQTT logging and API endpoint testing - Added sdk_config.py for SDK initialization and configuration with error suppression
This commit is contained in:
Alireza Vaezi
Binary file not shown.
@@ -19,7 +19,7 @@ from ..core.events import EventSystem, EventType, Event
|
||||
|
||||
class StorageManager:
|
||||
"""Manages storage and file organization for recorded videos"""
|
||||
|
||||
|
||||
def __init__(self, config: Config, state_manager: StateManager, event_system: Optional[EventSystem] = None):
|
||||
self.config = config
|
||||
self.storage_config = config.storage
|
||||
@@ -37,20 +37,20 @@ class StorageManager:
|
||||
# Subscribe to recording events if event system is available
|
||||
if self.event_system:
|
||||
self._setup_event_subscriptions()
|
||||
|
||||
|
||||
def _ensure_storage_structure(self) -> None:
|
||||
"""Ensure storage directory structure exists"""
|
||||
try:
|
||||
# Create base storage directory
|
||||
Path(self.storage_config.base_path).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
# Create camera-specific directories
|
||||
for camera_config in self.config.cameras:
|
||||
Path(camera_config.storage_path).mkdir(parents=True, exist_ok=True)
|
||||
self.logger.debug(f"Ensured storage directory: {camera_config.storage_path}")
|
||||
|
||||
|
||||
self.logger.info("Storage directory structure verified")
|
||||
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error creating storage structure: {e}")
|
||||
raise
|
||||
@@ -66,12 +66,7 @@ class StorageManager:
|
||||
camera_name = event.data.get("camera_name")
|
||||
filename = event.data.get("filename")
|
||||
if camera_name and filename:
|
||||
self.register_recording_file(
|
||||
camera_name=camera_name,
|
||||
filename=filename,
|
||||
start_time=event.timestamp,
|
||||
machine_trigger=event.data.get("machine_trigger")
|
||||
)
|
||||
self.register_recording_file(camera_name=camera_name, filename=filename, start_time=event.timestamp, machine_trigger=event.data.get("machine_trigger"))
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error handling recording started event: {e}")
|
||||
|
||||
@@ -81,64 +76,48 @@ class StorageManager:
|
||||
filename = event.data.get("filename")
|
||||
if filename:
|
||||
file_id = os.path.basename(filename)
|
||||
self.finalize_recording_file(
|
||||
file_id=file_id,
|
||||
end_time=event.timestamp,
|
||||
duration_seconds=event.data.get("duration_seconds")
|
||||
)
|
||||
self.finalize_recording_file(file_id=file_id, end_time=event.timestamp, duration_seconds=event.data.get("duration_seconds"))
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error handling recording stopped event: {e}")
|
||||
|
||||
# Subscribe to recording events
|
||||
self.event_system.subscribe(EventType.RECORDING_STARTED, on_recording_started)
|
||||
self.event_system.subscribe(EventType.RECORDING_STOPPED, on_recording_stopped)
|
||||
|
||||
|
||||
def _load_file_index(self) -> Dict[str, Any]:
|
||||
"""Load file index from disk"""
|
||||
try:
|
||||
if os.path.exists(self.file_index_path):
|
||||
with open(self.file_index_path, 'r') as f:
|
||||
with open(self.file_index_path, "r") as f:
|
||||
return json.load(f)
|
||||
else:
|
||||
return {"files": {}, "last_updated": None}
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error loading file index: {e}")
|
||||
return {"files": {}, "last_updated": None}
|
||||
|
||||
|
||||
def _save_file_index(self) -> None:
|
||||
"""Save file index to disk"""
|
||||
try:
|
||||
self.file_index["last_updated"] = datetime.now().isoformat()
|
||||
with open(self.file_index_path, 'w') as f:
|
||||
with open(self.file_index_path, "w") as f:
|
||||
json.dump(self.file_index, f, indent=2)
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error saving file index: {e}")
|
||||
|
||||
def register_recording_file(self, camera_name: str, filename: str, start_time: datetime,
|
||||
machine_trigger: Optional[str] = None) -> str:
|
||||
|
||||
def register_recording_file(self, camera_name: str, filename: str, start_time: datetime, machine_trigger: Optional[str] = None) -> str:
|
||||
"""Register a new recording file"""
|
||||
try:
|
||||
file_id = os.path.basename(filename)
|
||||
|
||||
file_info = {
|
||||
"camera_name": camera_name,
|
||||
"filename": filename,
|
||||
"file_id": file_id,
|
||||
"start_time": start_time.isoformat(),
|
||||
"end_time": None,
|
||||
"file_size_bytes": None,
|
||||
"duration_seconds": None,
|
||||
"machine_trigger": machine_trigger,
|
||||
"status": "recording",
|
||||
"created_at": datetime.now().isoformat()
|
||||
}
|
||||
|
||||
|
||||
file_info = {"camera_name": camera_name, "filename": filename, "file_id": file_id, "start_time": start_time.isoformat(), "end_time": None, "file_size_bytes": None, "duration_seconds": None, "machine_trigger": machine_trigger, "status": "recording", "created_at": datetime.now().isoformat()}
|
||||
|
||||
self.file_index["files"][file_id] = file_info
|
||||
self._save_file_index()
|
||||
|
||||
|
||||
self.logger.info(f"Registered recording file: {file_id}")
|
||||
return file_id
|
||||
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error registering recording file: {e}")
|
||||
return ""
|
||||
@@ -169,52 +148,50 @@ class StorageManager:
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error finalizing recording file: {e}")
|
||||
return False
|
||||
|
||||
def finalize_recording_file(self, file_id: str, end_time: datetime,
|
||||
duration_seconds: float, frame_count: Optional[int] = None) -> bool:
|
||||
|
||||
def finalize_recording_file(self, file_id: str, end_time: datetime, duration_seconds: float, frame_count: Optional[int] = None) -> bool:
|
||||
"""Finalize a recording file after recording stops"""
|
||||
try:
|
||||
if file_id not in self.file_index["files"]:
|
||||
self.logger.warning(f"File ID not found in index: {file_id}")
|
||||
return False
|
||||
|
||||
|
||||
file_info = self.file_index["files"][file_id]
|
||||
filename = file_info["filename"]
|
||||
|
||||
|
||||
# Update file information
|
||||
file_info["end_time"] = end_time.isoformat()
|
||||
file_info["duration_seconds"] = duration_seconds
|
||||
file_info["status"] = "completed"
|
||||
|
||||
|
||||
# Get file size if file exists
|
||||
if os.path.exists(filename):
|
||||
file_info["file_size_bytes"] = os.path.getsize(filename)
|
||||
|
||||
|
||||
if frame_count is not None:
|
||||
file_info["frame_count"] = frame_count
|
||||
|
||||
|
||||
self._save_file_index()
|
||||
|
||||
|
||||
self.logger.info(f"Finalized recording file: {file_id} (duration: {duration_seconds:.1f}s)")
|
||||
return True
|
||||
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error finalizing recording file: {e}")
|
||||
return False
|
||||
|
||||
def get_recording_files(self, camera_name: Optional[str] = None,
|
||||
start_date: Optional[datetime] = None,
|
||||
end_date: Optional[datetime] = None,
|
||||
limit: Optional[int] = None) -> List[Dict[str, Any]]:
|
||||
|
||||
def get_recording_files(self, camera_name: Optional[str] = None, start_date: Optional[datetime] = None, end_date: Optional[datetime] = None, limit: Optional[int] = None) -> List[Dict[str, Any]]:
|
||||
"""Get list of recording files with optional filters"""
|
||||
try:
|
||||
files = []
|
||||
|
||||
|
||||
# First, get files from the index (if available)
|
||||
indexed_files = set()
|
||||
for file_id, file_info in self.file_index["files"].items():
|
||||
# Filter by camera name
|
||||
if camera_name and file_info["camera_name"] != camera_name:
|
||||
continue
|
||||
|
||||
|
||||
# Filter by date range
|
||||
if start_date or end_date:
|
||||
file_start = datetime.fromisoformat(file_info["start_time"])
|
||||
@@ -222,88 +199,106 @@ class StorageManager:
|
||||
continue
|
||||
if end_date and file_start > end_date:
|
||||
continue
|
||||
|
||||
|
||||
files.append(file_info.copy())
|
||||
|
||||
indexed_files.add(file_info["filename"])
|
||||
|
||||
# Then, scan filesystem for files not in the index
|
||||
for camera_config in self.config.cameras:
|
||||
# Skip if filtering by camera name and this isn't the one
|
||||
if camera_name and camera_config.name != camera_name:
|
||||
continue
|
||||
|
||||
storage_path = Path(camera_config.storage_path)
|
||||
if storage_path.exists():
|
||||
for video_file in storage_path.glob("*.avi"):
|
||||
if video_file.is_file() and str(video_file) not in indexed_files:
|
||||
# Get file stats
|
||||
stat = video_file.stat()
|
||||
file_mtime = datetime.fromtimestamp(stat.st_mtime)
|
||||
|
||||
# Apply date filters
|
||||
if start_date and file_mtime < start_date:
|
||||
continue
|
||||
if end_date and file_mtime > end_date:
|
||||
continue
|
||||
|
||||
# Create file info for unindexed file
|
||||
file_info = {"camera_name": camera_config.name, "filename": str(video_file), "file_id": video_file.name, "start_time": file_mtime.isoformat(), "end_time": None, "file_size_bytes": stat.st_size, "duration_seconds": None, "machine_trigger": None, "status": "unknown", "created_at": file_mtime.isoformat()} # We don't know if it's completed or not
|
||||
files.append(file_info)
|
||||
|
||||
# Sort by start time (newest first)
|
||||
files.sort(key=lambda x: x["start_time"], reverse=True)
|
||||
|
||||
|
||||
# Apply limit
|
||||
if limit:
|
||||
files = files[:limit]
|
||||
|
||||
|
||||
return files
|
||||
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error getting recording files: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def get_storage_statistics(self) -> Dict[str, Any]:
|
||||
"""Get storage usage statistics"""
|
||||
try:
|
||||
stats = {
|
||||
"base_path": self.storage_config.base_path,
|
||||
"total_files": 0,
|
||||
"total_size_bytes": 0,
|
||||
"cameras": {},
|
||||
"disk_usage": {}
|
||||
}
|
||||
|
||||
stats = {"base_path": self.storage_config.base_path, "total_files": 0, "total_size_bytes": 0, "cameras": {}, "disk_usage": {}}
|
||||
|
||||
# Get disk usage for base path
|
||||
if os.path.exists(self.storage_config.base_path):
|
||||
disk_usage = shutil.disk_usage(self.storage_config.base_path)
|
||||
stats["disk_usage"] = {
|
||||
"total_bytes": disk_usage.total,
|
||||
"used_bytes": disk_usage.used,
|
||||
"free_bytes": disk_usage.free,
|
||||
"used_percent": (disk_usage.used / disk_usage.total) * 100
|
||||
}
|
||||
|
||||
# Analyze files by camera
|
||||
stats["disk_usage"] = {"total_bytes": disk_usage.total, "used_bytes": disk_usage.used, "free_bytes": disk_usage.free, "used_percent": (disk_usage.used / disk_usage.total) * 100}
|
||||
|
||||
# Scan actual filesystem for all video files
|
||||
# This ensures we count all files, not just those in the index
|
||||
for camera_config in self.config.cameras:
|
||||
camera_name = camera_config.name
|
||||
storage_path = Path(camera_config.storage_path)
|
||||
|
||||
if camera_name not in stats["cameras"]:
|
||||
stats["cameras"][camera_name] = {"file_count": 0, "total_size_bytes": 0, "total_duration_seconds": 0}
|
||||
|
||||
# Scan for video files in camera directory
|
||||
if storage_path.exists():
|
||||
for video_file in storage_path.glob("*.avi"):
|
||||
if video_file.is_file():
|
||||
stats["total_files"] += 1
|
||||
stats["cameras"][camera_name]["file_count"] += 1
|
||||
|
||||
# Get file size
|
||||
try:
|
||||
file_size = video_file.stat().st_size
|
||||
stats["total_size_bytes"] += file_size
|
||||
stats["cameras"][camera_name]["total_size_bytes"] += file_size
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Could not get size for {video_file}: {e}")
|
||||
|
||||
# Add duration information from index if available
|
||||
for file_info in self.file_index["files"].values():
|
||||
camera_name = file_info["camera_name"]
|
||||
|
||||
if camera_name not in stats["cameras"]:
|
||||
stats["cameras"][camera_name] = {
|
||||
"file_count": 0,
|
||||
"total_size_bytes": 0,
|
||||
"total_duration_seconds": 0
|
||||
}
|
||||
|
||||
stats["total_files"] += 1
|
||||
stats["cameras"][camera_name]["file_count"] += 1
|
||||
|
||||
if file_info.get("file_size_bytes"):
|
||||
size = file_info["file_size_bytes"]
|
||||
stats["total_size_bytes"] += size
|
||||
stats["cameras"][camera_name]["total_size_bytes"] += size
|
||||
|
||||
if file_info.get("duration_seconds"):
|
||||
if camera_name in stats["cameras"] and file_info.get("duration_seconds"):
|
||||
duration = file_info["duration_seconds"]
|
||||
stats["cameras"][camera_name]["total_duration_seconds"] += duration
|
||||
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error getting storage statistics: {e}")
|
||||
return {}
|
||||
|
||||
|
||||
def cleanup_old_files(self, max_age_days: Optional[int] = None) -> Dict[str, Any]:
|
||||
"""Clean up old recording files"""
|
||||
if max_age_days is None:
|
||||
max_age_days = self.storage_config.cleanup_older_than_days
|
||||
|
||||
|
||||
cutoff_date = datetime.now() - timedelta(days=max_age_days)
|
||||
|
||||
cleanup_stats = {
|
||||
"files_removed": 0,
|
||||
"bytes_freed": 0,
|
||||
"errors": []
|
||||
}
|
||||
|
||||
|
||||
cleanup_stats = {"files_removed": 0, "bytes_freed": 0, "errors": []}
|
||||
|
||||
try:
|
||||
files_to_remove = []
|
||||
|
||||
|
||||
# Find files older than cutoff date
|
||||
for file_id, file_info in self.file_index["files"].items():
|
||||
try:
|
||||
@@ -312,81 +307,74 @@ class StorageManager:
|
||||
files_to_remove.append((file_id, file_info))
|
||||
except Exception as e:
|
||||
cleanup_stats["errors"].append(f"Error parsing date for {file_id}: {e}")
|
||||
|
||||
|
||||
# Remove old files
|
||||
for file_id, file_info in files_to_remove:
|
||||
try:
|
||||
filename = file_info["filename"]
|
||||
|
||||
|
||||
# Remove physical file
|
||||
if os.path.exists(filename):
|
||||
file_size = os.path.getsize(filename)
|
||||
os.remove(filename)
|
||||
cleanup_stats["bytes_freed"] += file_size
|
||||
self.logger.info(f"Removed old file: {filename}")
|
||||
|
||||
|
||||
# Remove from index
|
||||
del self.file_index["files"][file_id]
|
||||
cleanup_stats["files_removed"] += 1
|
||||
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error removing file {file_id}: {e}"
|
||||
cleanup_stats["errors"].append(error_msg)
|
||||
self.logger.error(error_msg)
|
||||
|
||||
|
||||
# Save updated index
|
||||
if cleanup_stats["files_removed"] > 0:
|
||||
self._save_file_index()
|
||||
|
||||
self.logger.info(f"Cleanup completed: {cleanup_stats['files_removed']} files removed, "
|
||||
f"{cleanup_stats['bytes_freed']} bytes freed")
|
||||
|
||||
|
||||
self.logger.info(f"Cleanup completed: {cleanup_stats['files_removed']} files removed, " f"{cleanup_stats['bytes_freed']} bytes freed")
|
||||
|
||||
return cleanup_stats
|
||||
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error during cleanup: {e}")
|
||||
cleanup_stats["errors"].append(str(e))
|
||||
return cleanup_stats
|
||||
|
||||
|
||||
def get_file_info(self, file_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Get information about a specific file"""
|
||||
return self.file_index["files"].get(file_id)
|
||||
|
||||
|
||||
def delete_file(self, file_id: str) -> bool:
|
||||
"""Delete a specific recording file"""
|
||||
try:
|
||||
if file_id not in self.file_index["files"]:
|
||||
self.logger.warning(f"File ID not found: {file_id}")
|
||||
return False
|
||||
|
||||
|
||||
file_info = self.file_index["files"][file_id]
|
||||
filename = file_info["filename"]
|
||||
|
||||
|
||||
# Remove physical file
|
||||
if os.path.exists(filename):
|
||||
os.remove(filename)
|
||||
self.logger.info(f"Deleted file: {filename}")
|
||||
|
||||
|
||||
# Remove from index
|
||||
del self.file_index["files"][file_id]
|
||||
self._save_file_index()
|
||||
|
||||
|
||||
return True
|
||||
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error deleting file {file_id}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def verify_storage_integrity(self) -> Dict[str, Any]:
|
||||
"""Verify storage integrity and fix issues"""
|
||||
integrity_report = {
|
||||
"total_files_in_index": len(self.file_index["files"]),
|
||||
"missing_files": [],
|
||||
"orphaned_files": [],
|
||||
"corrupted_entries": [],
|
||||
"fixed_issues": 0
|
||||
}
|
||||
|
||||
integrity_report = {"total_files_in_index": len(self.file_index["files"]), "missing_files": [], "orphaned_files": [], "corrupted_entries": [], "fixed_issues": 0}
|
||||
|
||||
try:
|
||||
# Check for missing files (in index but not on disk)
|
||||
for file_id, file_info in list(self.file_index["files"].items()):
|
||||
@@ -396,7 +384,7 @@ class StorageManager:
|
||||
# Remove from index
|
||||
del self.file_index["files"][file_id]
|
||||
integrity_report["fixed_issues"] += 1
|
||||
|
||||
|
||||
# Check for orphaned files (on disk but not in index)
|
||||
for camera_config in self.config.cameras:
|
||||
storage_path = Path(camera_config.storage_path)
|
||||
@@ -405,15 +393,15 @@ class StorageManager:
|
||||
file_id = video_file.name
|
||||
if file_id not in self.file_index["files"]:
|
||||
integrity_report["orphaned_files"].append(str(video_file))
|
||||
|
||||
|
||||
# Save updated index if fixes were made
|
||||
if integrity_report["fixed_issues"] > 0:
|
||||
self._save_file_index()
|
||||
|
||||
|
||||
self.logger.info(f"Storage integrity check completed: {integrity_report['fixed_issues']} issues fixed")
|
||||
|
||||
|
||||
return integrity_report
|
||||
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error during integrity check: {e}")
|
||||
integrity_report["error"] = str(e)
|
||||
|
||||
Reference in New Issue
Block a user