""" Storage Manager for the USDA Vision Camera System. This module handles file organization, cleanup, and management for recorded videos. """ import os import logging import shutil from typing import Dict, List, Optional, Any, Tuple from datetime import datetime, timedelta from pathlib import Path import json from ..core.config import Config, StorageConfig from ..core.state_manager import StateManager from ..core.events import EventSystem, EventType, Event class StorageManager: """Manages storage and file organization for recorded videos""" def __init__(self, config: Config, state_manager: StateManager, event_system: Optional[EventSystem] = None): self.config = config self.storage_config = config.storage self.state_manager = state_manager self.event_system = event_system self.logger = logging.getLogger(__name__) # Ensure base storage directory exists self._ensure_storage_structure() # File tracking self.file_index_path = os.path.join(self.storage_config.base_path, "file_index.json") self.file_index = self._load_file_index() # Subscribe to recording events if event system is available if self.event_system: self._setup_event_subscriptions() def _ensure_storage_structure(self) -> None: """Ensure storage directory structure exists""" try: # Create base storage directory Path(self.storage_config.base_path).mkdir(parents=True, exist_ok=True) # Create camera-specific directories for camera_config in self.config.cameras: Path(camera_config.storage_path).mkdir(parents=True, exist_ok=True) self.logger.debug(f"Ensured storage directory: {camera_config.storage_path}") self.logger.info("Storage directory structure verified") except Exception as e: self.logger.error(f"Error creating storage structure: {e}") raise def _setup_event_subscriptions(self) -> None: """Setup event subscriptions for recording tracking""" if not self.event_system: return def on_recording_started(event: Event): """Handle recording started event""" try: camera_name = event.data.get("camera_name") filename = event.data.get("filename") if camera_name and filename: self.register_recording_file( camera_name=camera_name, filename=filename, start_time=event.timestamp, machine_trigger=event.data.get("machine_trigger") ) except Exception as e: self.logger.error(f"Error handling recording started event: {e}") def on_recording_stopped(event: Event): """Handle recording stopped event""" try: filename = event.data.get("filename") if filename: file_id = os.path.basename(filename) self.finalize_recording_file( file_id=file_id, end_time=event.timestamp, duration_seconds=event.data.get("duration_seconds") ) except Exception as e: self.logger.error(f"Error handling recording stopped event: {e}") # Subscribe to recording events self.event_system.subscribe(EventType.RECORDING_STARTED, on_recording_started) self.event_system.subscribe(EventType.RECORDING_STOPPED, on_recording_stopped) def _load_file_index(self) -> Dict[str, Any]: """Load file index from disk""" try: if os.path.exists(self.file_index_path): with open(self.file_index_path, 'r') as f: return json.load(f) else: return {"files": {}, "last_updated": None} except Exception as e: self.logger.error(f"Error loading file index: {e}") return {"files": {}, "last_updated": None} def _save_file_index(self) -> None: """Save file index to disk""" try: self.file_index["last_updated"] = datetime.now().isoformat() with open(self.file_index_path, 'w') as f: json.dump(self.file_index, f, indent=2) except Exception as e: self.logger.error(f"Error saving file index: {e}") def register_recording_file(self, camera_name: str, filename: str, start_time: datetime, machine_trigger: Optional[str] = None) -> str: """Register a new recording file""" try: file_id = os.path.basename(filename) file_info = { "camera_name": camera_name, "filename": filename, "file_id": file_id, "start_time": start_time.isoformat(), "end_time": None, "file_size_bytes": None, "duration_seconds": None, "machine_trigger": machine_trigger, "status": "recording", "created_at": datetime.now().isoformat() } self.file_index["files"][file_id] = file_info self._save_file_index() self.logger.info(f"Registered recording file: {file_id}") return file_id except Exception as e: self.logger.error(f"Error registering recording file: {e}") return "" def finalize_recording_file(self, file_id: str, end_time: datetime, duration_seconds: Optional[float] = None) -> bool: """Finalize a recording file when recording stops""" try: if file_id not in self.file_index["files"]: self.logger.warning(f"Recording file not found for finalization: {file_id}") return False file_info = self.file_index["files"][file_id] file_info["end_time"] = end_time.isoformat() file_info["status"] = "completed" if duration_seconds is not None: file_info["duration_seconds"] = duration_seconds # Get file size if file exists filename = file_info["filename"] if os.path.exists(filename): file_info["file_size_bytes"] = os.path.getsize(filename) self._save_file_index() self.logger.info(f"Finalized recording file: {file_id}") return True except Exception as e: self.logger.error(f"Error finalizing recording file: {e}") return False def finalize_recording_file(self, file_id: str, end_time: datetime, duration_seconds: float, frame_count: Optional[int] = None) -> bool: """Finalize a recording file after recording stops""" try: if file_id not in self.file_index["files"]: self.logger.warning(f"File ID not found in index: {file_id}") return False file_info = self.file_index["files"][file_id] filename = file_info["filename"] # Update file information file_info["end_time"] = end_time.isoformat() file_info["duration_seconds"] = duration_seconds file_info["status"] = "completed" # Get file size if file exists if os.path.exists(filename): file_info["file_size_bytes"] = os.path.getsize(filename) if frame_count is not None: file_info["frame_count"] = frame_count self._save_file_index() self.logger.info(f"Finalized recording file: {file_id} (duration: {duration_seconds:.1f}s)") return True except Exception as e: self.logger.error(f"Error finalizing recording file: {e}") return False def get_recording_files(self, camera_name: Optional[str] = None, start_date: Optional[datetime] = None, end_date: Optional[datetime] = None, limit: Optional[int] = None) -> List[Dict[str, Any]]: """Get list of recording files with optional filters""" try: files = [] for file_id, file_info in self.file_index["files"].items(): # Filter by camera name if camera_name and file_info["camera_name"] != camera_name: continue # Filter by date range if start_date or end_date: file_start = datetime.fromisoformat(file_info["start_time"]) if start_date and file_start < start_date: continue if end_date and file_start > end_date: continue files.append(file_info.copy()) # Sort by start time (newest first) files.sort(key=lambda x: x["start_time"], reverse=True) # Apply limit if limit: files = files[:limit] return files except Exception as e: self.logger.error(f"Error getting recording files: {e}") return [] def get_storage_statistics(self) -> Dict[str, Any]: """Get storage usage statistics""" try: stats = { "base_path": self.storage_config.base_path, "total_files": 0, "total_size_bytes": 0, "cameras": {}, "disk_usage": {} } # Get disk usage for base path if os.path.exists(self.storage_config.base_path): disk_usage = shutil.disk_usage(self.storage_config.base_path) stats["disk_usage"] = { "total_bytes": disk_usage.total, "used_bytes": disk_usage.used, "free_bytes": disk_usage.free, "used_percent": (disk_usage.used / disk_usage.total) * 100 } # Analyze files by camera for file_info in self.file_index["files"].values(): camera_name = file_info["camera_name"] if camera_name not in stats["cameras"]: stats["cameras"][camera_name] = { "file_count": 0, "total_size_bytes": 0, "total_duration_seconds": 0 } stats["total_files"] += 1 stats["cameras"][camera_name]["file_count"] += 1 if file_info.get("file_size_bytes"): size = file_info["file_size_bytes"] stats["total_size_bytes"] += size stats["cameras"][camera_name]["total_size_bytes"] += size if file_info.get("duration_seconds"): duration = file_info["duration_seconds"] stats["cameras"][camera_name]["total_duration_seconds"] += duration return stats except Exception as e: self.logger.error(f"Error getting storage statistics: {e}") return {} def cleanup_old_files(self, max_age_days: Optional[int] = None) -> Dict[str, Any]: """Clean up old recording files""" if max_age_days is None: max_age_days = self.storage_config.cleanup_older_than_days cutoff_date = datetime.now() - timedelta(days=max_age_days) cleanup_stats = { "files_removed": 0, "bytes_freed": 0, "errors": [] } try: files_to_remove = [] # Find files older than cutoff date for file_id, file_info in self.file_index["files"].items(): try: file_start = datetime.fromisoformat(file_info["start_time"]) if file_start < cutoff_date and file_info["status"] == "completed": files_to_remove.append((file_id, file_info)) except Exception as e: cleanup_stats["errors"].append(f"Error parsing date for {file_id}: {e}") # Remove old files for file_id, file_info in files_to_remove: try: filename = file_info["filename"] # Remove physical file if os.path.exists(filename): file_size = os.path.getsize(filename) os.remove(filename) cleanup_stats["bytes_freed"] += file_size self.logger.info(f"Removed old file: {filename}") # Remove from index del self.file_index["files"][file_id] cleanup_stats["files_removed"] += 1 except Exception as e: error_msg = f"Error removing file {file_id}: {e}" cleanup_stats["errors"].append(error_msg) self.logger.error(error_msg) # Save updated index if cleanup_stats["files_removed"] > 0: self._save_file_index() self.logger.info(f"Cleanup completed: {cleanup_stats['files_removed']} files removed, " f"{cleanup_stats['bytes_freed']} bytes freed") return cleanup_stats except Exception as e: self.logger.error(f"Error during cleanup: {e}") cleanup_stats["errors"].append(str(e)) return cleanup_stats def get_file_info(self, file_id: str) -> Optional[Dict[str, Any]]: """Get information about a specific file""" return self.file_index["files"].get(file_id) def delete_file(self, file_id: str) -> bool: """Delete a specific recording file""" try: if file_id not in self.file_index["files"]: self.logger.warning(f"File ID not found: {file_id}") return False file_info = self.file_index["files"][file_id] filename = file_info["filename"] # Remove physical file if os.path.exists(filename): os.remove(filename) self.logger.info(f"Deleted file: {filename}") # Remove from index del self.file_index["files"][file_id] self._save_file_index() return True except Exception as e: self.logger.error(f"Error deleting file {file_id}: {e}") return False def verify_storage_integrity(self) -> Dict[str, Any]: """Verify storage integrity and fix issues""" integrity_report = { "total_files_in_index": len(self.file_index["files"]), "missing_files": [], "orphaned_files": [], "corrupted_entries": [], "fixed_issues": 0 } try: # Check for missing files (in index but not on disk) for file_id, file_info in list(self.file_index["files"].items()): filename = file_info.get("filename") if filename and not os.path.exists(filename): integrity_report["missing_files"].append(file_id) # Remove from index del self.file_index["files"][file_id] integrity_report["fixed_issues"] += 1 # Check for orphaned files (on disk but not in index) for camera_config in self.config.cameras: storage_path = Path(camera_config.storage_path) if storage_path.exists(): for video_file in storage_path.glob("*.avi"): file_id = video_file.name if file_id not in self.file_index["files"]: integrity_report["orphaned_files"].append(str(video_file)) # Save updated index if fixes were made if integrity_report["fixed_issues"] > 0: self._save_file_index() self.logger.info(f"Storage integrity check completed: {integrity_report['fixed_issues']} issues fixed") return integrity_report except Exception as e: self.logger.error(f"Error during integrity check: {e}") integrity_report["error"] = str(e) return integrity_report