418 lines
18 KiB
Python
418 lines
18 KiB
Python
"""
|
|
Storage Manager for the USDA Vision Camera System.
|
|
|
|
This module handles file organization, cleanup, and management for recorded videos.
|
|
"""
|
|
|
|
import os
|
|
import logging
|
|
import shutil
|
|
from typing import Dict, List, Optional, Any, Tuple
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
import json
|
|
|
|
from ..core.config import Config, StorageConfig
|
|
from ..core.state_manager import StateManager
|
|
from ..core.events import EventSystem, EventType, Event
|
|
|
|
|
|
class StorageManager:
|
|
"""Manages storage and file organization for recorded videos"""
|
|
|
|
def __init__(self, config: Config, state_manager: StateManager, event_system: Optional[EventSystem] = None):
|
|
self.config = config
|
|
self.storage_config = config.storage
|
|
self.state_manager = state_manager
|
|
self.event_system = event_system
|
|
self.logger = logging.getLogger(__name__)
|
|
|
|
# Ensure base storage directory exists
|
|
self._ensure_storage_structure()
|
|
|
|
# File tracking
|
|
self.file_index_path = os.path.join(self.storage_config.base_path, "file_index.json")
|
|
self.file_index = self._load_file_index()
|
|
|
|
# Subscribe to recording events if event system is available
|
|
if self.event_system:
|
|
self._setup_event_subscriptions()
|
|
|
|
def _ensure_storage_structure(self) -> None:
|
|
"""Ensure storage directory structure exists"""
|
|
try:
|
|
# Create base storage directory
|
|
Path(self.storage_config.base_path).mkdir(parents=True, exist_ok=True)
|
|
|
|
# Create camera-specific directories
|
|
for camera_config in self.config.cameras:
|
|
Path(camera_config.storage_path).mkdir(parents=True, exist_ok=True)
|
|
self.logger.debug(f"Ensured storage directory: {camera_config.storage_path}")
|
|
|
|
self.logger.info("Storage directory structure verified")
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Error creating storage structure: {e}")
|
|
raise
|
|
|
|
def _setup_event_subscriptions(self) -> None:
|
|
"""Setup event subscriptions for recording tracking"""
|
|
if not self.event_system:
|
|
return
|
|
|
|
def on_recording_started(event: Event):
|
|
"""Handle recording started event"""
|
|
try:
|
|
camera_name = event.data.get("camera_name")
|
|
filename = event.data.get("filename")
|
|
if camera_name and filename:
|
|
self.register_recording_file(camera_name=camera_name, filename=filename, start_time=event.timestamp, machine_trigger=event.data.get("machine_trigger"))
|
|
except Exception as e:
|
|
self.logger.error(f"Error handling recording started event: {e}")
|
|
|
|
def on_recording_stopped(event: Event):
|
|
"""Handle recording stopped event"""
|
|
try:
|
|
filename = event.data.get("filename")
|
|
if filename:
|
|
file_id = os.path.basename(filename)
|
|
self.finalize_recording_file(file_id=file_id, end_time=event.timestamp, duration_seconds=event.data.get("duration_seconds"))
|
|
except Exception as e:
|
|
self.logger.error(f"Error handling recording stopped event: {e}")
|
|
|
|
# Subscribe to recording events
|
|
self.event_system.subscribe(EventType.RECORDING_STARTED, on_recording_started)
|
|
self.event_system.subscribe(EventType.RECORDING_STOPPED, on_recording_stopped)
|
|
|
|
def _load_file_index(self) -> Dict[str, Any]:
|
|
"""Load file index from disk"""
|
|
try:
|
|
if os.path.exists(self.file_index_path):
|
|
with open(self.file_index_path, "r") as f:
|
|
return json.load(f)
|
|
else:
|
|
return {"files": {}, "last_updated": None}
|
|
except Exception as e:
|
|
self.logger.error(f"Error loading file index: {e}")
|
|
return {"files": {}, "last_updated": None}
|
|
|
|
def _save_file_index(self) -> None:
|
|
"""Save file index to disk"""
|
|
try:
|
|
self.file_index["last_updated"] = datetime.now().isoformat()
|
|
with open(self.file_index_path, "w") as f:
|
|
json.dump(self.file_index, f, indent=2)
|
|
except Exception as e:
|
|
self.logger.error(f"Error saving file index: {e}")
|
|
|
|
def register_recording_file(self, camera_name: str, filename: str, start_time: datetime, machine_trigger: Optional[str] = None) -> str:
|
|
"""Register a new recording file"""
|
|
try:
|
|
file_id = os.path.basename(filename)
|
|
|
|
file_info = {"camera_name": camera_name, "filename": filename, "file_id": file_id, "start_time": start_time.isoformat(), "end_time": None, "file_size_bytes": None, "duration_seconds": None, "machine_trigger": machine_trigger, "status": "recording", "created_at": datetime.now().isoformat()}
|
|
|
|
self.file_index["files"][file_id] = file_info
|
|
self._save_file_index()
|
|
|
|
self.logger.info(f"Registered recording file: {file_id}")
|
|
return file_id
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Error registering recording file: {e}")
|
|
return ""
|
|
|
|
def finalize_recording_file(self, file_id: str, end_time: datetime, duration_seconds: Optional[float] = None) -> bool:
|
|
"""Finalize a recording file when recording stops"""
|
|
try:
|
|
if file_id not in self.file_index["files"]:
|
|
self.logger.warning(f"Recording file not found for finalization: {file_id}")
|
|
return False
|
|
|
|
file_info = self.file_index["files"][file_id]
|
|
file_info["end_time"] = end_time.isoformat()
|
|
file_info["status"] = "completed"
|
|
|
|
if duration_seconds is not None:
|
|
file_info["duration_seconds"] = duration_seconds
|
|
|
|
# Get file size if file exists
|
|
filename = file_info["filename"]
|
|
if os.path.exists(filename):
|
|
file_info["file_size_bytes"] = os.path.getsize(filename)
|
|
|
|
self._save_file_index()
|
|
self.logger.info(f"Finalized recording file: {file_id}")
|
|
return True
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Error finalizing recording file: {e}")
|
|
return False
|
|
|
|
def finalize_recording_file(self, file_id: str, end_time: datetime, duration_seconds: float, frame_count: Optional[int] = None) -> bool:
|
|
"""Finalize a recording file after recording stops"""
|
|
try:
|
|
if file_id not in self.file_index["files"]:
|
|
self.logger.warning(f"File ID not found in index: {file_id}")
|
|
return False
|
|
|
|
file_info = self.file_index["files"][file_id]
|
|
filename = file_info["filename"]
|
|
|
|
# Update file information
|
|
file_info["end_time"] = end_time.isoformat()
|
|
file_info["duration_seconds"] = duration_seconds
|
|
file_info["status"] = "completed"
|
|
|
|
# Get file size if file exists
|
|
if os.path.exists(filename):
|
|
file_info["file_size_bytes"] = os.path.getsize(filename)
|
|
|
|
if frame_count is not None:
|
|
file_info["frame_count"] = frame_count
|
|
|
|
self._save_file_index()
|
|
|
|
self.logger.info(f"Finalized recording file: {file_id} (duration: {duration_seconds:.1f}s)")
|
|
return True
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Error finalizing recording file: {e}")
|
|
return False
|
|
|
|
def get_recording_files(self, camera_name: Optional[str] = None, start_date: Optional[datetime] = None, end_date: Optional[datetime] = None, limit: Optional[int] = None) -> List[Dict[str, Any]]:
|
|
"""Get list of recording files with optional filters"""
|
|
try:
|
|
files = []
|
|
|
|
# First, get files from the index (if available)
|
|
indexed_files = set()
|
|
for file_id, file_info in self.file_index["files"].items():
|
|
# Filter by camera name
|
|
if camera_name and file_info["camera_name"] != camera_name:
|
|
continue
|
|
|
|
# Filter by date range
|
|
if start_date or end_date:
|
|
file_start = datetime.fromisoformat(file_info["start_time"])
|
|
if start_date and file_start < start_date:
|
|
continue
|
|
if end_date and file_start > end_date:
|
|
continue
|
|
|
|
files.append(file_info.copy())
|
|
indexed_files.add(file_info["filename"])
|
|
|
|
# Then, scan filesystem for files not in the index
|
|
for camera_config in self.config.cameras:
|
|
# Skip if filtering by camera name and this isn't the one
|
|
if camera_name and camera_config.name != camera_name:
|
|
continue
|
|
|
|
storage_path = Path(camera_config.storage_path)
|
|
if storage_path.exists():
|
|
# Scan for all supported video formats
|
|
video_extensions = ["*.avi", "*.mp4", "*.webm"]
|
|
for pattern in video_extensions:
|
|
for video_file in storage_path.glob(pattern):
|
|
if video_file.is_file() and str(video_file) not in indexed_files:
|
|
# Get file stats
|
|
stat = video_file.stat()
|
|
file_mtime = datetime.fromtimestamp(stat.st_mtime)
|
|
|
|
# Apply date filters
|
|
if start_date and file_mtime < start_date:
|
|
continue
|
|
if end_date and file_mtime > end_date:
|
|
continue
|
|
|
|
# Create file info for unindexed file
|
|
file_info = {"camera_name": camera_config.name, "filename": str(video_file), "file_id": video_file.name, "start_time": file_mtime.isoformat(), "end_time": None, "file_size_bytes": stat.st_size, "duration_seconds": None, "machine_trigger": None, "status": "unknown", "created_at": file_mtime.isoformat()} # We don't know if it's completed or not
|
|
files.append(file_info)
|
|
|
|
# Sort by start time (newest first)
|
|
files.sort(key=lambda x: x["start_time"], reverse=True)
|
|
|
|
# Apply limit
|
|
if limit:
|
|
files = files[:limit]
|
|
|
|
return files
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Error getting recording files: {e}")
|
|
return []
|
|
|
|
def get_storage_statistics(self) -> Dict[str, Any]:
|
|
"""Get storage usage statistics"""
|
|
try:
|
|
stats = {"base_path": self.storage_config.base_path, "total_files": 0, "total_size_bytes": 0, "cameras": {}, "disk_usage": {}}
|
|
|
|
# Get disk usage for base path
|
|
if os.path.exists(self.storage_config.base_path):
|
|
disk_usage = shutil.disk_usage(self.storage_config.base_path)
|
|
stats["disk_usage"] = {"total_bytes": disk_usage.total, "used_bytes": disk_usage.used, "free_bytes": disk_usage.free, "used_percent": (disk_usage.used / disk_usage.total) * 100}
|
|
|
|
# Scan actual filesystem for all video files
|
|
# This ensures we count all files, not just those in the index
|
|
for camera_config in self.config.cameras:
|
|
camera_name = camera_config.name
|
|
storage_path = Path(camera_config.storage_path)
|
|
|
|
if camera_name not in stats["cameras"]:
|
|
stats["cameras"][camera_name] = {"file_count": 0, "total_size_bytes": 0, "total_duration_seconds": 0}
|
|
|
|
# Scan for video files in camera directory
|
|
if storage_path.exists():
|
|
# Scan for all supported video formats
|
|
video_extensions = ["*.avi", "*.mp4", "*.webm"]
|
|
for pattern in video_extensions:
|
|
for video_file in storage_path.glob(pattern):
|
|
if video_file.is_file():
|
|
stats["total_files"] += 1
|
|
stats["cameras"][camera_name]["file_count"] += 1
|
|
|
|
# Get file size
|
|
try:
|
|
file_size = video_file.stat().st_size
|
|
stats["total_size_bytes"] += file_size
|
|
stats["cameras"][camera_name]["total_size_bytes"] += file_size
|
|
except Exception as e:
|
|
self.logger.warning(f"Could not get size for {video_file}: {e}")
|
|
|
|
# Add duration information from index if available
|
|
for file_info in self.file_index["files"].values():
|
|
camera_name = file_info["camera_name"]
|
|
if camera_name in stats["cameras"] and file_info.get("duration_seconds"):
|
|
duration = file_info["duration_seconds"]
|
|
stats["cameras"][camera_name]["total_duration_seconds"] += duration
|
|
|
|
return stats
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Error getting storage statistics: {e}")
|
|
return {}
|
|
|
|
def cleanup_old_files(self, max_age_days: Optional[int] = None) -> Dict[str, Any]:
|
|
"""Clean up old recording files"""
|
|
if max_age_days is None:
|
|
max_age_days = self.storage_config.cleanup_older_than_days
|
|
|
|
cutoff_date = datetime.now() - timedelta(days=max_age_days)
|
|
|
|
cleanup_stats = {"files_removed": 0, "bytes_freed": 0, "errors": []}
|
|
|
|
try:
|
|
files_to_remove = []
|
|
|
|
# Find files older than cutoff date
|
|
for file_id, file_info in self.file_index["files"].items():
|
|
try:
|
|
file_start = datetime.fromisoformat(file_info["start_time"])
|
|
if file_start < cutoff_date and file_info["status"] == "completed":
|
|
files_to_remove.append((file_id, file_info))
|
|
except Exception as e:
|
|
cleanup_stats["errors"].append(f"Error parsing date for {file_id}: {e}")
|
|
|
|
# Remove old files
|
|
for file_id, file_info in files_to_remove:
|
|
try:
|
|
filename = file_info["filename"]
|
|
|
|
# Remove physical file
|
|
if os.path.exists(filename):
|
|
file_size = os.path.getsize(filename)
|
|
os.remove(filename)
|
|
cleanup_stats["bytes_freed"] += file_size
|
|
self.logger.info(f"Removed old file: {filename}")
|
|
|
|
# Remove from index
|
|
del self.file_index["files"][file_id]
|
|
cleanup_stats["files_removed"] += 1
|
|
|
|
except Exception as e:
|
|
error_msg = f"Error removing file {file_id}: {e}"
|
|
cleanup_stats["errors"].append(error_msg)
|
|
self.logger.error(error_msg)
|
|
|
|
# Save updated index
|
|
if cleanup_stats["files_removed"] > 0:
|
|
self._save_file_index()
|
|
|
|
self.logger.info(f"Cleanup completed: {cleanup_stats['files_removed']} files removed, " f"{cleanup_stats['bytes_freed']} bytes freed")
|
|
|
|
return cleanup_stats
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Error during cleanup: {e}")
|
|
cleanup_stats["errors"].append(str(e))
|
|
return cleanup_stats
|
|
|
|
def get_file_info(self, file_id: str) -> Optional[Dict[str, Any]]:
|
|
"""Get information about a specific file"""
|
|
return self.file_index["files"].get(file_id)
|
|
|
|
def delete_file(self, file_id: str) -> bool:
|
|
"""Delete a specific recording file"""
|
|
try:
|
|
if file_id not in self.file_index["files"]:
|
|
self.logger.warning(f"File ID not found: {file_id}")
|
|
return False
|
|
|
|
file_info = self.file_index["files"][file_id]
|
|
filename = file_info["filename"]
|
|
|
|
# Remove physical file
|
|
if os.path.exists(filename):
|
|
os.remove(filename)
|
|
self.logger.info(f"Deleted file: {filename}")
|
|
|
|
# Remove from index
|
|
del self.file_index["files"][file_id]
|
|
self._save_file_index()
|
|
|
|
return True
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Error deleting file {file_id}: {e}")
|
|
return False
|
|
|
|
def verify_storage_integrity(self) -> Dict[str, Any]:
|
|
"""Verify storage integrity and fix issues"""
|
|
integrity_report = {"total_files_in_index": len(self.file_index["files"]), "missing_files": [], "orphaned_files": [], "corrupted_entries": [], "fixed_issues": 0}
|
|
|
|
try:
|
|
# Check for missing files (in index but not on disk)
|
|
for file_id, file_info in list(self.file_index["files"].items()):
|
|
filename = file_info.get("filename")
|
|
if filename and not os.path.exists(filename):
|
|
integrity_report["missing_files"].append(file_id)
|
|
# Remove from index
|
|
del self.file_index["files"][file_id]
|
|
integrity_report["fixed_issues"] += 1
|
|
|
|
# Check for orphaned files (on disk but not in index)
|
|
for camera_config in self.config.cameras:
|
|
storage_path = Path(camera_config.storage_path)
|
|
if storage_path.exists():
|
|
# Check for all supported video formats
|
|
video_extensions = ["*.avi", "*.mp4", "*.webm"]
|
|
for pattern in video_extensions:
|
|
for video_file in storage_path.glob(pattern):
|
|
file_id = video_file.name
|
|
if file_id not in self.file_index["files"]:
|
|
integrity_report["orphaned_files"].append(str(video_file))
|
|
|
|
# Save updated index if fixes were made
|
|
if integrity_report["fixed_issues"] > 0:
|
|
self._save_file_index()
|
|
|
|
self.logger.info(f"Storage integrity check completed: {integrity_report['fixed_issues']} issues fixed")
|
|
|
|
return integrity_report
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Error during integrity check: {e}")
|
|
integrity_report["error"] = str(e)
|
|
return integrity_report
|