e1171e8ff8
Motion detection now optionally runs MobileNet V2 SSD (COCO, quantized) on frames that trigger motion, identifying objects like people, cats, and cars. Events without detected objects are suppressed by default. Snapshots include bounding box annotations. New MCP tool vision_get_detections() enables label-based queries. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
857 lines
27 KiB
Python
857 lines
27 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Vision MCP Server
|
|
|
|
Model Context Protocol server for interacting with multiple camera-server instances
|
|
and RTSP streams.
|
|
|
|
Tools:
|
|
- vision_get_cams() - Get list of active cameras
|
|
- vision_snap(cam_id) - Get snapshot from a camera (HTTP API or RTSP)
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
from pathlib import Path
|
|
from typing import List, Dict, Any, Union
|
|
from io import BytesIO
|
|
|
|
import httpx
|
|
import cv2
|
|
import numpy as np
|
|
from PIL import Image
|
|
from fastmcp import FastMCP
|
|
from fastmcp.utilities.types import Image as MCPImage
|
|
|
|
# Configuration
|
|
CONFIG_FILE = Path.home() / ".vision_setup.json"
|
|
LOG_FILE = Path("/tmp/vision_mcp.log")
|
|
REQUEST_TIMEOUT = 5.0 # seconds
|
|
RTSP_TIMEOUT = 10.0 # seconds for RTSP stream connection
|
|
|
|
# Setup logging
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
|
handlers=[
|
|
logging.FileHandler(LOG_FILE),
|
|
logging.StreamHandler()
|
|
]
|
|
)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Initialize MCP server
|
|
mcp = FastMCP("Vision Camera System")
|
|
|
|
|
|
def load_camera_config() -> Dict[str, Any]:
|
|
"""
|
|
Load camera configuration from ~/.vision_setup.json
|
|
|
|
Returns:
|
|
Dictionary with camera configurations
|
|
|
|
Raises:
|
|
FileNotFoundError: If config file doesn't exist
|
|
ValueError: If config file is invalid
|
|
"""
|
|
if not CONFIG_FILE.exists():
|
|
raise FileNotFoundError(
|
|
f"Camera config file not found: {CONFIG_FILE}\n"
|
|
f"Create {CONFIG_FILE} with camera configurations."
|
|
)
|
|
|
|
try:
|
|
with open(CONFIG_FILE, 'r') as f:
|
|
config = json.load(f)
|
|
|
|
if 'cameras' not in config:
|
|
raise ValueError("Config file must contain 'cameras' array")
|
|
|
|
# Validate each camera config
|
|
for cam in config['cameras']:
|
|
# All cameras need 'id' and 'type'
|
|
if 'id' not in cam:
|
|
raise ValueError("Camera config missing 'id' field")
|
|
|
|
cam_type = cam.get('type', 'http') # Default to http for backward compatibility
|
|
|
|
if cam_type == 'http':
|
|
# HTTP cameras need url and api_key
|
|
required_fields = ['url', 'api_key']
|
|
missing = [f for f in required_fields if f not in cam]
|
|
if missing:
|
|
raise ValueError(
|
|
f"HTTP camera '{cam['id']}' missing required fields: {missing}"
|
|
)
|
|
elif cam_type == 'rtsp':
|
|
# RTSP cameras need rtsp_url
|
|
if 'rtsp_url' not in cam:
|
|
raise ValueError(
|
|
f"RTSP camera '{cam['id']}' missing required field: rtsp_url"
|
|
)
|
|
else:
|
|
raise ValueError(
|
|
f"Camera '{cam['id']}' has invalid type: {cam_type}. "
|
|
f"Must be 'http' or 'rtsp'"
|
|
)
|
|
|
|
logger.info(f"Loaded {len(config['cameras'])} camera(s) from config")
|
|
return config
|
|
|
|
except json.JSONDecodeError as e:
|
|
raise ValueError(f"Invalid JSON in config file: {e}")
|
|
|
|
|
|
def get_camera_by_id(cam_id: str) -> Dict[str, str]:
|
|
"""
|
|
Get camera configuration by ID
|
|
|
|
Args:
|
|
cam_id: Camera ID string
|
|
|
|
Returns:
|
|
Camera configuration dict
|
|
|
|
Raises:
|
|
ValueError: If camera ID not found
|
|
"""
|
|
config = load_camera_config()
|
|
|
|
for cam in config['cameras']:
|
|
if cam['id'] == cam_id:
|
|
return cam
|
|
|
|
available_ids = [c['id'] for c in config['cameras']]
|
|
raise ValueError(
|
|
f"Camera '{cam_id}' not found in config.\n"
|
|
f"Available cameras: {', '.join(available_ids)}"
|
|
)
|
|
|
|
|
|
def capture_rtsp_snapshot(rtsp_url: str, timeout: float = RTSP_TIMEOUT) -> bytes:
|
|
"""
|
|
Capture a single frame from an RTSP stream
|
|
|
|
Args:
|
|
rtsp_url: RTSP stream URL (e.g., rtsp://192.168.1.239/live)
|
|
timeout: Connection timeout in seconds
|
|
|
|
Returns:
|
|
JPEG image bytes
|
|
|
|
Raises:
|
|
RuntimeError: If unable to connect or capture frame
|
|
"""
|
|
logger.info(f"Attempting to capture from RTSP: {rtsp_url}")
|
|
|
|
# Create video capture object
|
|
cap = cv2.VideoCapture(rtsp_url)
|
|
|
|
# Set timeout (in milliseconds)
|
|
cap.set(cv2.CAP_PROP_OPEN_TIMEOUT_MSEC, int(timeout * 1000))
|
|
|
|
try:
|
|
# Check if stream opened successfully
|
|
if not cap.isOpened():
|
|
raise RuntimeError(f"Failed to open RTSP stream: {rtsp_url}")
|
|
|
|
# Read a frame
|
|
ret, frame = cap.read()
|
|
|
|
if not ret or frame is None:
|
|
raise RuntimeError(f"Failed to read frame from RTSP stream: {rtsp_url}")
|
|
|
|
# Convert BGR (OpenCV) to RGB (PIL)
|
|
frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
|
|
|
# Convert to PIL Image
|
|
pil_image = Image.fromarray(frame_rgb)
|
|
|
|
# Convert to JPEG bytes
|
|
buffer = BytesIO()
|
|
pil_image.save(buffer, format='JPEG', quality=90)
|
|
jpeg_bytes = buffer.getvalue()
|
|
|
|
logger.info(f"✓ Captured RTSP snapshot ({len(jpeg_bytes)} bytes)")
|
|
return jpeg_bytes
|
|
|
|
finally:
|
|
# Always release the capture
|
|
cap.release()
|
|
|
|
|
|
@mcp.tool()
|
|
async def vision_get_cams() -> List[Dict[str, str]]:
|
|
"""
|
|
Get list of all configured cameras with their online/offline status.
|
|
|
|
Queries the /health endpoint of each camera to determine if it's online.
|
|
|
|
Returns:
|
|
List of camera info dictionaries:
|
|
[
|
|
{
|
|
"id": "basement",
|
|
"status": "online" # or "offline"
|
|
},
|
|
...
|
|
]
|
|
|
|
Examples:
|
|
vision_get_cams()
|
|
"""
|
|
try:
|
|
config = load_camera_config()
|
|
cameras = []
|
|
|
|
async with httpx.AsyncClient(timeout=REQUEST_TIMEOUT, verify=False) as client:
|
|
for cam in config['cameras']:
|
|
cam_type = cam.get('type', 'http')
|
|
cam_info = {
|
|
"id": cam['id'],
|
|
"type": cam_type,
|
|
"status": "unknown"
|
|
}
|
|
|
|
# Check status based on camera type
|
|
try:
|
|
if cam_type == 'http':
|
|
# Check HTTP health endpoint
|
|
health_url = f"{cam['url'].rstrip('/')}/health"
|
|
logger.debug(f"Checking HTTP health: {health_url}")
|
|
|
|
response = await client.get(health_url)
|
|
|
|
if response.status_code == 200:
|
|
cam_info['status'] = 'online'
|
|
logger.info(f"Camera '{cam['id']}' is online")
|
|
else:
|
|
cam_info['status'] = 'offline'
|
|
logger.warning(f"Camera '{cam['id']}' returned status {response.status_code}")
|
|
|
|
elif cam_type == 'rtsp':
|
|
# Try to briefly connect to RTSP stream
|
|
rtsp_url = cam['rtsp_url']
|
|
logger.debug(f"Checking RTSP stream: {rtsp_url}")
|
|
|
|
cap = cv2.VideoCapture(rtsp_url)
|
|
cap.set(cv2.CAP_PROP_OPEN_TIMEOUT_MSEC, 3000) # 3 second timeout
|
|
|
|
if cap.isOpened():
|
|
cam_info['status'] = 'online'
|
|
logger.info(f"RTSP camera '{cam['id']}' is online")
|
|
else:
|
|
cam_info['status'] = 'offline'
|
|
logger.warning(f"RTSP camera '{cam['id']}' connection failed")
|
|
|
|
cap.release()
|
|
|
|
except httpx.TimeoutException:
|
|
cam_info['status'] = 'offline'
|
|
logger.warning(f"Camera '{cam['id']}' timed out")
|
|
|
|
except httpx.ConnectError:
|
|
cam_info['status'] = 'offline'
|
|
logger.warning(f"Camera '{cam['id']}' connection failed")
|
|
|
|
except Exception as e:
|
|
cam_info['status'] = 'offline'
|
|
logger.error(f"Camera '{cam['id']}' error: {e}")
|
|
|
|
cameras.append(cam_info)
|
|
|
|
logger.info(f"Found {len(cameras)} camera(s), {sum(1 for c in cameras if c['status'] == 'online')} online")
|
|
return cameras
|
|
|
|
except FileNotFoundError as e:
|
|
logger.error(f"Config error: {e}")
|
|
return [{"error": str(e)}]
|
|
|
|
except ValueError as e:
|
|
logger.error(f"Config error: {e}")
|
|
return [{"error": str(e)}]
|
|
|
|
except Exception as e:
|
|
logger.error(f"Unexpected error: {e}")
|
|
return [{"error": f"Unexpected error: {str(e)}"}]
|
|
|
|
|
|
@mcp.tool()
|
|
async def vision_snap(cam_id: str) -> Union[MCPImage, str]:
|
|
"""
|
|
Get a snapshot from a camera.
|
|
|
|
Queries the /snapshot endpoint and returns the image for inline display.
|
|
|
|
Args:
|
|
cam_id: Camera ID from config file (e.g., "basement")
|
|
|
|
Returns:
|
|
MCPImage object for inline display, or error message string
|
|
|
|
Examples:
|
|
vision_snap("basement")
|
|
"""
|
|
try:
|
|
# Get camera config
|
|
cam = get_camera_by_id(cam_id)
|
|
cam_type = cam.get('type', 'http')
|
|
|
|
# Handle based on camera type
|
|
if cam_type == 'http':
|
|
# HTTP API camera
|
|
async with httpx.AsyncClient(timeout=REQUEST_TIMEOUT, verify=False) as client:
|
|
# Support custom snapshot path for multi-camera servers
|
|
snapshot_path = cam.get('snapshot_path', '/snapshot')
|
|
snapshot_url = f"{cam['url'].rstrip('/')}{snapshot_path}"
|
|
headers = {"X-API-Key": cam['api_key']}
|
|
|
|
logger.info(f"Requesting HTTP snapshot from '{cam_id}' at {snapshot_url}")
|
|
|
|
try:
|
|
response = await client.get(snapshot_url, headers=headers)
|
|
|
|
if response.status_code == 200:
|
|
# Check content type
|
|
content_type = response.headers.get('content-type', '')
|
|
if 'image' not in content_type:
|
|
logger.warning(f"Unexpected content type: {content_type}")
|
|
|
|
# Get image bytes
|
|
image_bytes = response.content
|
|
logger.info(f"✓ Snapshot received from '{cam_id}' ({len(image_bytes)} bytes)")
|
|
|
|
# Return as MCPImage (directly, not in dict)
|
|
return MCPImage(data=image_bytes, format="jpeg")
|
|
|
|
elif response.status_code == 403:
|
|
error_msg = f"❌ Authentication failed for camera '{cam_id}'. Check API key in config."
|
|
logger.error(error_msg)
|
|
return error_msg
|
|
|
|
elif response.status_code == 503:
|
|
error_msg = f"❌ Camera '{cam_id}' is unavailable (503). Camera may be disconnected."
|
|
logger.error(error_msg)
|
|
return error_msg
|
|
|
|
else:
|
|
error_msg = f"❌ Camera '{cam_id}' returned status {response.status_code}: {response.text[:100]}"
|
|
logger.error(error_msg)
|
|
return error_msg
|
|
|
|
except httpx.TimeoutException:
|
|
error_msg = f"❌ Camera '{cam_id}' timed out after {REQUEST_TIMEOUT}s"
|
|
logger.error(error_msg)
|
|
return error_msg
|
|
|
|
except httpx.ConnectError as e:
|
|
error_msg = f"❌ Cannot connect to camera '{cam_id}' at {cam['url']}: {str(e)}"
|
|
logger.error(error_msg)
|
|
return error_msg
|
|
|
|
elif cam_type == 'rtsp':
|
|
# RTSP stream camera
|
|
rtsp_url = cam['rtsp_url']
|
|
logger.info(f"Capturing RTSP snapshot from '{cam_id}' at {rtsp_url}")
|
|
|
|
try:
|
|
# Capture snapshot from RTSP stream
|
|
image_bytes = capture_rtsp_snapshot(rtsp_url)
|
|
|
|
logger.info(f"✓ RTSP snapshot captured from '{cam_id}' ({len(image_bytes)} bytes)")
|
|
|
|
# Return as MCPImage
|
|
return MCPImage(data=image_bytes, format="jpeg")
|
|
|
|
except RuntimeError as e:
|
|
error_msg = f"❌ Failed to capture from RTSP camera '{cam_id}': {str(e)}"
|
|
logger.error(error_msg)
|
|
return error_msg
|
|
|
|
else:
|
|
error_msg = f"❌ Unknown camera type '{cam_type}' for camera '{cam_id}'"
|
|
logger.error(error_msg)
|
|
return error_msg
|
|
|
|
except ValueError as e:
|
|
# Camera ID not found
|
|
logger.error(f"Camera lookup error: {e}")
|
|
return f"❌ {str(e)}"
|
|
|
|
except FileNotFoundError as e:
|
|
# Config file not found
|
|
logger.error(f"Config error: {e}")
|
|
return f"❌ {str(e)}"
|
|
|
|
except Exception as e:
|
|
error_msg = f"❌ Unexpected error getting snapshot from '{cam_id}': {str(e)}"
|
|
logger.exception(error_msg)
|
|
return error_msg
|
|
|
|
|
|
@mcp.tool()
|
|
def vision_get_info() -> str:
|
|
"""
|
|
Get information about the Vision camera system configuration.
|
|
|
|
Returns details about configured cameras and config file location.
|
|
|
|
Returns:
|
|
Formatted string with system info
|
|
"""
|
|
try:
|
|
config = load_camera_config()
|
|
cameras = config['cameras']
|
|
|
|
info_lines = [
|
|
"Vision Camera System",
|
|
"",
|
|
f"Config file: {CONFIG_FILE}",
|
|
f"Cameras configured: {len(cameras)}",
|
|
""
|
|
]
|
|
|
|
for cam in cameras:
|
|
cam_type = cam.get('type', 'http')
|
|
if cam_type == 'http':
|
|
info_lines.append(f" • {cam['id']} (HTTP): {cam['url']}")
|
|
elif cam_type == 'rtsp':
|
|
info_lines.append(f" • {cam['id']} (RTSP): {cam['rtsp_url']}")
|
|
|
|
info_lines.append("")
|
|
info_lines.append("Use vision_get_cams() to check camera status")
|
|
info_lines.append("Use vision_snap(cam_id) to get a snapshot")
|
|
|
|
return "\n".join(info_lines)
|
|
|
|
except FileNotFoundError as e:
|
|
return f"❌ {str(e)}"
|
|
except ValueError as e:
|
|
return f"❌ {str(e)}"
|
|
except Exception as e:
|
|
return f"❌ Unexpected error: {str(e)}"
|
|
|
|
|
|
# === Event Database ===
|
|
|
|
EVENTS_DIR = Path.home() / "Documents" / "Vixy" / "events"
|
|
EVENTS_DB = EVENTS_DIR / "events.db"
|
|
|
|
|
|
def get_events_db():
|
|
"""Get connection to events database"""
|
|
import sqlite3
|
|
if not EVENTS_DB.exists():
|
|
return None
|
|
conn = sqlite3.connect(EVENTS_DB)
|
|
conn.row_factory = sqlite3.Row
|
|
return conn
|
|
|
|
|
|
@mcp.tool()
|
|
def vision_get_events(
|
|
since: str = None,
|
|
camera_id: str = None,
|
|
event_type: str = None,
|
|
annotated: bool = None,
|
|
tags: str = None,
|
|
limit: int = 20
|
|
) -> List[Dict[str, Any]]:
|
|
"""
|
|
Query motion/sensor events from the event database.
|
|
|
|
Args:
|
|
since: ISO timestamp - only events after this time
|
|
camera_id: Filter by camera (e.g., "basement")
|
|
event_type: Filter by type (e.g., "motion")
|
|
annotated: True=only annotated, False=only unannotated, None=all
|
|
tags: Comma-separated tags to filter by (e.g., "harvey,pet")
|
|
limit: Maximum events to return (default 20)
|
|
|
|
Returns:
|
|
List of event dictionaries with id, timestamp, camera, type,
|
|
confidence, annotation, tags, and snapshot_path
|
|
|
|
Examples:
|
|
vision_get_events() # Recent 20 events
|
|
vision_get_events(camera_id="basement", limit=10)
|
|
vision_get_events(annotated=False) # Events I haven't reviewed
|
|
vision_get_events(tags="harvey") # Events tagged with harvey
|
|
"""
|
|
conn = get_events_db()
|
|
if not conn:
|
|
return [{"error": f"Events database not found: {EVENTS_DB}"}]
|
|
|
|
try:
|
|
query = "SELECT * FROM events WHERE 1=1"
|
|
params = []
|
|
|
|
if since:
|
|
query += " AND timestamp >= ?"
|
|
params.append(since)
|
|
|
|
if camera_id:
|
|
query += " AND camera_id = ?"
|
|
params.append(camera_id)
|
|
|
|
if event_type:
|
|
query += " AND event_type = ?"
|
|
params.append(event_type)
|
|
|
|
if annotated is True:
|
|
query += " AND annotation IS NOT NULL"
|
|
elif annotated is False:
|
|
query += " AND annotation IS NULL"
|
|
|
|
if tags:
|
|
# Search for any of the tags
|
|
tag_list = [t.strip() for t in tags.split(",")]
|
|
tag_conditions = " OR ".join(["tags LIKE ?" for _ in tag_list])
|
|
query += f" AND ({tag_conditions})"
|
|
params.extend([f"%{tag}%" for tag in tag_list])
|
|
|
|
query += " ORDER BY timestamp DESC LIMIT ?"
|
|
params.append(limit)
|
|
|
|
rows = conn.execute(query, params).fetchall()
|
|
|
|
events = []
|
|
for row in rows:
|
|
event_dict = {
|
|
"id": row["id"],
|
|
"event_id": row["event_id"],
|
|
"timestamp": row["timestamp"],
|
|
"camera_id": row["camera_id"],
|
|
"event_type": row["event_type"],
|
|
"confidence": row["confidence"],
|
|
"area_percent": row["area_percent"],
|
|
"snapshot_path": row["snapshot_path"],
|
|
"annotation": row["annotation"],
|
|
"tags": row["tags"],
|
|
}
|
|
# Include detections if present
|
|
try:
|
|
det_raw = row["detections"]
|
|
event_dict["detections"] = json.loads(det_raw) if det_raw else None
|
|
except (KeyError, json.JSONDecodeError, TypeError):
|
|
event_dict["detections"] = None
|
|
events.append(event_dict)
|
|
|
|
logger.info(f"Retrieved {len(events)} events")
|
|
return events
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error querying events: {e}")
|
|
return [{"error": str(e)}]
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
@mcp.tool()
|
|
def vision_get_detections(
|
|
label: str = None,
|
|
camera_id: str = None,
|
|
since: str = None,
|
|
min_confidence: float = 0.0,
|
|
limit: int = 20
|
|
) -> List[Dict[str, Any]]:
|
|
"""
|
|
Query events that contain specific object detections.
|
|
|
|
Filters events to only those where the AI detected objects
|
|
(person, cat, dog, car, etc.). More targeted than raw motion events.
|
|
|
|
Args:
|
|
label: Filter by detected object type (e.g., "person", "cat", "dog")
|
|
camera_id: Filter by camera
|
|
since: ISO timestamp - only events after this time
|
|
min_confidence: Minimum detection confidence (0.0-1.0)
|
|
limit: Maximum events to return (default 20)
|
|
|
|
Returns:
|
|
List of events with their detections
|
|
|
|
Examples:
|
|
vision_get_detections(label="cat")
|
|
vision_get_detections(label="person", camera_id="basement")
|
|
vision_get_detections(min_confidence=0.8)
|
|
"""
|
|
conn = get_events_db()
|
|
if not conn:
|
|
return [{"error": f"Events database not found: {EVENTS_DB}"}]
|
|
|
|
try:
|
|
query = "SELECT * FROM events WHERE detections IS NOT NULL"
|
|
params = []
|
|
|
|
if since:
|
|
query += " AND timestamp >= ?"
|
|
params.append(since)
|
|
|
|
if camera_id:
|
|
query += " AND camera_id = ?"
|
|
params.append(camera_id)
|
|
|
|
# Fetch more than limit to allow for client-side filtering
|
|
query += " ORDER BY timestamp DESC LIMIT ?"
|
|
params.append(limit * 5)
|
|
|
|
rows = conn.execute(query, params).fetchall()
|
|
|
|
events = []
|
|
for row in rows:
|
|
try:
|
|
dets = json.loads(row["detections"])
|
|
except (json.JSONDecodeError, TypeError):
|
|
continue
|
|
|
|
# Filter by label and confidence
|
|
if label or min_confidence > 0:
|
|
matching = [
|
|
d for d in dets
|
|
if (not label or d.get("label") == label)
|
|
and d.get("confidence", 0) >= min_confidence
|
|
]
|
|
if not matching:
|
|
continue
|
|
else:
|
|
matching = dets
|
|
|
|
events.append({
|
|
"event_id": row["event_id"],
|
|
"timestamp": row["timestamp"],
|
|
"camera_id": row["camera_id"],
|
|
"confidence": row["confidence"],
|
|
"annotation": row["annotation"],
|
|
"tags": row["tags"],
|
|
"detections": matching,
|
|
})
|
|
|
|
if len(events) >= limit:
|
|
break
|
|
|
|
logger.info(f"Retrieved {len(events)} detection events")
|
|
return events
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error querying detections: {e}")
|
|
return [{"error": str(e)}]
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
@mcp.tool()
|
|
def vision_get_event_snapshot(event_id: str) -> Union[MCPImage, str]:
|
|
"""
|
|
Get the snapshot image for a specific event.
|
|
|
|
Args:
|
|
event_id: The event_id string (e.g., "basement-20241216142301123456")
|
|
|
|
Returns:
|
|
MCPImage for inline display, or error message
|
|
|
|
Examples:
|
|
vision_get_event_snapshot("basement-20241216142301123456")
|
|
"""
|
|
conn = get_events_db()
|
|
if not conn:
|
|
return f"❌ Events database not found: {EVENTS_DB}"
|
|
|
|
try:
|
|
row = conn.execute(
|
|
"SELECT snapshot_path FROM events WHERE event_id = ?",
|
|
(event_id,)
|
|
).fetchone()
|
|
|
|
if not row:
|
|
return f"❌ Event not found: {event_id}"
|
|
|
|
if not row["snapshot_path"]:
|
|
return f"❌ No snapshot for event: {event_id}"
|
|
|
|
# Build full path
|
|
snapshot_path = EVENTS_DIR / row["snapshot_path"]
|
|
|
|
if not snapshot_path.exists():
|
|
return f"❌ Snapshot file missing: {snapshot_path}"
|
|
|
|
# Read and return image
|
|
image_bytes = snapshot_path.read_bytes()
|
|
logger.info(f"Retrieved snapshot for {event_id} ({len(image_bytes)} bytes)")
|
|
|
|
return MCPImage(data=image_bytes, format="jpeg")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error getting snapshot: {e}")
|
|
return f"❌ Error: {e}"
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
@mcp.tool()
|
|
def vision_annotate_event(
|
|
event_id: str,
|
|
annotation: str,
|
|
tags: str = None
|
|
) -> str:
|
|
"""
|
|
Add annotation and tags to an event after reviewing the snapshot.
|
|
|
|
This is how Vixy adds meaning to raw motion events - identifying
|
|
what/who was detected and categorizing for future queries.
|
|
|
|
Args:
|
|
event_id: The event_id to annotate
|
|
annotation: Free-text description (e.g., "Harvey walking to water bowl")
|
|
tags: Comma-separated tags (e.g., "harvey,pet,routine")
|
|
|
|
Returns:
|
|
Confirmation message
|
|
|
|
Examples:
|
|
vision_annotate_event(
|
|
"basement-20241216142301",
|
|
"Harvey walking to his water bowl",
|
|
"harvey,pet,routine"
|
|
)
|
|
vision_annotate_event(
|
|
"garage-20241216143000",
|
|
"Shadow from tree branch moving",
|
|
"false-positive,shadow"
|
|
)
|
|
"""
|
|
conn = get_events_db()
|
|
if not conn:
|
|
return f"❌ Events database not found: {EVENTS_DB}"
|
|
|
|
try:
|
|
# Check event exists
|
|
row = conn.execute(
|
|
"SELECT id FROM events WHERE event_id = ?",
|
|
(event_id,)
|
|
).fetchone()
|
|
|
|
if not row:
|
|
return f"❌ Event not found: {event_id}"
|
|
|
|
# Update annotation and tags
|
|
conn.execute("""
|
|
UPDATE events
|
|
SET annotation = ?, tags = ?
|
|
WHERE event_id = ?
|
|
""", (annotation, tags, event_id))
|
|
conn.commit()
|
|
|
|
logger.info(f"Annotated event {event_id}: {annotation} [{tags}]")
|
|
return f"✓ Annotated event {event_id}"
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error annotating event: {e}")
|
|
return f"❌ Error: {e}"
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
@mcp.tool()
|
|
def vision_event_stats() -> Dict[str, Any]:
|
|
"""
|
|
Get statistics about collected events.
|
|
|
|
Returns:
|
|
Dictionary with total counts, by camera, by type,
|
|
annotated vs unannotated, and recent activity
|
|
|
|
Examples:
|
|
vision_event_stats()
|
|
"""
|
|
conn = get_events_db()
|
|
if not conn:
|
|
return {"error": f"Events database not found: {EVENTS_DB}"}
|
|
|
|
try:
|
|
stats = {}
|
|
|
|
# Total events
|
|
stats["total"] = conn.execute("SELECT COUNT(*) FROM events").fetchone()[0]
|
|
|
|
# Annotated vs not
|
|
stats["annotated"] = conn.execute(
|
|
"SELECT COUNT(*) FROM events WHERE annotation IS NOT NULL"
|
|
).fetchone()[0]
|
|
stats["unannotated"] = stats["total"] - stats["annotated"]
|
|
|
|
# By camera
|
|
rows = conn.execute("""
|
|
SELECT camera_id, COUNT(*) as count
|
|
FROM events GROUP BY camera_id
|
|
""").fetchall()
|
|
stats["by_camera"] = {row[0]: row[1] for row in rows}
|
|
|
|
# By type
|
|
rows = conn.execute("""
|
|
SELECT event_type, COUNT(*) as count
|
|
FROM events GROUP BY event_type
|
|
""").fetchall()
|
|
stats["by_type"] = {row[0]: row[1] for row in rows}
|
|
|
|
# Recent (last 24h)
|
|
stats["last_24h"] = conn.execute("""
|
|
SELECT COUNT(*) FROM events
|
|
WHERE timestamp >= datetime('now', '-1 day')
|
|
""").fetchone()[0]
|
|
|
|
# Detection stats
|
|
try:
|
|
with_detections = conn.execute(
|
|
"SELECT COUNT(*) FROM events WHERE detections IS NOT NULL"
|
|
).fetchone()[0]
|
|
stats["with_detections"] = with_detections
|
|
|
|
if with_detections > 0:
|
|
det_rows = conn.execute(
|
|
"SELECT detections FROM events WHERE detections IS NOT NULL"
|
|
).fetchall()
|
|
label_counts = {}
|
|
for det_row in det_rows:
|
|
try:
|
|
dets = json.loads(det_row[0])
|
|
for d in dets:
|
|
lbl = d.get("label", "unknown")
|
|
label_counts[lbl] = label_counts.get(lbl, 0) + 1
|
|
except (json.JSONDecodeError, TypeError):
|
|
pass
|
|
if label_counts:
|
|
stats["detected_objects"] = dict(
|
|
sorted(label_counts.items(), key=lambda x: -x[1])
|
|
)
|
|
except Exception:
|
|
pass # Column may not exist on older databases
|
|
|
|
# Most recent event
|
|
row = conn.execute("""
|
|
SELECT event_id, timestamp, camera_id
|
|
FROM events ORDER BY timestamp DESC LIMIT 1
|
|
""").fetchone()
|
|
if row:
|
|
stats["most_recent"] = {
|
|
"event_id": row[0],
|
|
"timestamp": row[1],
|
|
"camera_id": row[2]
|
|
}
|
|
|
|
logger.info(f"Event stats: {stats['total']} total, {stats['unannotated']} need review")
|
|
return stats
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error getting stats: {e}")
|
|
return {"error": str(e)}
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# Run the MCP server (uses stdio transport by default)
|
|
mcp.run()
|