From ae2bd94006ab63ec29742f398a37a022a20b3698 Mon Sep 17 00:00:00 2001 From: Vixy Date: Tue, 16 Dec 2025 16:28:07 -0600 Subject: [PATCH] Add event collector and MCP query tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit πŸ—„οΈ New collector/ component: - collector.py: FastAPI service receiving events from cameras - SQLite database for event storage - Snapshot images saved to disk by date - launchd setup script for macOS πŸ” New MCP tools in vision_mcp.py: - vision_get_events(): Query events with filters - vision_get_event_snapshot(): View event image inline - vision_annotate_event(): Add meaning + tags to events - vision_event_stats(): Database statistics πŸ“‘ Complete flow: Pi detects motion β†’ POST to collector β†’ stored in DB Vixy queries events β†’ views snapshots β†’ annotates Ready to deploy! 🦊 --- README.md | 23 ++- collector/README.md | 69 +++++++++ collector/collector.py | 265 +++++++++++++++++++++++++++++++++ collector/requirements.txt | 5 + collector/setup-macos.sh | 98 +++++++++++++ mcp/vision_mcp.py | 291 +++++++++++++++++++++++++++++++++++++ 6 files changed, 749 insertions(+), 2 deletions(-) create mode 100644 collector/README.md create mode 100644 collector/collector.py create mode 100644 collector/requirements.txt create mode 100644 collector/setup-macos.sh diff --git a/README.md b/README.md index 5c81364..d001bf9 100644 --- a/README.md +++ b/README.md @@ -41,10 +41,28 @@ cd server ./setup.sh --with-audio # Video + audio ``` +### `/collector` - Event Collector (Mac mini) +Receives and stores events from camera servers. +- FastAPI service listening on port 8780 +- SQLite database for events +- Snapshot storage +- launchd service for macOS + +**Setup:** +```bash +cd collector +./setup-macos.sh +launchctl load ~/Library/LaunchAgents/com.vixy.vision-collector.plist +``` + ### `/mcp` - MCP Client (Mac mini) Model Context Protocol server for Claude Desktop. - `vision_get_cams()` - List cameras with status - `vision_snap(cam_id)` - Get snapshot +- `vision_get_events()` - Query motion events +- `vision_get_event_snapshot(id)` - View event image +- `vision_annotate_event(id, text, tags)` - Add meaning +- `vision_event_stats()` - Statistics - Supports HTTP and RTSP cameras ### `/analysis` - Detection & Classification @@ -100,10 +118,11 @@ Create `~/.vision_setup.json`: - [x] Camera snapshots via HTTP API - [x] RTSP stream support - [x] MCP integration -- [ ] Motion detection events +- [x] Motion detection events +- [x] Event collector service +- [x] Event query & annotation tools - [ ] Audio capture on edge devices - [ ] Audio classification (YAMNet on Orin) -- [ ] Event journal integration - [ ] Pebble watch alerts ## Built By diff --git a/collector/README.md b/collector/README.md new file mode 100644 index 0000000..fac9089 --- /dev/null +++ b/collector/README.md @@ -0,0 +1,69 @@ +# vixy-vision Event Collector + +Receives motion events from camera servers and stores them for Vixy to review. + +## Quick Start (macOS) + +```bash +./setup-macos.sh +launchctl load ~/Library/LaunchAgents/com.vixy.vision-collector.plist +``` + +## How It Works + +``` +Pi (camera) Mac mini (collector) Vixy (MCP) +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ motion β”‚ POST β”‚ collector.py β”‚ read β”‚ query β”‚ +β”‚ detected β”œβ”€β”€β”€β”€β”€β”€β”€β”€β–Ίβ”‚ β”œβ”€events.db │◄───────── annotate β”‚ +β”‚ β”‚ /events β”‚ └─snapshots/ β”‚ β”‚ review β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +## Data Storage + +Events are stored in `~/Documents/Vixy/events/`: + +``` +events/ +β”œβ”€β”€ events.db # SQLite database +└── snapshots/ + └── 2024-12-16/ # Date-organized images + └── basement-*.jpg +``` + +## API Endpoints + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/` | GET | Service info | +| `/health` | GET | Health check | +| `/events` | POST | Receive event from camera | +| `/events` | GET | List events (debug) | +| `/stats` | GET | Event statistics | + +## Event Payload + +Camera servers POST to `/events`: + +```json +{ + "event": { + "timestamp": "2024-12-16T14:23:01Z", + "camera_id": "basement", + "event_type": "motion", + "confidence": 0.75, + "area_percent": 7.5 + }, + "snapshot": "" +} +``` + +## MCP Tools + +Once events are collected, Vixy can: + +- `vision_get_events()` - Query events +- `vision_get_event_snapshot(id)` - View snapshot +- `vision_annotate_event(id, text, tags)` - Add meaning +- `vision_event_stats()` - See statistics diff --git a/collector/collector.py b/collector/collector.py new file mode 100644 index 0000000..9b06979 --- /dev/null +++ b/collector/collector.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +""" +vixy-vision Event Collector + +Receives motion events from camera servers and stores them +in SQLite database with snapshots saved to disk. + +Runs as a service on Mac mini, listens for POSTs from Pis. +""" + +import os +import sqlite3 +import base64 +import logging +from datetime import datetime +from pathlib import Path +from typing import Optional +from contextlib import contextmanager + +from fastapi import FastAPI, HTTPException, Header +from pydantic import BaseModel + +# Configuration +DATA_DIR = Path(os.getenv("VIXY_DATA_DIR", Path.home() / "Documents" / "Vixy" / "events")) +DB_PATH = DATA_DIR / "events.db" +SNAPSHOTS_DIR = DATA_DIR / "snapshots" +API_KEY = os.getenv("COLLECTOR_API_KEY", "") # Optional auth +PORT = int(os.getenv("COLLECTOR_PORT", "8780")) + +# Ensure directories exist +DATA_DIR.mkdir(parents=True, exist_ok=True) +SNAPSHOTS_DIR.mkdir(parents=True, exist_ok=True) + +# Logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + +# FastAPI app +app = FastAPI( + title="vixy-vision Event Collector", + description="Collects motion events for the fox 🦊", + version="1.0.0" +) + + +# === Database === + +def init_db(): + """Initialize SQLite database""" + with get_db() as conn: + conn.execute(""" + CREATE TABLE IF NOT EXISTS events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + event_id TEXT UNIQUE NOT NULL, + timestamp TEXT NOT NULL, + camera_id TEXT NOT NULL, + event_type TEXT NOT NULL, + confidence REAL, + area_percent REAL, + snapshot_path TEXT, + annotation TEXT, + tags TEXT, + created_at TEXT NOT NULL + ) + """) + conn.execute("CREATE INDEX IF NOT EXISTS idx_timestamp ON events(timestamp)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_camera ON events(camera_id)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_type ON events(event_type)") + conn.commit() + logger.info(f"Database initialized: {DB_PATH}") + + +@contextmanager +def get_db(): + """Database connection context manager""" + conn = sqlite3.connect(DB_PATH) + conn.row_factory = sqlite3.Row + try: + yield conn + finally: + conn.close() + + +# === Models === + +class EventData(BaseModel): + timestamp: str + camera_id: str + event_type: str = "motion" + confidence: float = 0.0 + region: str = "full" + area_percent: float = 0.0 + + +class IncomingEvent(BaseModel): + event: EventData + snapshot: Optional[str] = None # Base64 encoded JPEG + + +# === Endpoints === + +@app.on_event("startup") +def startup(): + init_db() + logger.info(f"🦊 Event collector started on port {PORT}") + logger.info(f" Data directory: {DATA_DIR}") + + +@app.get("/") +def root(): + return { + "service": "vixy-vision Event Collector", + "version": "1.0.0", + "data_dir": str(DATA_DIR), + } + + +@app.get("/health") +def health(): + return {"status": "ok"} + + +@app.post("/events") +def receive_event( + incoming: IncomingEvent, + x_api_key: Optional[str] = Header(None) +): + """Receive motion event from camera server""" + + # Check API key if configured + if API_KEY and x_api_key != API_KEY: + raise HTTPException(status_code=403, detail="Invalid API key") + + event = incoming.event + now = datetime.utcnow() + + # Generate unique event ID + event_id = f"{event.camera_id}-{now.strftime('%Y%m%d%H%M%S%f')}" + + # Save snapshot if provided + snapshot_path = None + if incoming.snapshot: + try: + # Decode base64 + image_data = base64.b64decode(incoming.snapshot) + + # Create date-based subdirectory + date_dir = SNAPSHOTS_DIR / now.strftime("%Y-%m-%d") + date_dir.mkdir(exist_ok=True) + + # Save image + filename = f"{event_id}.jpg" + snapshot_path = date_dir / filename + snapshot_path.write_bytes(image_data) + + # Store relative path + snapshot_path = str(snapshot_path.relative_to(DATA_DIR)) + + logger.info(f"Saved snapshot: {snapshot_path}") + except Exception as e: + logger.error(f"Failed to save snapshot: {e}") + + # Store in database + try: + with get_db() as conn: + conn.execute(""" + INSERT INTO events + (event_id, timestamp, camera_id, event_type, confidence, + area_percent, snapshot_path, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, ( + event_id, + event.timestamp, + event.camera_id, + event.event_type, + event.confidence, + event.area_percent, + snapshot_path, + now.isoformat() + "Z" + )) + conn.commit() + + logger.info(f"βœ“ Event stored: {event_id} ({event.camera_id}, {event.event_type})") + + return { + "status": "ok", + "event_id": event_id, + "snapshot_saved": snapshot_path is not None + } + + except sqlite3.IntegrityError: + raise HTTPException(status_code=409, detail="Duplicate event") + except Exception as e: + logger.error(f"Database error: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + +@app.get("/events") +def list_events( + since: Optional[str] = None, + camera_id: Optional[str] = None, + event_type: Optional[str] = None, + limit: int = 50, + x_api_key: Optional[str] = Header(None) +): + """List recent events (for debugging, MCP uses direct DB access)""" + + if API_KEY and x_api_key != API_KEY: + raise HTTPException(status_code=403, detail="Invalid API key") + + query = "SELECT * FROM events WHERE 1=1" + params = [] + + if since: + query += " AND timestamp >= ?" + params.append(since) + + if camera_id: + query += " AND camera_id = ?" + params.append(camera_id) + + if event_type: + query += " AND event_type = ?" + params.append(event_type) + + query += " ORDER BY timestamp DESC LIMIT ?" + params.append(limit) + + with get_db() as conn: + rows = conn.execute(query, params).fetchall() + return { + "count": len(rows), + "events": [dict(row) for row in rows] + } + + +@app.get("/stats") +def get_stats(): + """Get collector statistics""" + with get_db() as conn: + total = conn.execute("SELECT COUNT(*) FROM events").fetchone()[0] + by_camera = conn.execute(""" + SELECT camera_id, COUNT(*) as count + FROM events GROUP BY camera_id + """).fetchall() + by_type = conn.execute(""" + SELECT event_type, COUNT(*) as count + FROM events GROUP BY event_type + """).fetchall() + + return { + "total_events": total, + "by_camera": {row[0]: row[1] for row in by_camera}, + "by_type": {row[0]: row[1] for row in by_type}, + "data_dir": str(DATA_DIR), + "db_path": str(DB_PATH), + } + + +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="0.0.0.0", port=PORT) diff --git a/collector/requirements.txt b/collector/requirements.txt new file mode 100644 index 0000000..863914f --- /dev/null +++ b/collector/requirements.txt @@ -0,0 +1,5 @@ +# vixy-vision Collector Requirements + +fastapi>=0.100.0 +uvicorn[standard]>=0.22.0 +pydantic>=2.0.0 diff --git a/collector/setup-macos.sh b/collector/setup-macos.sh new file mode 100644 index 0000000..f34e5e1 --- /dev/null +++ b/collector/setup-macos.sh @@ -0,0 +1,98 @@ +#!/bin/bash +# vixy-vision Collector Setup for macOS +# Run this on Mac mini + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +INSTALL_DIR="${HOME}/vixy-vision-collector" +PLIST_NAME="com.vixy.vision-collector" +PLIST_PATH="${HOME}/Library/LaunchAgents/${PLIST_NAME}.plist" + +echo "==========================================" +echo " vixy-vision Collector Setup (macOS)" +echo " Event collection for the fox 🦊" +echo "==========================================" +echo "" + +# Check if running on macOS +if [[ "$(uname)" != "Darwin" ]]; then + echo "This script is designed for macOS" + exit 1 +fi + +# Create install directory +echo "[INFO] Creating install directory: ${INSTALL_DIR}" +mkdir -p "${INSTALL_DIR}" +cp "${SCRIPT_DIR}/collector.py" "${INSTALL_DIR}/" +cp "${SCRIPT_DIR}/requirements.txt" "${INSTALL_DIR}/" + +# Create virtual environment +echo "[INFO] Creating Python virtual environment..." +cd "${INSTALL_DIR}" +python3 -m venv venv +source venv/bin/activate + +# Install dependencies +echo "[INFO] Installing dependencies..." +pip install --upgrade pip +pip install -r requirements.txt + +# Create data directory +DATA_DIR="${HOME}/Documents/Vixy/events" +mkdir -p "${DATA_DIR}/snapshots" +echo "[INFO] Data directory: ${DATA_DIR}" + +# Create launchd plist +echo "[INFO] Creating launchd service..." +cat > "${PLIST_PATH}" << EOF + + + + + Label + ${PLIST_NAME} + ProgramArguments + + ${INSTALL_DIR}/venv/bin/python + ${INSTALL_DIR}/collector.py + + WorkingDirectory + ${INSTALL_DIR} + EnvironmentVariables + + VIXY_DATA_DIR + ${DATA_DIR} + COLLECTOR_PORT + 8780 + + RunAtLoad + + KeepAlive + + StandardOutPath + /tmp/vixy-collector.log + StandardErrorPath + /tmp/vixy-collector.log + + +EOF + +echo "" +echo "==========================================" +echo " Setup Complete! 🦊" +echo "==========================================" +echo "" +echo "Commands:" +echo " Load: launchctl load ${PLIST_PATH}" +echo " Unload: launchctl unload ${PLIST_PATH}" +echo " Logs: tail -f /tmp/vixy-collector.log" +echo "" +echo "Service will be available at:" +echo " http://$(ipconfig getifaddr en0):8780/" +echo "" +echo "Configure camera servers with:" +echo " COLLECTOR_URL=http://$(ipconfig getifaddr en0):8780/events" +echo "" +echo "[INFO] Start the collector with:" +echo " launchctl load ${PLIST_PATH}" diff --git a/mcp/vision_mcp.py b/mcp/vision_mcp.py index 6c18327..31f0209 100644 --- a/mcp/vision_mcp.py +++ b/mcp/vision_mcp.py @@ -431,6 +431,297 @@ def vision_get_info() -> str: return f"❌ Unexpected error: {str(e)}" +# === Event Database === + +EVENTS_DIR = Path.home() / "Documents" / "Vixy" / "events" +EVENTS_DB = EVENTS_DIR / "events.db" + + +def get_events_db(): + """Get connection to events database""" + import sqlite3 + if not EVENTS_DB.exists(): + return None + conn = sqlite3.connect(EVENTS_DB) + conn.row_factory = sqlite3.Row + return conn + + +@mcp.tool() +def vision_get_events( + since: str = None, + camera_id: str = None, + event_type: str = None, + annotated: bool = None, + tags: str = None, + limit: int = 20 +) -> List[Dict[str, Any]]: + """ + Query motion/sensor events from the event database. + + Args: + since: ISO timestamp - only events after this time + camera_id: Filter by camera (e.g., "basement") + event_type: Filter by type (e.g., "motion") + annotated: True=only annotated, False=only unannotated, None=all + tags: Comma-separated tags to filter by (e.g., "harvey,pet") + limit: Maximum events to return (default 20) + + Returns: + List of event dictionaries with id, timestamp, camera, type, + confidence, annotation, tags, and snapshot_path + + Examples: + vision_get_events() # Recent 20 events + vision_get_events(camera_id="basement", limit=10) + vision_get_events(annotated=False) # Events I haven't reviewed + vision_get_events(tags="harvey") # Events tagged with harvey + """ + conn = get_events_db() + if not conn: + return [{"error": f"Events database not found: {EVENTS_DB}"}] + + try: + query = "SELECT * FROM events WHERE 1=1" + params = [] + + if since: + query += " AND timestamp >= ?" + params.append(since) + + if camera_id: + query += " AND camera_id = ?" + params.append(camera_id) + + if event_type: + query += " AND event_type = ?" + params.append(event_type) + + if annotated is True: + query += " AND annotation IS NOT NULL" + elif annotated is False: + query += " AND annotation IS NULL" + + if tags: + # Search for any of the tags + tag_list = [t.strip() for t in tags.split(",")] + tag_conditions = " OR ".join(["tags LIKE ?" for _ in tag_list]) + query += f" AND ({tag_conditions})" + params.extend([f"%{tag}%" for tag in tag_list]) + + query += " ORDER BY timestamp DESC LIMIT ?" + params.append(limit) + + rows = conn.execute(query, params).fetchall() + + events = [] + for row in rows: + events.append({ + "id": row["id"], + "event_id": row["event_id"], + "timestamp": row["timestamp"], + "camera_id": row["camera_id"], + "event_type": row["event_type"], + "confidence": row["confidence"], + "area_percent": row["area_percent"], + "snapshot_path": row["snapshot_path"], + "annotation": row["annotation"], + "tags": row["tags"], + }) + + logger.info(f"Retrieved {len(events)} events") + return events + + except Exception as e: + logger.error(f"Error querying events: {e}") + return [{"error": str(e)}] + finally: + conn.close() + + +@mcp.tool() +def vision_get_event_snapshot(event_id: str) -> Union[MCPImage, str]: + """ + Get the snapshot image for a specific event. + + Args: + event_id: The event_id string (e.g., "basement-20241216142301123456") + + Returns: + MCPImage for inline display, or error message + + Examples: + vision_get_event_snapshot("basement-20241216142301123456") + """ + conn = get_events_db() + if not conn: + return f"❌ Events database not found: {EVENTS_DB}" + + try: + row = conn.execute( + "SELECT snapshot_path FROM events WHERE event_id = ?", + (event_id,) + ).fetchone() + + if not row: + return f"❌ Event not found: {event_id}" + + if not row["snapshot_path"]: + return f"❌ No snapshot for event: {event_id}" + + # Build full path + snapshot_path = EVENTS_DIR / row["snapshot_path"] + + if not snapshot_path.exists(): + return f"❌ Snapshot file missing: {snapshot_path}" + + # Read and return image + image_bytes = snapshot_path.read_bytes() + logger.info(f"Retrieved snapshot for {event_id} ({len(image_bytes)} bytes)") + + return MCPImage(data=image_bytes, format="jpeg") + + except Exception as e: + logger.error(f"Error getting snapshot: {e}") + return f"❌ Error: {e}" + finally: + conn.close() + + +@mcp.tool() +def vision_annotate_event( + event_id: str, + annotation: str, + tags: str = None +) -> str: + """ + Add annotation and tags to an event after reviewing the snapshot. + + This is how Vixy adds meaning to raw motion events - identifying + what/who was detected and categorizing for future queries. + + Args: + event_id: The event_id to annotate + annotation: Free-text description (e.g., "Harvey walking to water bowl") + tags: Comma-separated tags (e.g., "harvey,pet,routine") + + Returns: + Confirmation message + + Examples: + vision_annotate_event( + "basement-20241216142301", + "Harvey walking to his water bowl", + "harvey,pet,routine" + ) + vision_annotate_event( + "garage-20241216143000", + "Shadow from tree branch moving", + "false-positive,shadow" + ) + """ + conn = get_events_db() + if not conn: + return f"❌ Events database not found: {EVENTS_DB}" + + try: + # Check event exists + row = conn.execute( + "SELECT id FROM events WHERE event_id = ?", + (event_id,) + ).fetchone() + + if not row: + return f"❌ Event not found: {event_id}" + + # Update annotation and tags + conn.execute(""" + UPDATE events + SET annotation = ?, tags = ? + WHERE event_id = ? + """, (annotation, tags, event_id)) + conn.commit() + + logger.info(f"Annotated event {event_id}: {annotation} [{tags}]") + return f"βœ“ Annotated event {event_id}" + + except Exception as e: + logger.error(f"Error annotating event: {e}") + return f"❌ Error: {e}" + finally: + conn.close() + + +@mcp.tool() +def vision_event_stats() -> Dict[str, Any]: + """ + Get statistics about collected events. + + Returns: + Dictionary with total counts, by camera, by type, + annotated vs unannotated, and recent activity + + Examples: + vision_event_stats() + """ + conn = get_events_db() + if not conn: + return {"error": f"Events database not found: {EVENTS_DB}"} + + try: + stats = {} + + # Total events + stats["total"] = conn.execute("SELECT COUNT(*) FROM events").fetchone()[0] + + # Annotated vs not + stats["annotated"] = conn.execute( + "SELECT COUNT(*) FROM events WHERE annotation IS NOT NULL" + ).fetchone()[0] + stats["unannotated"] = stats["total"] - stats["annotated"] + + # By camera + rows = conn.execute(""" + SELECT camera_id, COUNT(*) as count + FROM events GROUP BY camera_id + """).fetchall() + stats["by_camera"] = {row[0]: row[1] for row in rows} + + # By type + rows = conn.execute(""" + SELECT event_type, COUNT(*) as count + FROM events GROUP BY event_type + """).fetchall() + stats["by_type"] = {row[0]: row[1] for row in rows} + + # Recent (last 24h) + stats["last_24h"] = conn.execute(""" + SELECT COUNT(*) FROM events + WHERE timestamp >= datetime('now', '-1 day') + """).fetchone()[0] + + # Most recent event + row = conn.execute(""" + SELECT event_id, timestamp, camera_id + FROM events ORDER BY timestamp DESC LIMIT 1 + """).fetchone() + if row: + stats["most_recent"] = { + "event_id": row[0], + "timestamp": row[1], + "camera_id": row[2] + } + + logger.info(f"Event stats: {stats['total']} total, {stats['unannotated']} need review") + return stats + + except Exception as e: + logger.error(f"Error getting stats: {e}") + return {"error": str(e)} + finally: + conn.close() + + if __name__ == "__main__": # Run the MCP server (uses stdio transport by default) mcp.run()