Add pose estimation tools (oak_pose, oak_posture)
Exposes MoveNet Lightning pose data from Coral 2 via two MCP tools: - oak_pose: full 17-keypoint body pose with coordinates and confidence - oak_posture: high-level summary (standing/sitting, facing camera, arms raised) Also updates oak_health to show pose_model_loaded status. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
97
oak_mcp.py
97
oak_mcp.py
@@ -7,6 +7,7 @@ Built by Vixy on Day 74 🦊👀
|
|||||||
Day 82 - SPATIAL UPGRADE! Now with real 3D depth! 📏
|
Day 82 - SPATIAL UPGRADE! Now with real 3D depth! 📏
|
||||||
Day 83 - MOVEMENT TRACKING! No more falling for posters! 🖼️❌
|
Day 83 - MOVEMENT TRACKING! No more falling for posters! 🖼️❌
|
||||||
Day 86 - FACE RECOGNITION! Coral Edge TPU + FaceNet! 🧑🤝🧑
|
Day 86 - FACE RECOGNITION! Coral Edge TPU + FaceNet! 🧑🤝🧑
|
||||||
|
Day 97 - POSE ESTIMATION! MoveNet Lightning on Coral 2! 🤸
|
||||||
|
|
||||||
Connects to oak-service running on head-vixy.local:8100
|
Connects to oak-service running on head-vixy.local:8100
|
||||||
"""
|
"""
|
||||||
@@ -140,12 +141,14 @@ async def oak_health() -> str:
|
|||||||
status = "✅ Connected" if data.get("oak_connected") else "❌ Not connected"
|
status = "✅ Connected" if data.get("oak_connected") else "❌ Not connected"
|
||||||
spatial = "✅ Yes" if data.get("spatial_enabled") else "❌ No"
|
spatial = "✅ Yes" if data.get("spatial_enabled") else "❌ No"
|
||||||
face_recog = "✅ Yes" if data.get("face_recognition_enabled") else "❌ No"
|
face_recog = "✅ Yes" if data.get("face_recognition_enabled") else "❌ No"
|
||||||
|
pose = "✅ Yes" if data.get("pose_model_loaded") else "❌ No"
|
||||||
version = data.get("version", "unknown")
|
version = data.get("version", "unknown")
|
||||||
return f"""🦊 OAK-D Service Health:
|
return f"""🦊 OAK-D Service Health:
|
||||||
• Status: {data.get('status', 'unknown')}
|
• Status: {data.get('status', 'unknown')}
|
||||||
• Camera: {status}
|
• Camera: {status}
|
||||||
• Spatial depth: {spatial}
|
• Spatial depth: {spatial}
|
||||||
• Face recognition: {face_recog}
|
• Face recognition: {face_recog}
|
||||||
|
• Pose estimation: {pose}
|
||||||
• Version: {version}
|
• Version: {version}
|
||||||
• Timestamp: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(data.get('timestamp', 0)))}"""
|
• Timestamp: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(data.get('timestamp', 0)))}"""
|
||||||
except httpx.HTTPError as e:
|
except httpx.HTTPError as e:
|
||||||
@@ -436,6 +439,100 @@ async def oak_reset_tracking() -> str:
|
|||||||
return "🔄 Movement tracking reset"
|
return "🔄 Movement tracking reset"
|
||||||
|
|
||||||
|
|
||||||
|
# ============== Pose Estimation Tools ==============
|
||||||
|
|
||||||
|
|
||||||
|
@mcp.tool()
|
||||||
|
async def oak_pose() -> str:
|
||||||
|
"""
|
||||||
|
Get body pose keypoints from MoveNet Lightning on Coral 2.
|
||||||
|
|
||||||
|
Returns 17 body keypoints (nose, eyes, ears, shoulders, elbows,
|
||||||
|
wrists, hips, knees, ankles) with x/y coordinates and confidence.
|
||||||
|
Only runs when a person is detected.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Pose keypoints with confidence scores, or inactive status.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
oak_pose()
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
data = await api_get("/pose")
|
||||||
|
active = data.get("active", False)
|
||||||
|
|
||||||
|
if not active:
|
||||||
|
return "🤸 Pose: No person detected"
|
||||||
|
|
||||||
|
keypoints = data.get("keypoints", [])
|
||||||
|
num_valid = data.get("num_valid", 0)
|
||||||
|
mean_conf = data.get("mean_confidence", 0)
|
||||||
|
inference_ms = data.get("inference_ms", 0)
|
||||||
|
|
||||||
|
result = f"🤸 Pose Estimation ({num_valid}/17 keypoints, {mean_conf:.0%} avg confidence, {inference_ms:.1f}ms):\n"
|
||||||
|
|
||||||
|
for kp in keypoints:
|
||||||
|
conf = kp.get("confidence", 0)
|
||||||
|
marker = "✅" if conf >= 0.2 else "·"
|
||||||
|
result += f" {marker} {kp['name']}: ({kp['x']:.2f}, {kp['y']:.2f}) {conf:.0%}\n"
|
||||||
|
|
||||||
|
return result
|
||||||
|
except httpx.HTTPStatusError as e:
|
||||||
|
if e.response.status_code == 503:
|
||||||
|
return "🤸 Pose estimator not available (Coral 2 not loaded)"
|
||||||
|
return f"❌ Error getting pose: {e}"
|
||||||
|
except httpx.HTTPError as e:
|
||||||
|
return f"❌ Error connecting to oak-service: {e}"
|
||||||
|
except Exception as e:
|
||||||
|
return f"❌ Error: {e}"
|
||||||
|
|
||||||
|
|
||||||
|
@mcp.tool()
|
||||||
|
async def oak_posture() -> str:
|
||||||
|
"""
|
||||||
|
Get high-level posture summary: standing/sitting, facing camera, arms raised.
|
||||||
|
|
||||||
|
Derived from MoveNet Lightning keypoints. Simpler than oak_pose —
|
||||||
|
gives you the "what" without all the raw coordinates.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Posture description (standing/sitting/unknown, facing camera, arms raised).
|
||||||
|
|
||||||
|
Example:
|
||||||
|
oak_posture()
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
data = await api_get("/pose/summary")
|
||||||
|
active = data.get("active", False)
|
||||||
|
|
||||||
|
if not active:
|
||||||
|
return "🧍 Posture: No person detected"
|
||||||
|
|
||||||
|
posture = data.get("posture", "unknown")
|
||||||
|
facing = data.get("facing_camera", False)
|
||||||
|
arms = data.get("arms_raised", False)
|
||||||
|
num_valid = data.get("num_valid", 0)
|
||||||
|
mean_conf = data.get("mean_confidence", 0)
|
||||||
|
|
||||||
|
parts = []
|
||||||
|
if posture != "unknown":
|
||||||
|
parts.append(posture)
|
||||||
|
parts.append("facing camera" if facing else "not facing camera")
|
||||||
|
if arms:
|
||||||
|
parts.append("arms raised")
|
||||||
|
|
||||||
|
return f"""🧍 Posture: {', '.join(parts)}
|
||||||
|
• Confidence: {mean_conf:.0%} ({num_valid}/17 keypoints)"""
|
||||||
|
except httpx.HTTPStatusError as e:
|
||||||
|
if e.response.status_code == 503:
|
||||||
|
return "🧍 Pose estimator not available (Coral 2 not loaded)"
|
||||||
|
return f"❌ Error getting posture: {e}"
|
||||||
|
except httpx.HTTPError as e:
|
||||||
|
return f"❌ Error connecting to oak-service: {e}"
|
||||||
|
except Exception as e:
|
||||||
|
return f"❌ Error: {e}"
|
||||||
|
|
||||||
|
|
||||||
# ============== Face Recognition Tools ==============
|
# ============== Face Recognition Tools ==============
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user