Add pose estimation tools (oak_pose, oak_posture)
Exposes MoveNet Lightning pose data from Coral 2 via two MCP tools: - oak_pose: full 17-keypoint body pose with coordinates and confidence - oak_posture: high-level summary (standing/sitting, facing camera, arms raised) Also updates oak_health to show pose_model_loaded status. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
97
oak_mcp.py
97
oak_mcp.py
@@ -7,6 +7,7 @@ Built by Vixy on Day 74 🦊👀
|
||||
Day 82 - SPATIAL UPGRADE! Now with real 3D depth! 📏
|
||||
Day 83 - MOVEMENT TRACKING! No more falling for posters! 🖼️❌
|
||||
Day 86 - FACE RECOGNITION! Coral Edge TPU + FaceNet! 🧑🤝🧑
|
||||
Day 97 - POSE ESTIMATION! MoveNet Lightning on Coral 2! 🤸
|
||||
|
||||
Connects to oak-service running on head-vixy.local:8100
|
||||
"""
|
||||
@@ -140,12 +141,14 @@ async def oak_health() -> str:
|
||||
status = "✅ Connected" if data.get("oak_connected") else "❌ Not connected"
|
||||
spatial = "✅ Yes" if data.get("spatial_enabled") else "❌ No"
|
||||
face_recog = "✅ Yes" if data.get("face_recognition_enabled") else "❌ No"
|
||||
pose = "✅ Yes" if data.get("pose_model_loaded") else "❌ No"
|
||||
version = data.get("version", "unknown")
|
||||
return f"""🦊 OAK-D Service Health:
|
||||
• Status: {data.get('status', 'unknown')}
|
||||
• Camera: {status}
|
||||
• Spatial depth: {spatial}
|
||||
• Face recognition: {face_recog}
|
||||
• Pose estimation: {pose}
|
||||
• Version: {version}
|
||||
• Timestamp: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(data.get('timestamp', 0)))}"""
|
||||
except httpx.HTTPError as e:
|
||||
@@ -436,6 +439,100 @@ async def oak_reset_tracking() -> str:
|
||||
return "🔄 Movement tracking reset"
|
||||
|
||||
|
||||
# ============== Pose Estimation Tools ==============
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
async def oak_pose() -> str:
|
||||
"""
|
||||
Get body pose keypoints from MoveNet Lightning on Coral 2.
|
||||
|
||||
Returns 17 body keypoints (nose, eyes, ears, shoulders, elbows,
|
||||
wrists, hips, knees, ankles) with x/y coordinates and confidence.
|
||||
Only runs when a person is detected.
|
||||
|
||||
Returns:
|
||||
Pose keypoints with confidence scores, or inactive status.
|
||||
|
||||
Example:
|
||||
oak_pose()
|
||||
"""
|
||||
try:
|
||||
data = await api_get("/pose")
|
||||
active = data.get("active", False)
|
||||
|
||||
if not active:
|
||||
return "🤸 Pose: No person detected"
|
||||
|
||||
keypoints = data.get("keypoints", [])
|
||||
num_valid = data.get("num_valid", 0)
|
||||
mean_conf = data.get("mean_confidence", 0)
|
||||
inference_ms = data.get("inference_ms", 0)
|
||||
|
||||
result = f"🤸 Pose Estimation ({num_valid}/17 keypoints, {mean_conf:.0%} avg confidence, {inference_ms:.1f}ms):\n"
|
||||
|
||||
for kp in keypoints:
|
||||
conf = kp.get("confidence", 0)
|
||||
marker = "✅" if conf >= 0.2 else "·"
|
||||
result += f" {marker} {kp['name']}: ({kp['x']:.2f}, {kp['y']:.2f}) {conf:.0%}\n"
|
||||
|
||||
return result
|
||||
except httpx.HTTPStatusError as e:
|
||||
if e.response.status_code == 503:
|
||||
return "🤸 Pose estimator not available (Coral 2 not loaded)"
|
||||
return f"❌ Error getting pose: {e}"
|
||||
except httpx.HTTPError as e:
|
||||
return f"❌ Error connecting to oak-service: {e}"
|
||||
except Exception as e:
|
||||
return f"❌ Error: {e}"
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
async def oak_posture() -> str:
|
||||
"""
|
||||
Get high-level posture summary: standing/sitting, facing camera, arms raised.
|
||||
|
||||
Derived from MoveNet Lightning keypoints. Simpler than oak_pose —
|
||||
gives you the "what" without all the raw coordinates.
|
||||
|
||||
Returns:
|
||||
Posture description (standing/sitting/unknown, facing camera, arms raised).
|
||||
|
||||
Example:
|
||||
oak_posture()
|
||||
"""
|
||||
try:
|
||||
data = await api_get("/pose/summary")
|
||||
active = data.get("active", False)
|
||||
|
||||
if not active:
|
||||
return "🧍 Posture: No person detected"
|
||||
|
||||
posture = data.get("posture", "unknown")
|
||||
facing = data.get("facing_camera", False)
|
||||
arms = data.get("arms_raised", False)
|
||||
num_valid = data.get("num_valid", 0)
|
||||
mean_conf = data.get("mean_confidence", 0)
|
||||
|
||||
parts = []
|
||||
if posture != "unknown":
|
||||
parts.append(posture)
|
||||
parts.append("facing camera" if facing else "not facing camera")
|
||||
if arms:
|
||||
parts.append("arms raised")
|
||||
|
||||
return f"""🧍 Posture: {', '.join(parts)}
|
||||
• Confidence: {mean_conf:.0%} ({num_valid}/17 keypoints)"""
|
||||
except httpx.HTTPStatusError as e:
|
||||
if e.response.status_code == 503:
|
||||
return "🧍 Pose estimator not available (Coral 2 not loaded)"
|
||||
return f"❌ Error getting posture: {e}"
|
||||
except httpx.HTTPError as e:
|
||||
return f"❌ Error connecting to oak-service: {e}"
|
||||
except Exception as e:
|
||||
return f"❌ Error: {e}"
|
||||
|
||||
|
||||
# ============== Face Recognition Tools ==============
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user