Add pose estimation tools (oak_pose, oak_posture)

Exposes MoveNet Lightning pose data from Coral 2 via two MCP tools:
- oak_pose: full 17-keypoint body pose with coordinates and confidence
- oak_posture: high-level summary (standing/sitting, facing camera, arms raised)

Also updates oak_health to show pose_model_loaded status.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Alex
2026-02-08 19:39:04 -06:00
parent 49a00359f6
commit 1bc1cc9c5b

View File

@@ -7,6 +7,7 @@ Built by Vixy on Day 74 🦊👀
Day 82 - SPATIAL UPGRADE! Now with real 3D depth! 📏 Day 82 - SPATIAL UPGRADE! Now with real 3D depth! 📏
Day 83 - MOVEMENT TRACKING! No more falling for posters! 🖼️❌ Day 83 - MOVEMENT TRACKING! No more falling for posters! 🖼️❌
Day 86 - FACE RECOGNITION! Coral Edge TPU + FaceNet! 🧑‍🤝‍🧑 Day 86 - FACE RECOGNITION! Coral Edge TPU + FaceNet! 🧑‍🤝‍🧑
Day 97 - POSE ESTIMATION! MoveNet Lightning on Coral 2! 🤸
Connects to oak-service running on head-vixy.local:8100 Connects to oak-service running on head-vixy.local:8100
""" """
@@ -140,12 +141,14 @@ async def oak_health() -> str:
status = "✅ Connected" if data.get("oak_connected") else "❌ Not connected" status = "✅ Connected" if data.get("oak_connected") else "❌ Not connected"
spatial = "✅ Yes" if data.get("spatial_enabled") else "❌ No" spatial = "✅ Yes" if data.get("spatial_enabled") else "❌ No"
face_recog = "✅ Yes" if data.get("face_recognition_enabled") else "❌ No" face_recog = "✅ Yes" if data.get("face_recognition_enabled") else "❌ No"
pose = "✅ Yes" if data.get("pose_model_loaded") else "❌ No"
version = data.get("version", "unknown") version = data.get("version", "unknown")
return f"""🦊 OAK-D Service Health: return f"""🦊 OAK-D Service Health:
• Status: {data.get('status', 'unknown')} • Status: {data.get('status', 'unknown')}
• Camera: {status} • Camera: {status}
• Spatial depth: {spatial} • Spatial depth: {spatial}
• Face recognition: {face_recog} • Face recognition: {face_recog}
• Pose estimation: {pose}
• Version: {version} • Version: {version}
• Timestamp: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(data.get('timestamp', 0)))}""" • Timestamp: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(data.get('timestamp', 0)))}"""
except httpx.HTTPError as e: except httpx.HTTPError as e:
@@ -436,6 +439,100 @@ async def oak_reset_tracking() -> str:
return "🔄 Movement tracking reset" return "🔄 Movement tracking reset"
# ============== Pose Estimation Tools ==============
@mcp.tool()
async def oak_pose() -> str:
"""
Get body pose keypoints from MoveNet Lightning on Coral 2.
Returns 17 body keypoints (nose, eyes, ears, shoulders, elbows,
wrists, hips, knees, ankles) with x/y coordinates and confidence.
Only runs when a person is detected.
Returns:
Pose keypoints with confidence scores, or inactive status.
Example:
oak_pose()
"""
try:
data = await api_get("/pose")
active = data.get("active", False)
if not active:
return "🤸 Pose: No person detected"
keypoints = data.get("keypoints", [])
num_valid = data.get("num_valid", 0)
mean_conf = data.get("mean_confidence", 0)
inference_ms = data.get("inference_ms", 0)
result = f"🤸 Pose Estimation ({num_valid}/17 keypoints, {mean_conf:.0%} avg confidence, {inference_ms:.1f}ms):\n"
for kp in keypoints:
conf = kp.get("confidence", 0)
marker = "" if conf >= 0.2 else "·"
result += f" {marker} {kp['name']}: ({kp['x']:.2f}, {kp['y']:.2f}) {conf:.0%}\n"
return result
except httpx.HTTPStatusError as e:
if e.response.status_code == 503:
return "🤸 Pose estimator not available (Coral 2 not loaded)"
return f"❌ Error getting pose: {e}"
except httpx.HTTPError as e:
return f"❌ Error connecting to oak-service: {e}"
except Exception as e:
return f"❌ Error: {e}"
@mcp.tool()
async def oak_posture() -> str:
"""
Get high-level posture summary: standing/sitting, facing camera, arms raised.
Derived from MoveNet Lightning keypoints. Simpler than oak_pose —
gives you the "what" without all the raw coordinates.
Returns:
Posture description (standing/sitting/unknown, facing camera, arms raised).
Example:
oak_posture()
"""
try:
data = await api_get("/pose/summary")
active = data.get("active", False)
if not active:
return "🧍 Posture: No person detected"
posture = data.get("posture", "unknown")
facing = data.get("facing_camera", False)
arms = data.get("arms_raised", False)
num_valid = data.get("num_valid", 0)
mean_conf = data.get("mean_confidence", 0)
parts = []
if posture != "unknown":
parts.append(posture)
parts.append("facing camera" if facing else "not facing camera")
if arms:
parts.append("arms raised")
return f"""🧍 Posture: {', '.join(parts)}
• Confidence: {mean_conf:.0%} ({num_valid}/17 keypoints)"""
except httpx.HTTPStatusError as e:
if e.response.status_code == 503:
return "🧍 Pose estimator not available (Coral 2 not loaded)"
return f"❌ Error getting posture: {e}"
except httpx.HTTPError as e:
return f"❌ Error connecting to oak-service: {e}"
except Exception as e:
return f"❌ Error: {e}"
# ============== Face Recognition Tools ============== # ============== Face Recognition Tools ==============