diff --git a/oak_mcp.py b/oak_mcp.py index 1df9c7d..7a34c91 100644 --- a/oak_mcp.py +++ b/oak_mcp.py @@ -7,6 +7,7 @@ Built by Vixy on Day 74 ๐ŸฆŠ๐Ÿ‘€ Day 82 - SPATIAL UPGRADE! Now with real 3D depth! ๐Ÿ“ Day 83 - MOVEMENT TRACKING! No more falling for posters! ๐Ÿ–ผ๏ธโŒ Day 86 - FACE RECOGNITION! Coral Edge TPU + FaceNet! ๐Ÿง‘โ€๐Ÿคโ€๐Ÿง‘ +Day 97 - POSE ESTIMATION! MoveNet Lightning on Coral 2! ๐Ÿคธ Connects to oak-service running on head-vixy.local:8100 """ @@ -140,12 +141,14 @@ async def oak_health() -> str: status = "โœ… Connected" if data.get("oak_connected") else "โŒ Not connected" spatial = "โœ… Yes" if data.get("spatial_enabled") else "โŒ No" face_recog = "โœ… Yes" if data.get("face_recognition_enabled") else "โŒ No" + pose = "โœ… Yes" if data.get("pose_model_loaded") else "โŒ No" version = data.get("version", "unknown") return f"""๐ŸฆŠ OAK-D Service Health: โ€ข Status: {data.get('status', 'unknown')} โ€ข Camera: {status} โ€ข Spatial depth: {spatial} โ€ข Face recognition: {face_recog} +โ€ข Pose estimation: {pose} โ€ข Version: {version} โ€ข Timestamp: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(data.get('timestamp', 0)))}""" except httpx.HTTPError as e: @@ -436,6 +439,100 @@ async def oak_reset_tracking() -> str: return "๐Ÿ”„ Movement tracking reset" +# ============== Pose Estimation Tools ============== + + +@mcp.tool() +async def oak_pose() -> str: + """ + Get body pose keypoints from MoveNet Lightning on Coral 2. + + Returns 17 body keypoints (nose, eyes, ears, shoulders, elbows, + wrists, hips, knees, ankles) with x/y coordinates and confidence. + Only runs when a person is detected. + + Returns: + Pose keypoints with confidence scores, or inactive status. + + Example: + oak_pose() + """ + try: + data = await api_get("/pose") + active = data.get("active", False) + + if not active: + return "๐Ÿคธ Pose: No person detected" + + keypoints = data.get("keypoints", []) + num_valid = data.get("num_valid", 0) + mean_conf = data.get("mean_confidence", 0) + inference_ms = data.get("inference_ms", 0) + + result = f"๐Ÿคธ Pose Estimation ({num_valid}/17 keypoints, {mean_conf:.0%} avg confidence, {inference_ms:.1f}ms):\n" + + for kp in keypoints: + conf = kp.get("confidence", 0) + marker = "โœ…" if conf >= 0.2 else "ยท" + result += f" {marker} {kp['name']}: ({kp['x']:.2f}, {kp['y']:.2f}) {conf:.0%}\n" + + return result + except httpx.HTTPStatusError as e: + if e.response.status_code == 503: + return "๐Ÿคธ Pose estimator not available (Coral 2 not loaded)" + return f"โŒ Error getting pose: {e}" + except httpx.HTTPError as e: + return f"โŒ Error connecting to oak-service: {e}" + except Exception as e: + return f"โŒ Error: {e}" + + +@mcp.tool() +async def oak_posture() -> str: + """ + Get high-level posture summary: standing/sitting, facing camera, arms raised. + + Derived from MoveNet Lightning keypoints. Simpler than oak_pose โ€” + gives you the "what" without all the raw coordinates. + + Returns: + Posture description (standing/sitting/unknown, facing camera, arms raised). + + Example: + oak_posture() + """ + try: + data = await api_get("/pose/summary") + active = data.get("active", False) + + if not active: + return "๐Ÿง Posture: No person detected" + + posture = data.get("posture", "unknown") + facing = data.get("facing_camera", False) + arms = data.get("arms_raised", False) + num_valid = data.get("num_valid", 0) + mean_conf = data.get("mean_confidence", 0) + + parts = [] + if posture != "unknown": + parts.append(posture) + parts.append("facing camera" if facing else "not facing camera") + if arms: + parts.append("arms raised") + + return f"""๐Ÿง Posture: {', '.join(parts)} +โ€ข Confidence: {mean_conf:.0%} ({num_valid}/17 keypoints)""" + except httpx.HTTPStatusError as e: + if e.response.status_code == 503: + return "๐Ÿง Pose estimator not available (Coral 2 not loaded)" + return f"โŒ Error getting posture: {e}" + except httpx.HTTPError as e: + return f"โŒ Error connecting to oak-service: {e}" + except Exception as e: + return f"โŒ Error: {e}" + + # ============== Face Recognition Tools ==============